From 9faeca0351bc52b7c1fb9986bc4a0823307e59ec Mon Sep 17 00:00:00 2001 From: kartikeyaagrawal <52993905+kartikeyaagrawal@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:10:08 +0530 Subject: [PATCH 001/255] chore(docker): reduce base_java17 and spark_base image size (#18542) Addresses #18523. Shrinks the Java 17 integration-test image from ~3.56 GB to ~2.58 GB (~27%) without changing any runtime behavior. The container commands, environment variables, exposed ports, and entrypoint are identical. base_java17/Dockerfile: - switch base image from eclipse-temurin:17-jdk to 17-jre-jammy. The container only runs Hadoop; no Java compilation happens inside it, so the JDK toolchain is not needed. - convert to a multi-stage build. Stage 1 downloads and extracts the Hadoop tarball; stage 2 only COPYs the extracted tree. curl, ca-certificates, and the tar.gz no longer land in the final layer. - use --no-install-recommends and clean apt lists in the runtime stage. - drop the unused .asc signature download and the now-dead wget dep. spark_base/Dockerfile: - replace the Python-3.10.14-from-source build (which pulled in build-essential and a full compile toolchain, then built CPython with --enable-optimizations inside the image) with the distro python3-minimal + python3-pip packages. PySpark only needs a Python runtime at runtime. (cherry picked from commit 0a28d695eb1558b01bb38b11996a436a2e3e23dd) --- docker/hoodie/hadoop/base_java17/Dockerfile | 41 +++++++++++++-------- docker/hoodie/hadoop/spark_base/Dockerfile | 21 ++--------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/docker/hoodie/hadoop/base_java17/Dockerfile b/docker/hoodie/hadoop/base_java17/Dockerfile index 45108610b19e2..a5c265d3aaace 100644 --- a/docker/hoodie/hadoop/base_java17/Dockerfile +++ b/docker/hoodie/hadoop/base_java17/Dockerfile @@ -15,7 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM eclipse-temurin:17-jdk +# --- Stage 1: fetch + extract Hadoop (throwaway) --- +FROM eclipse-temurin:17-jre-jammy AS hadoop-builder + +ARG HADOOP_VERSION=3.4.0 +ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN set -x \ + && DEBIAN_FRONTEND=noninteractive apt-get -yq update \ + && apt-get -yq install --no-install-recommends curl ca-certificates \ + && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ + && mkdir -p /opt \ + && tar -xzf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz \ + && mkdir -p /opt/hadoop-${HADOOP_VERSION}/logs + +# --- Stage 2: runtime --- +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="Hoodie" USER root @@ -23,28 +39,23 @@ USER root ENV LANG C.UTF-8 ARG HADOOP_VERSION=3.4.0 -ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -ENV HADOOP_VERSION ${HADOOP_VERSION} -ENV HADOOP_URL ${HADOOP_URL} +ENV HADOOP_VERSION=${HADOOP_VERSION} -RUN set -x \ - && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat-openbsd procps \ - && echo "Fetch URL2 is : ${HADOOP_URL}" \ - && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ - && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc \ - && mkdir -p /opt/hadoop-$HADOOP_VERSION/logs \ - && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ - && rm /tmp/hadoop.tar.gz* \ - && ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop \ +RUN DEBIAN_FRONTEND=noninteractive apt-get -yq update \ + && apt-get -yq install --no-install-recommends netcat-openbsd procps \ + && rm -rf /var/lib/apt/lists/* \ && mkdir /hadoop-data -ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION +COPY --from=hadoop-builder /opt/hadoop-${HADOOP_VERSION} /opt/hadoop-${HADOOP_VERSION} +RUN ln -s /opt/hadoop-${HADOOP_VERSION}/etc/hadoop /etc/hadoop + +ENV HADOOP_PREFIX=/opt/hadoop-${HADOOP_VERSION} ENV HADOOP_CONF_DIR=/etc/hadoop ENV MULTIHOMED_NETWORK=1 ENV HADOOP_HOME=${HADOOP_PREFIX} ENV HADOOP_INSTALL=${HADOOP_HOME} ENV USER=root -ENV PATH /usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH +ENV PATH=/usr/bin:/bin:${HADOOP_PREFIX}/bin/:$PATH # Exposing a union of ports across hadoop versions # Well known ports including ssh diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile index 68bfbaae76d83..4210faf21bfe6 100644 --- a/docker/hoodie/hadoop/spark_base/Dockerfile +++ b/docker/hoodie/hadoop/spark_base/Dockerfile @@ -41,23 +41,10 @@ RUN echo "Installing Spark-version (${SPARK_VERSION})" \ && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ && cd / -# Install build dependencies -RUN apt-get update && apt-get install -y \ - wget build-essential libncursesw5-dev \ - libssl-dev libgdbm-dev libreadline-dev libbz2-dev \ - libsqlite3-dev libffi-dev zlib1g-dev curl \ - && cd /usr/src \ - && wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz \ - && tar xzf Python-3.10.14.tgz \ - && cd Python-3.10.14 \ - && ./configure --enable-optimizations \ - && make -j"$(nproc)" \ - && make altinstall \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python3 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python \ - && pip install --upgrade pip \ - && cd / && rm -rf /usr/src/Python-3.10.14* \ +# Install Python runtime from distro package (avoids ~400MB build toolchain) +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3-minimal python3-pip \ + && ln -sf /usr/bin/python3 /usr/bin/python \ && rm -rf /var/lib/apt/lists/* #Give permission to execute scripts From d030eb17dd899d7f59d9f7d9a5c2e8cb1e633193 Mon Sep 17 00:00:00 2001 From: ashokkumar-allu <65997235+ashokkumar-allu@users.noreply.github.com> Date: Fri, 1 May 2026 12:32:02 -0500 Subject: [PATCH 002/255] test(spark): Add date logical type test to TestAvroConversionUtils (#18584) * [MINOR] Add date logical type test to TestAvroConversionUtils Add a test case that verifies createConverterToRow correctly handles Avro's date logical type (int with logicalType=date), ensuring the conversion from epoch-days integer to java.sql.Date preserves the correct date value. * [MINOR] Address review comments: rename convertor to converter, add comments --------- Co-authored-by: gallu (cherry picked from commit f7508ded95a534f5e6e5ffc5150a83a5662d05db) --- .../apache/hudi/TestAvroConversionUtils.scala | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala index 5f4ec78c13cf3..0430dd081237a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, DataTypes, M import org.scalatest.{FunSuite, Matchers} import java.nio.ByteBuffer +import java.time.LocalDate import java.util.Objects class TestAvroConversionUtils extends FunSuite with Matchers { @@ -456,4 +457,30 @@ class TestAvroConversionUtils extends FunSuite with Matchers { HoodieSchemaConversionUtils.convertStructTypeToHoodieSchema(struct, "SchemaName", "SchemaNS") } should have message "Duplicate field name in record SchemaNS.SchemaName: name type:UNION pos:2 and name type:UNION pos:1." } + + test("Logical type: date") { + val dateSchema = s""" + { + "namespace": "logical", + "type": "record", + "name": "test", + "fields": [ + {"name": "date", "type": {"type": "int", "logicalType": "date"}} + ] + } + """ + + val dateInputData = Seq(7, 365, 0) // one week, one year, epoch + val schema = HoodieSchema.parse(dateSchema) + val converter = AvroConversionUtils.createConverterToRow(schema, HoodieSchemaConversionUtils.convertHoodieSchemaToStructType(schema)) + + val dateOutputData = dateInputData.map(x => { + val record = new GenericData.Record(schema.toAvroSchema) {{ put("date", x) }} + converter(record).get(0) + }) + + assert(dateOutputData(0).toString === LocalDate.ofEpochDay(dateInputData(0)).toString) + assert(dateOutputData(1).toString === LocalDate.ofEpochDay(dateInputData(1)).toString) + assert(dateOutputData(2).toString === LocalDate.ofEpochDay(dateInputData(2)).toString) + } } From 69ec50cc07c48aba3af7bc72233881bd82cd8702 Mon Sep 17 00:00:00 2001 From: Krishen <22875197+kbuci@users.noreply.github.com> Date: Mon, 4 May 2026 13:33:22 -0700 Subject: [PATCH 003/255] feat(common): roll over commit metadata to clean (#18590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When rolling metadata is configured (hoodie.write.rolling.metadata.keys), important metadata like schema and checkpoint keys are carried forward across commits. However, clean instants do not participate in this rolling mechanism, they neither receive rolled-over metadata nor serve as a source for subsequent lookups. After archival removes old ingestion commits, if only clean instants remain on the active timeline between surviving commits, the chain of rolled-over metadata can break. This PR ensures that clean commits also carry rolled-over metadata in their extraMetadata field, preserving the rolling metadata chain across archival. --------- Co-authored-by: Krishen Bhan <“bkrishen@uber.com”> (cherry picked from commit 127c6ee031867a7a6b49c6f3ccadbba8161ee8ef) --- .../apache/hudi/client/BaseHoodieClient.java | 170 +++++++++++------- .../apache/hudi/config/HoodieWriteConfig.java | 19 +- .../action/clean/CleanActionExecutor.java | 6 +- .../common/table/timeline/HoodieTimeline.java | 5 + .../TestHoodieClientOnCopyOnWriteStorage.java | 86 +++++++++ 5 files changed, 210 insertions(+), 76 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java index 842dc38177a59..002ac1e7454f8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java @@ -18,6 +18,7 @@ package org.apache.hudi.client; +import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.callback.HoodieClientInitCallback; import org.apache.hudi.client.embedded.EmbeddedTimelineServerHelper; import org.apache.hudi.client.embedded.EmbeddedTimelineService; @@ -58,6 +59,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -313,19 +315,24 @@ protected boolean isStreamingWriteToMetadataEnabled(HoodieTable table) { } /** - * Merges rolling metadata from recent completed commits into the current commit metadata. + * Merges rolling metadata from recent completed instants into the current commit metadata. * This method MUST be called within the transaction lock after conflict resolution. * *

Rolling metadata keys configured via {@link HoodieWriteConfig#ROLLING_METADATA_KEYS} will be - * automatically carried forward from recent commits. The system walks back up to - * {@link HoodieWriteConfig#ROLLING_METADATA_TIMELINE_LOOKBACK_COMMITS} commits to find the most - * recent value for each key. This ensures that important metadata like checkpoint information - * remains accessible without worrying about archival or missing keys in individual commits. + * automatically carried forward from recent instants. The system walks back through completed + * commits and clean instants (in reverse completion-time order) up to + * {@link HoodieWriteConfig#ROLLING_METADATA_TIMELINE_LOOKBACK_COMMITS} to find the most + * recent value for each key. * * @param table HoodieTable instance (may have refreshed timeline after conflict resolution) * @param metadata Current commit metadata to be augmented with rolling metadata */ protected void mergeRollingMetadata(HoodieTable table, HoodieCommitMetadata metadata) { + // IMPORTANT: We're inside the lock here. The timeline in 'table' is either: + // 1. Fresh from createTable() if no conflict resolution happened + // 2. Reloaded during resolveWriteConflict() if conflicts were checked + // In both cases, we have the latest view of the timeline. + // Skip for metadata table - rolling metadata is only for data tables if (table.isMetadataTable()) { return; @@ -336,88 +343,119 @@ protected void mergeRollingMetadata(HoodieTable table, HoodieCommitMetadata meta return; // No rolling metadata configured } - // IMPORTANT: We're inside the lock here. The timeline in 'table' is either: - // 1. Fresh from createTable() if no conflict resolution happened - // 2. Reloaded during resolveWriteConflict() if conflicts were checked - // In both cases, we have the latest view of the timeline. + Map foundRollingMetadata = collectRollingMetadataFromTimeline(table, config, rollingKeys, metadata.getExtraMetadata()); + for (Map.Entry entry : foundRollingMetadata.entrySet()) { + metadata.addMetadata(entry.getKey(), entry.getValue()); + } + } - HoodieTimeline commitsTimeline = table.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(); + /** + * Overload of {@link #mergeRollingMetadata(HoodieTable, HoodieCommitMetadata)} for clean + * commits. Populates {@link HoodieCleanMetadata#getExtraMetadata()} with rolling metadata + * values found on the active timeline. + * + *

This is {@code public static} so that {@code CleanActionExecutor} (which does not extend + * {@code BaseHoodieClient}) can invoke it. + */ + public static void mergeRollingMetadata(HoodieTable table, HoodieWriteConfig config, HoodieCleanMetadata metadata) { + if (table.isMetadataTable()) { + return; + } + Set rollingKeys = config.getRollingMetadataKeys(); + if (rollingKeys.isEmpty()) { + return; + } - if (commitsTimeline.empty()) { - log.info("No previous commits found. Rolling metadata will start with current commit."); - return; // First commit - nothing to roll forward + Map existing = metadata.getExtraMetadata() != null + ? metadata.getExtraMetadata() : Collections.emptyMap(); + Map foundRollingMetadata = collectRollingMetadataFromTimeline(table, config, rollingKeys, existing); + if (!foundRollingMetadata.isEmpty()) { + Map merged = new HashMap<>(existing); + merged.putAll(foundRollingMetadata); + metadata.setExtraMetadata(merged); } + } - try { - Map existingExtraMetadata = metadata.getExtraMetadata(); - Map foundRollingMetadata = new HashMap<>(); - Set remainingKeys = new HashSet<>(rollingKeys); - - // Remove keys that are already present with non-empty values in current commit (current values take precedence) - for (String key : rollingKeys) { - if (existingExtraMetadata.containsKey(key) && !StringUtils.isNullOrEmpty(existingExtraMetadata.get(key))) { - remainingKeys.remove(key); - } - } + /** + * Walks backwards through completed instants (commits, replace-commits, delta-commits, and + * clean) on the active timeline, extracting extra-metadata values for the requested rolling + * keys. For commit-type instants the values come from {@link HoodieCommitMetadata#getMetadata}; + * for clean instants they come from {@link HoodieCleanMetadata#getExtraMetadata()}. + * + *

Keys already present with a non-empty value in {@code existingExtra} are skipped (empty + * strings are treated as "missing"). + */ + private static Map collectRollingMetadataFromTimeline( + HoodieTable table, HoodieWriteConfig config, + Set rollingKeys, Map existingExtra) { - if (remainingKeys.isEmpty()) { - log.debug("All rolling metadata keys are present in current commit. No walkback needed."); - return; - } + Map foundRollingMetadata = new HashMap<>(); + Set remaining = new HashSet<>(rollingKeys); - int lookbackLimit = config.getRollingMetadataTimelineLookbackCommits(); - int commitsWalkedBack = 0; + for (String key : rollingKeys) { + if (existingExtra.containsKey(key) && !StringUtils.isNullOrEmpty(existingExtra.get(key))) { + remaining.remove(key); + } + } + if (remaining.isEmpty()) { + log.debug("All rolling metadata keys already present. No walkback needed."); + return foundRollingMetadata; + } - // Walk back through the timeline in reverse order (most recent first) to find values for all remaining keys - List recentCommits = commitsTimeline.getReverseOrderedInstantsByCompletionTime() - .limit(lookbackLimit) - .collect(Collectors.toList()); + int lookbackLimit = config.getRollingMetadataTimelineLookbackCommits(); + HoodieTimeline completed = table.getActiveTimeline().filterCompletedInstants(); + List instants = completed.getReverseOrderedInstantsByCompletionTime() + .filter(i -> HoodieTimeline.VALID_ACTIONS_FOR_ROLLING_METADATA.contains(i.getAction())) + .limit(lookbackLimit) + .collect(Collectors.toList()); - log.debug("Walking back up to {} commits to find rolling metadata for keys: {}", - lookbackLimit, remainingKeys); + log.debug("Walking back up to {} instants to find rolling metadata for keys: {}", lookbackLimit, remaining); + int instantsWalkedBack = 0; - for (HoodieInstant instant : recentCommits) { - if (remainingKeys.isEmpty()) { - break; // Found all keys + try { + for (HoodieInstant instant : instants) { + if (remaining.isEmpty()) { + break; } + String action = instant.getAction(); + Map extraMeta = null; - commitsWalkedBack++; - HoodieCommitMetadata commitMetadata = table.getMetaClient().getActiveTimeline().readInstantContent(instant, HoodieCommitMetadata.class); + if (HoodieTimeline.CLEAN_ACTION.equals(action)) { + HoodieCleanMetadata cleanMeta = table.getActiveTimeline().readCleanMetadata(instant); + extraMeta = cleanMeta.getExtraMetadata(); + } else { + HoodieCommitMetadata commitMeta = table.getMetaClient().getActiveTimeline() + .readInstantContent(instant, HoodieCommitMetadata.class); + extraMeta = commitMeta.getExtraMetadata(); + } + instantsWalkedBack++; - // Check for remaining keys in this commit - for (String key : new HashSet<>(remainingKeys)) { - String value = commitMetadata.getMetadata(key); + if (extraMeta == null) { + continue; + } + for (String key : new HashSet<>(remaining)) { + String value = extraMeta.get(key); if (!StringUtils.isNullOrEmpty(value)) { foundRollingMetadata.put(key, value); - remainingKeys.remove(key); - log.debug("Found rolling metadata key '{}' in commit {} with value: {}", - key, instant.requestedTime(), value); + remaining.remove(key); + log.debug("Found rolling metadata key '{}' in {} instant {} with value: {}", + key, action, instant.requestedTime(), value); } } } - // Add found rolling metadata to current commit - for (Map.Entry entry : foundRollingMetadata.entrySet()) { - metadata.addMetadata(entry.getKey(), entry.getValue()); - } - - int rolledForwardCount = foundRollingMetadata.size(); - int updatedCount = rollingKeys.size() - remainingKeys.size() - rolledForwardCount; - - if (rolledForwardCount > 0 || updatedCount > 0 || !remainingKeys.isEmpty()) { - log.info("Rolling metadata merge completed. Walked back {} commits. " - + "Rolled forward: {}, Updated in current: {}, Not found: {}, Total rolling keys: {}", - commitsWalkedBack, rolledForwardCount, updatedCount, remainingKeys.size(), rollingKeys.size()); + if (!foundRollingMetadata.isEmpty() || !remaining.isEmpty()) { + log.info("Rolling metadata: walked {} instants. Rolled forward: {}, Not found: {}, Total keys: {}", + instantsWalkedBack, foundRollingMetadata.size(), remaining.size(), rollingKeys.size()); } - - if (!remainingKeys.isEmpty()) { - log.warn("Rolling metadata keys not found in last {} commits: {}. " - + "These keys will not be included in the current commit.", lookbackLimit, remainingKeys); + if (!remaining.isEmpty()) { + log.warn("Rolling metadata keys not found in last {} instants: {}.", instantsWalkedBack, remaining); } - } catch (IOException e) { - log.error("Failed to read previous commit metadata for rolling metadata keys: {}.", rollingKeys, e); - throw new HoodieIOException("Failed to read previous commit metadata for rolling metadata keys: " + rollingKeys, e); + log.error("Failed to read previous metadata for rolling metadata keys: {}.", rollingKeys, e); + throw new HoodieIOException("Failed to read previous metadata for rolling keys: " + rollingKeys, e); } + + return foundRollingMetadata; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 5df834121bf90..a663c63dba780 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -768,21 +768,22 @@ public class HoodieWriteConfig extends HoodieConfig { .markAdvanced() .sinceVersion("1.2.0") .withDocumentation("Comma-separated list of extra metadata keys that should be automatically carried forward " - + "to every new commit. These keys will be read from recent commit metadata and included in new commits, " - + "ensuring they remain accessible without walking the timeline or worrying about archival. " - + "This is useful for tracking checkpoint information (e.g., Kafka offsets, Flink checkpoints) or any metadata " - + "that needs to persist across commits. New values override old ones. Only applies to data table commits."); + + "to every new commit and clean instant. These keys will be read from recent commit and clean metadata " + + "and included in new commits/cleans, ensuring they remain accessible without walking the timeline or " + + "worrying about archival. This is useful for tracking checkpoint information (e.g., Kafka offsets, " + + "Flink checkpoints) or any metadata that needs to persist across commits. New values override old ones. " + + "Only applies to data table commits and clean instants."); public static final ConfigProperty ROLLING_METADATA_TIMELINE_LOOKBACK_COMMITS = ConfigProperty .key("hoodie.write.rolling.metadata.timeline.lookback.commits") .defaultValue(10) .markAdvanced() .sinceVersion("1.2.0") - .withDocumentation("Maximum number of completed commits to walk back in the timeline when searching for " - + "rolling metadata keys. If a rolling metadata key is not found in the latest commit, the system will " - + "walk back up to this many commits to find the most recent value. This ensures rolling metadata is " - + "preserved even if some commits don't update all keys. Higher values provide more resilience but may " - + "impact performance. Only applies when hoodie.write.rolling.metadata.keys is configured."); + .withDocumentation("Maximum number of completed instants (commits and clean) to walk back in the timeline " + + "when searching for rolling metadata keys. If a rolling metadata key is not found in the latest instant, " + + "the system will walk back up to this many instants to find the most recent value. This ensures rolling " + + "metadata is preserved even if some instants don't carry all keys. Higher values provide more resilience " + + "but may impact performance. Only applies when hoodie.write.rolling.metadata.keys is configured."); public static final ConfigProperty ALLOW_OPERATION_METADATA_FIELD = ConfigProperty .key("hoodie.allow.operation.metadata.field") diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java index 3fad30b22bdf0..63a6e7d8f994d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java @@ -21,6 +21,7 @@ import org.apache.hudi.avro.model.HoodieActionInstant; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanerPlan; +import org.apache.hudi.client.BaseHoodieClient; import org.apache.hudi.client.transaction.TransactionManager; import org.apache.hudi.common.HoodieCleanStat; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -215,7 +216,6 @@ private HoodieCleanMetadata runClean(HoodieTable table, HoodieInstan } List cleanStats = clean(context, cleanerPlan); - table.getMetaClient().reloadActiveTimeline(); HoodieCleanMetadata metadata; if (cleanStats.isEmpty()) { metadata = createEmptyCleanMetadata(cleanerPlan, inflightInstant, timer.endTimer()); @@ -228,6 +228,10 @@ private HoodieCleanMetadata runClean(HoodieTable table, HoodieInstan ); } this.txnManager.beginStateChange(Option.of(inflightInstant), Option.empty()); + // Reload inside the lock so mergeRollingMetadata reads the latest timeline, + // matching the same contract as mergeRollingMetadata for commit metadata. + table.getMetaClient().reloadActiveTimeline(); + BaseHoodieClient.mergeRollingMetadata(table, config, metadata); writeTableMetadata(metadata, inflightInstant.requestedTime()); table.getActiveTimeline().transitionCleanInflightToComplete( false, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java index f9a59c6ab6233..4ad8ec769ac32 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java @@ -30,6 +30,7 @@ import org.apache.hudi.avro.model.HoodieSavepointMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.storage.HoodieInstantWriter; @@ -77,6 +78,10 @@ public interface HoodieTimeline extends HoodieInstantReader, Serializable { CLEAN_ACTION, SAVEPOINT_ACTION, RESTORE_ACTION, ROLLBACK_ACTION, COMPACTION_ACTION, LOG_COMPACTION_ACTION, REPLACE_COMMIT_ACTION, CLUSTERING_ACTION, INDEXING_ACTION}; + Set VALID_ACTIONS_FOR_ROLLING_METADATA = CollectionUtils.createSet( + COMMIT_ACTION, DELTA_COMMIT_ACTION, CLEAN_ACTION, + COMPACTION_ACTION, LOG_COMPACTION_ACTION, REPLACE_COMMIT_ACTION, CLUSTERING_ACTION); + String COMMIT_EXTENSION = "." + COMMIT_ACTION; String DELTA_COMMIT_EXTENSION = "." + DELTA_COMMIT_ACTION; String CLEAN_EXTENSION = "." + CLEAN_ACTION; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 2a4fa731e04bb..ffb829165c7b1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -18,6 +18,7 @@ package org.apache.hudi.client.functional; +import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; import org.apache.hudi.client.BaseHoodieWriteClient; @@ -68,6 +69,7 @@ import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.FileFormatUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieCleanConfig; @@ -2066,6 +2068,90 @@ public void testRollingMetadataPreservedAcrossClusteringAfterArchival() throws E "TableSchemaResolver should find schema even with clustering-only timeline"); } + @Test + public void testRollingMetadataPreservedInCleanCommits() throws Exception { + String schemaKey = HoodieCommitMetadata.SCHEMA_KEY; + dataGen = new HoodieTestDataGenerator(new String[] {DEFAULT_FIRST_PARTITION_PATH}); + + HoodieWriteConfig config = getConfigBuilder(TRIP_EXAMPLE_SCHEMA) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .compactionSmallFileSize(0).build()) + .withRollingMetadataKeys(schemaKey) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withAutoClean(false) + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .retainCommits(1) + .build()) + .withArchivalConfig(HoodieArchivalConfig.newBuilder() + .archiveCommitsWith(10, 12).build()) + .build(); + + SparkRDDWriteClient client = getHoodieWriteClient(config); + + String firstCommit = client.startCommit(); + List records = dataGen.generateInserts(firstCommit, 100); + JavaRDD firstResult = client.insert(jsc.parallelize(records, 1), firstCommit); + client.commit(firstCommit, firstResult); + + // Upsert several times to create superseded file versions + for (int i = 0; i < 4; i++) { + String commitTime = client.startCommit(); + List updates = dataGen.generateUpdates(commitTime, records); + JavaRDD result = client.upsert(jsc.parallelize(updates, 1), commitTime); + client.commit(commitTime, result); + } + + // Run clean — retainCommits(1) means old file versions from earlier commits are eligible + HoodieCleanMetadata cleanResult = client.clean(); + assertTrue(cleanResult != null, "Clean should produce metadata (files to clean exist)"); + + HoodieTableMetaClient freshMeta = HoodieTableMetaClient.reload(metaClient); + HoodieTimeline cleanTimeline = freshMeta.getActiveTimeline() + .getCleanerTimeline().filterCompletedInstants(); + assertFalse(cleanTimeline.empty(), "Should have at least one clean instant"); + + HoodieInstant lastClean = cleanTimeline.lastInstant().get(); + HoodieCleanMetadata cleanMetadata = cleanTimeline.readCleanMetadata(lastClean); + + Map cleanExtraMetadata = cleanMetadata.getExtraMetadata(); + assertTrue(cleanExtraMetadata != null, "Clean metadata should have extraMetadata map"); + assertTrue(cleanExtraMetadata.containsKey(schemaKey), + "Clean's extraMetadata should contain rolled-over schema key"); + assertFalse(cleanExtraMetadata.get(schemaKey).isEmpty(), + "Rolled-over schema in clean should be non-empty"); + + // Now make one more commit with lookback=1. The rolling metadata walk should find + // the schema key in the clean instant (the most recent instant with the key). + HoodieWriteConfig lookback1Config = getConfigBuilder(TRIP_EXAMPLE_SCHEMA) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .compactionSmallFileSize(0).build()) + .withRollingMetadataKeys(schemaKey) + .withRollingMetadataTimelineLookbackCommits(1) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withAutoClean(false) + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .retainCommits(1) + .build()) + .withArchivalConfig(HoodieArchivalConfig.newBuilder() + .archiveCommitsWith(10, 12).build()) + .build(); + + SparkRDDWriteClient lookbackClient = getHoodieWriteClient(lookback1Config); + String nextCommit = lookbackClient.startCommit(); + List nextUpdates = dataGen.generateUpdates(nextCommit, records); + JavaRDD nextResult = lookbackClient.upsert(jsc.parallelize(nextUpdates, 1), nextCommit); + lookbackClient.commit(nextCommit, nextResult); + + freshMeta = HoodieTableMetaClient.reload(metaClient); + HoodieTimeline commitsTimeline = freshMeta.getActiveTimeline() + .getCommitsTimeline().filterCompletedInstants(); + HoodieInstant latestCommit = commitsTimeline.lastInstant().get(); + HoodieCommitMetadata latestMeta = commitsTimeline.readCommitMetadata(latestCommit); + String rolledSchema = latestMeta.getMetadata(schemaKey); + assertFalse(StringUtils.isNullOrEmpty(rolledSchema), + "Schema should be rolled over into new commit even with lookback=1"); + } + /** * Disabling row writer here as clustering tests will throw the error below if it is used. * java.util.concurrent.CompletionException: java.lang.ClassNotFoundException From 481eb8047c65d431c1994a780c4196e7ad888a45 Mon Sep 17 00:00:00 2001 From: Surya Prasanna Date: Mon, 4 May 2026 13:42:43 -0700 Subject: [PATCH 004/255] refactor: move checkpoint metadata lookup helper to hudi-common (#18489) This PR moves the checkpoint metadata lookup helper into hudi-common so ingestion-related code can reuse the same timeline utility instead of keeping the logic in utilities-only code. (cherry picked from commit 471bb48338bd9642bd593472b029915180156038) --- .../common/table/timeline/TimelineUtils.java | 27 +++++++++++++++++++ .../streamer/StreamerCheckpointUtils.java | 22 +++++---------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java index cfc1520b23933..4b2e6ee55fc62 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java @@ -49,6 +49,7 @@ import java.io.InputStream; import java.text.ParseException; import java.util.AbstractMap; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -647,4 +648,30 @@ public static Option getHoodieInstantWriterOption(Hoodi } return writerOption; } + + /** + * Returns the latest reverse-ordered instant whose commit metadata contains at least one of the provided + * checkpoint metadata keys. + * + * @param timeline timeline to scan; expected to contain completed instants + * @param checkpointKeys checkpoint metadata keys to look for in commit metadata + * @return an {@link Option} containing a {@link Pair} of the matching instant's + * {@link HoodieInstant#toString()} value and its + * {@link HoodieCommitMetadata}; {@link Option#empty()} if no matching instant is found + * @throws IOException if reading commit metadata fails + */ + @SuppressWarnings("unchecked") + public static Option> getLatestInstantAndCommitMetadataWithValidCheckpointInfo( + HoodieTimeline timeline, String... checkpointKeys) throws IOException { + return (Option>) timeline.getReverseOrderedInstants().map(instant -> { + try { + HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(instant); + boolean hasCheckpointMetadata = Arrays.stream(checkpointKeys) + .anyMatch(key -> !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(key))); + return hasCheckpointMetadata ? Option.of(Pair.of(instant.toString(), commitMetadata)) : Option.empty(); + } catch (IOException e) { + throw new HoodieIOException("Failed to parse HoodieCommitMetadata for " + instant.toString(), e); + } + }).filter(Option::isPresent).findFirst().orElse(Option.empty()); + } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java index 03db67b9d03f0..560d9fd2172a8 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.table.checkpoint.CheckpointUtils; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.TimelineUtils; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; @@ -201,21 +202,12 @@ private static boolean ignoreCkpCfgPrevailsOverCkpFromPrevCommit(HoodieStreamer. public static Option> getLatestInstantAndCommitMetadataWithValidCheckpointInfo(HoodieTimeline timeline) throws IOException { - return (Option>) timeline.getReverseOrderedInstants().map(instant -> { - try { - HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(instant); - if (!StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_KEY)) - || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_RESET_KEY)) - || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(STREAMER_CHECKPOINT_KEY_V2)) - || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(STREAMER_CHECKPOINT_RESET_KEY_V2))) { - return Option.of(Pair.of(instant.toString(), commitMetadata)); - } else { - return Option.empty(); - } - } catch (IOException e) { - throw new HoodieIOException("Failed to parse HoodieCommitMetadata for " + instant.toString(), e); - } - }).filter(Option::isPresent).findFirst().orElse(Option.empty()); + return TimelineUtils.getLatestInstantAndCommitMetadataWithValidCheckpointInfo( + timeline, + HoodieStreamer.CHECKPOINT_KEY, + HoodieStreamer.CHECKPOINT_RESET_KEY, + STREAMER_CHECKPOINT_KEY_V2, + STREAMER_CHECKPOINT_RESET_KEY_V2); } public static Option getLatestCommitMetadataWithValidCheckpointInfo(HoodieTimeline timeline) throws IOException { From b5342e756b838b9f562e7e8007eab6f4a897d524 Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Wed, 6 May 2026 23:23:48 -0700 Subject: [PATCH 005/255] feat(flink): Backport Flink 2.1 nested Parquet column readers and INT64 timestamp dispatch (FLINK-35702) (#18636) * feat(flink): Backport Flink 2.1 nested Parquet column readers and INT64 timestamp dispatch (FLINK-35702) * Minor fixes (cherry picked from commit 40295605aae691da07e93a1a2919d38b41cbb6dd) --- .../format/cow/vector/HeapArrayVector.java | 31 + .../cow/vector/HeapMapColumnVector.java | 63 +- .../cow/vector/HeapRowColumnVector.java | 15 + .../cow/vector/reader/NestedColumnReader.java | 257 +++++++ .../reader/NestedPrimitiveColumnReader.java | 639 ++++++++++++++++++ .../ParquetDataColumnReaderFactory.java | 108 ++- .../vector/TestHeapColumnVectorAccessors.java | 138 ++++ .../TestParquetDataColumnReaderFactory.java | 272 ++++++++ 8 files changed, 1516 insertions(+), 7 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java index a0dced01e5e8d..2f21a323302f1 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java @@ -57,6 +57,37 @@ public int getLen() { return this.isNull.length; } + // --------------------------------------------------------------------------------------------- + // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths} + // and {@code child} fields so legacy callers continue to work; the new {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + // future Flink-2.1-style caller use these accessors. + // --------------------------------------------------------------------------------------------- + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public ColumnVector getChild() { + return child; + } + + public void setChild(ColumnVector child) { + this.child = child; + } + @Override public ArrayData getArray(int i) { long offset = offsets[i]; diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index 0d83f82baedf3..a98bdebd707a7 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -20,6 +20,7 @@ import lombok.Getter; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; @@ -31,16 +32,74 @@ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { @Getter - private final WritableColumnVector keys; + private WritableColumnVector keys; @Getter - private final WritableColumnVector values; + private WritableColumnVector values; + + // --------------------------------------------------------------------------------------------- + // Flink 2.1 Dremel-style state. Populated by {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port). The + // legacy {@link #getMap(int)} implementation below continues to use {@code ColumnarGroupMapData} + // — wiring it through these offsets/lengths happens in a follow-up PR that switches the read + // path. Left here so the new readers can compile against the additive surface. + // --------------------------------------------------------------------------------------------- + private long[] offsets; + private long[] lengths; + private int size; public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) { super(len); + this.offsets = new long[len]; + this.lengths = new long[len]; this.keys = keys; this.values = values; } + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public void setKeys(WritableColumnVector keys) { + this.keys = keys; + } + + public void setValues(WritableColumnVector values) { + this.values = values; + } + + /** + * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract + * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}. + */ + public ColumnVector getKeyColumnVector() { + return keys; + } + + /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */ + public ColumnVector getValueColumnVector() { + return values; + } + @Override public MapData getMap(int rowId) { return new ColumnarGroupMapData(keys, values, rowId); diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java index ae194e4e6ab05..0c640ce92ee40 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java @@ -37,6 +37,21 @@ public HeapRowColumnVector(int len, WritableColumnVector... vectors) { this.vectors = vectors; } + /** + * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code + * vectors} field so legacy callers continue to work; the new {@link + * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + * future Flink-2.1-style caller use this accessor. + */ + public WritableColumnVector[] getFields() { + return vectors; + } + + /** Counterpart of {@link #getFields()}. */ + public void setFields(WritableColumnVector[] fields) { + this.vectors = fields; + } + @Override public ColumnarRowData getRow(int i) { ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors)); diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java new file mode 100644 index 0000000000000..27eab298b3207 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.NestedPositionUtil; +import org.apache.hudi.table.format.cow.vector.HeapArrayVector; +import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; +import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.Preconditions; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code + * Row}). Resolves nested structures using Dremel striping/assembly; see the + * striping and assembly algorithms from the Dremel paper. + * + *

Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream: + * + *

+ */ +public class NestedColumnReader implements ColumnReader { + + private final Map columnReaders; + private final boolean isUtcTimestamp; + + private final PageReadStore pages; + + private final ParquetField field; + + public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) { + this.isUtcTimestamp = isUtcTimestamp; + this.pages = pages; + this.field = field; + this.columnReaders = new HashMap<>(); + } + + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + readData(field, readNumber, vector, false); + } + + private Tuple2 readData( + ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException { + if (field.getType() instanceof RowType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) { + return readMap((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof ArrayType) { + return readArray((ParquetGroupField) field, readNumber, vector, inside); + } else { + return readPrimitive((ParquetPrimitiveField) field, readNumber, vector); + } + } + + private Tuple2 readRow( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector; + LevelDelegation levelDelegation = null; + List children = field.getChildren(); + WritableColumnVector[] childrenVectors = heapRowVector.getFields(); + WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length]; + for (int i = 0; i < children.size(); i++) { + ParquetField child = children.get(i); + if (child == null) { + // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // vector is expected to be pre-populated with nulls by the caller (lands in a follow-up + // PR that rewires ParquetSplitReaderUtil); keep it as is and skip contributing to the + // level stream. + finalChildrenVectors[i] = childrenVectors[i]; + continue; + } + Tuple2 tuple = + readData(child, readNumber, childrenVectors[i], true); + levelDelegation = tuple.f0; + finalChildrenVectors[i] = tuple.f1; + } + if (levelDelegation == null) { + throw new FlinkRuntimeException( + String.format("Row field does not have any non-null children: %s.", field)); + } + + RowPosition rowPosition = + NestedPositionUtil.calculateRowOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If row was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors); + } else { + heapRowVector.setFields(finalChildrenVectors); + } + + if (rowPosition.getIsNull() != null) { + setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); + } + return Tuple2.of(levelDelegation, heapRowVector); + } + + private Tuple2 readMap( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapMapColumnVector mapVector = (HeapMapColumnVector) vector; + mapVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 2, + "Maps must have two type parameters, found %s", + children.size()); + Tuple2 keyTuple = + readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true); + Tuple2 valueTuple = + readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true); + + LevelDelegation levelDelegation = keyTuple.f0; + + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If map was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1); + } else { + mapVector.setKeys(keyTuple.f1); + mapVector.setValues(valueTuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), mapVector); + } + + mapVector.setLengths(collectionPosition.getLength()); + mapVector.setOffsets(collectionPosition.getOffsets()); + + return Tuple2.of(levelDelegation, mapVector); + } + + private Tuple2 readArray( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapArrayVector arrayVector = (HeapArrayVector) vector; + arrayVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 1, + "Arrays must have a single type parameter, found %s", + children.size()); + Tuple2 tuple = + readData(children.get(0), readNumber, arrayVector.getChild(), true); + + LevelDelegation levelDelegation = tuple.f0; + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If array was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1); + } else { + arrayVector.setChild(tuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), arrayVector); + } + arrayVector.setLengths(collectionPosition.getLength()); + arrayVector.setOffsets(collectionPosition.getOffsets()); + return Tuple2.of(levelDelegation, arrayVector); + } + + private Tuple2 readPrimitive( + ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException { + ColumnDescriptor descriptor = field.getDescriptor(); + NestedPrimitiveColumnReader reader = columnReaders.get(descriptor); + if (reader == null) { + reader = + new NestedPrimitiveColumnReader( + descriptor, + pages.getPageReader(descriptor), + isUtcTimestamp, + descriptor.getPrimitiveType(), + field.getType()); + columnReaders.put(descriptor, reader); + } + WritableColumnVector writableColumnVector = + reader.readAndNewVector(readNumber, (WritableColumnVector) vector); + return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); + } + + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { + for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { + if (nullFlags[index]) { + vector.setNullAt(index); + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java new file mode 100644 index 0000000000000..3f0aefe2af74f --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -0,0 +1,639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.IntArrayList; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; + +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.LogicalType; + +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; +import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; +import static org.apache.parquet.column.ValuesType.VALUES; + +/** + * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure. + * + *

Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package + * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} + * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code + * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- + * reader creation boundary in {@code ParquetSplitReaderUtil} (lands in a follow-up PR), not inside + * this class — keeping it a faithful copy of upstream. + */ +public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); + + private final IntArrayList repetitionLevelList = new IntArrayList(0); + private final IntArrayList definitionLevelList = new IntArrayList(0); + + private final PageReader pageReader; + private final ColumnDescriptor descriptor; + private final Type type; + private final LogicalType logicalType; + + /** The dictionary, if this column has dictionary encoding. */ + private final ParquetDataColumnReader dictionary; + + /** Maximum definition level for this column. */ + private final int maxDefLevel; + + private boolean isUtcTimestamp; + + /** Total number of values read. */ + private long valuesRead; + + /** + * value that indicates the end of the current page. That is, if valuesRead == + * endOfPageValueCount, we are at the end of the page. + */ + private long endOfPageValueCount; + + /** If true, the current page is dictionary encoded. */ + private boolean isCurrentPageDictionaryEncoded; + + private int definitionLevel; + private int repetitionLevel; + + /** Repetition/Definition/Value readers. */ + private IntIterator repetitionLevelColumn; + + private IntIterator definitionLevelColumn; + private ParquetDataColumnReader dataColumn; + + /** Total values in the current page. */ + private int pageValueCount; + + // flag to indicate if there is no data in parquet data page + private boolean eof = false; + + private boolean isFirstRow = true; + + private Object lastValue; + + public NestedPrimitiveColumnReader( + ColumnDescriptor descriptor, + PageReader pageReader, + boolean isUtcTimestamp, + Type parquetType, + LogicalType logicalType) + throws IOException { + this.descriptor = descriptor; + this.type = parquetType; + this.pageReader = pageReader; + this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.isUtcTimestamp = isUtcTimestamp; + this.logicalType = logicalType; + + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); + if (dictionaryPage != null) { + try { + this.dictionary = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + parquetType.asPrimitiveType(), + dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } catch (IOException e) { + throw new IOException( + String.format("Could not decode the dictionary for %s", descriptor), e); + } + } else { + this.dictionary = null; + this.isCurrentPageDictionaryEncoded = false; + } + } + + // Not invoked directly; callers use readAndNewVector instead. + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + throw new UnsupportedOperationException("This function should not be called."); + } + + public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector) + throws IOException { + if (isFirstRow) { + if (!readValue()) { + return vector; + } + isFirstRow = false; + } + + // index to set value. + int index = 0; + int valueIndex = 0; + List valueList = new ArrayList<>(); + + // repeated type need two loops to read data. + while (!eof && index < readNumber) { + do { + valueList.add(lastValue); + valueIndex++; + } while (readValue() && (repetitionLevel != 0)); + index++; + } + + return fillColumnVector(valueIndex, valueList); + } + + public LevelDelegation getLevelDelegation() { + int[] repetition = repetitionLevelList.toArray(); + int[] definition = definitionLevelList.toArray(); + repetitionLevelList.clear(); + definitionLevelList.clear(); + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + return new LevelDelegation(repetition, definition); + } + + private boolean readValue() throws IOException { + int left = readPageIfNeed(); + if (left > 0) { + // get the values of repetition and definitionLevel + readAndSaveRepetitionAndDefinitionLevels(); + // read the data if it isn't null + if (definitionLevel == maxDefLevel) { + if (isCurrentPageDictionaryEncoded) { + int dictionaryId = dataColumn.readValueDictionaryId(); + lastValue = dictionaryDecodeValue(logicalType, dictionaryId); + } else { + lastValue = readPrimitiveTypedRow(logicalType); + } + } else { + lastValue = null; + } + return true; + } else { + eof = true; + return false; + } + } + + private void readAndSaveRepetitionAndDefinitionLevels() { + // get the values of repetition and definitionLevel + repetitionLevel = repetitionLevelColumn.nextInt(); + definitionLevel = definitionLevelColumn.nextInt(); + valuesRead++; + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + } + + private int readPageIfNeed() throws IOException { + // Compute the number of values we want to read in this page. + int leftInPage = (int) (endOfPageValueCount - valuesRead); + if (leftInPage == 0) { + // no data left in current page, load data from new page + readPage(); + leftInPage = (int) (endOfPageValueCount - valuesRead); + } + return leftInPage; + } + + private Object readPrimitiveTypedRow(LogicalType category) { + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dataColumn.readBytes(); + case BOOLEAN: + return dataColumn.readBoolean(); + case TIME_WITHOUT_TIME_ZONE: + case DATE: + case INTEGER: + return dataColumn.readInteger(); + case TINYINT: + return dataColumn.readTinyInt(); + case SMALLINT: + return dataColumn.readSmallInt(); + case BIGINT: + return dataColumn.readLong(); + case FLOAT: + return dataColumn.readFloat(); + case DOUBLE: + return dataColumn.readDouble(); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dataColumn.readInteger(); + case INT64: + return dataColumn.readLong(); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return dataColumn.readBytes(); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dataColumn.readTimestamp(); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { + if (dictionaryValue == null) { + return null; + } + + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dictionary.readBytes(dictionaryValue); + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTEGER: + return dictionary.readInteger(dictionaryValue); + case BOOLEAN: + return dictionary.readBoolean(dictionaryValue) ? 1 : 0; + case DOUBLE: + return dictionary.readDouble(dictionaryValue); + case FLOAT: + return dictionary.readFloat(dictionaryValue); + case TINYINT: + return dictionary.readTinyInt(dictionaryValue); + case SMALLINT: + return dictionary.readSmallInt(dictionaryValue); + case BIGINT: + return dictionary.readLong(dictionaryValue); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dictionary.readInteger(dictionaryValue); + case INT64: + return dictionary.readLong(dictionaryValue); + case FIXED_LEN_BYTE_ARRAY: + case BINARY: + return dictionary.readBytes(dictionaryValue); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dictionary.readTimestamp(dictionaryValue); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private WritableColumnVector fillColumnVector(int total, List valueList) { + switch (logicalType.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + HeapBytesVector heapBytesVector = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (src == null) { + heapBytesVector.setNullAt(i); + } else { + heapBytesVector.appendBytes(i, src, 0, src.length); + } + } + return heapBytesVector; + case BOOLEAN: + HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapBooleanVector.setNullAt(i); + } else { + heapBooleanVector.vector[i] = ((List) valueList).get(i); + } + } + return heapBooleanVector; + case TINYINT: + HeapByteVector heapByteVector = new HeapByteVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapByteVector.setNullAt(i); + } else { + heapByteVector.vector[i] = (byte) ((List) valueList).get(i).intValue(); + } + } + return heapByteVector; + case SMALLINT: + HeapShortVector heapShortVector = new HeapShortVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapShortVector.setNullAt(i); + } else { + heapShortVector.vector[i] = (short) ((List) valueList).get(i).intValue(); + } + } + return heapShortVector; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + HeapIntVector heapIntVector = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapIntVector.setNullAt(i); + } else { + heapIntVector.vector[i] = ((List) valueList).get(i); + } + } + return heapIntVector; + case FLOAT: + HeapFloatVector heapFloatVector = new HeapFloatVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapFloatVector.setNullAt(i); + } else { + heapFloatVector.vector[i] = ((List) valueList).get(i); + } + } + return heapFloatVector; + case BIGINT: + HeapLongVector heapLongVector = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapLongVector.setNullAt(i); + } else { + heapLongVector.vector[i] = ((List) valueList).get(i); + } + } + return heapLongVector; + case DOUBLE: + HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapDoubleVector.setNullAt(i); + } else { + heapDoubleVector.vector[i] = ((List) valueList).get(i); + } + } + return heapDoubleVector; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapTimestampVector.setNullAt(i); + } else { + heapTimestampVector.setTimestamp(i, ((List) valueList).get(i)); + } + } + return heapTimestampVector; + case DECIMAL: + PrimitiveType.PrimitiveTypeName primitiveTypeName = + descriptor.getPrimitiveType().getPrimitiveTypeName(); + switch (primitiveTypeName) { + case INT32: + HeapIntVector phiv = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phiv.setNullAt(i); + } else { + phiv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phiv); + case INT64: + HeapLongVector phlv = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phlv.setNullAt(i); + } else { + phlv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phlv); + default: + HeapBytesVector phbv = getHeapBytesVector(total, valueList); + return new ParquetDecimalVector(phbv); + } + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static HeapBytesVector getHeapBytesVector(int total, List valueList) { + HeapBytesVector phbv = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (valueList.get(i) == null) { + phbv.setNullAt(i); + } else { + phbv.appendBytes(i, src, 0, src.length); + } + } + return phbv; + } + + protected void readPage() { + DataPage page = pageReader.readPage(); + + if (page == null) { + return; + } + + page.accept( + new DataPage.Visitor() { + @Override + public Void visit(DataPageV1 dataPageV1) { + readPageV1(dataPageV1); + return null; + } + + @Override + public Void visit(DataPageV2 dataPageV2) { + readPageV2(dataPageV2); + return null; + } + }); + } + + private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) + throws IOException { + this.pageValueCount = valueCount; + this.endOfPageValueCount = valuesRead + pageValueCount; + if (dataEncoding.usesDictionary()) { + this.dataColumn = null; + if (dictionary == null) { + throw new IOException( + String.format( + "Could not read page in col %s because the dictionary was missing for encoding %s.", + descriptor, dataEncoding)); + } + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getDictionaryBasedValuesReader( + descriptor, VALUES, dictionary.getDictionary()), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } else { + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getValuesReader(descriptor, VALUES), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = false; + } + + try { + dataColumn.initFromPage(pageValueCount, in); + } catch (IOException e) { + throw new IOException(String.format("Could not read page in col %s.", descriptor), e); + } + } + + private void readPageV1(DataPageV1 page) { + ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); + ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); + this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); + this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); + try { + BytesInput bytes = page.getBytes(); + LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount); + ByteBufferInputStream in = bytes.toInputStream(); + LOG.debug("Reading repetition levels at {}.", in.position()); + rlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading definition levels at {}.", in.position()); + dlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading data at {}.", in.position()); + initDataReader(page.getValueEncoding(), in, page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private void readPageV2(DataPageV2 page) { + this.pageValueCount = page.getValueCount(); + this.repetitionLevelColumn = + newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); + this.definitionLevelColumn = + newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); + try { + LOG.debug( + "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); + initDataReader( + page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) { + try { + if (maxLevel == 0) { + return new NullIntIterator(); + } + return new RLEIntIterator( + new RunLengthBitPackingHybridDecoder( + BytesUtils.getWidthFromMaxInt(maxLevel), + new ByteArrayInputStream(bytes.toByteArray()))); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read levels in page for col %s.", descriptor), e); + } + } + + /** Utility interface to abstract over different way to read ints with different encodings. */ + interface IntIterator { + int nextInt(); + } + + /** Reading int from {@link ValuesReader}. */ + protected static final class ValuesReaderIntIterator implements IntIterator { + ValuesReader delegate; + + public ValuesReaderIntIterator(ValuesReader delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + return delegate.readInteger(); + } + } + + /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */ + protected static final class RLEIntIterator implements IntIterator { + RunLengthBitPackingHybridDecoder delegate; + + public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + try { + return delegate.readInt(); + } catch (IOException e) { + throw new ParquetDecodingException(e); + } + } + } + + /** Reading zero always. */ + protected static final class NullIntIterator implements IntIterator { + @Override + public int nextInt() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index fdfe5d6fa3a33..3748e53fc2843 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -26,12 +26,16 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY; @@ -252,21 +256,115 @@ public TimestampData readTimestamp() { } } + /** + * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard + * timestamp encoding defined by Parquet's + * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy + * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations. + * (The older INT96 encoding is marked deprecated by the Parquet format spec — see + * + * LogicalTypes.md — but is still supported here via {@link TypesFromInt96PageReader} for + * backwards compatibility with files written by older Hive / Spark / Impala versions.) + * + *

Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a + * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use + * {@link Int64TimestampColumnReader} for batched-vector efficiency. + */ + public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader { + private final boolean isUtcTimestamp; + private final ChronoUnit chronoUnit; + + public TypesFromInt64PageReader( + ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(realReader); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + public TypesFromInt64PageReader( + Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(dict); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + @Override + public TimestampData readTimestamp() { + return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit); + } + + @Override + public TimestampData readTimestamp(int id) { + return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper( boolean isDictionary, PrimitiveType parquetType, Dictionary dictionary, ValuesReader valuesReader, boolean isUtcTimestamp) { - if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { + PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName(); + if (typeName == PrimitiveType.PrimitiveTypeName.INT96) { return isDictionary ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp) : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp); - } else { - return isDictionary - ? new DefaultParquetDataColumnReader(dictionary) - : new DefaultParquetDataColumnReader(valuesReader); } + if (typeName == PrimitiveType.PrimitiveTypeName.INT64) { + ChronoUnit unit = resolveInt64TimestampUnit(parquetType); + if (unit != null) { + return isDictionary + ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit) + : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit); + } + } + return isDictionary + ? new DefaultParquetDataColumnReader(dictionary) + : new DefaultParquetDataColumnReader(valuesReader); + } + + /** + * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the + * column is a plain INT64 (not a timestamp). + * + *

Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and + * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} + * encodings. + */ + private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { + LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); + if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + LogicalTypeAnnotation.TimeUnit unit = + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit(); + switch (unit) { + case MILLIS: + return ChronoUnit.MILLIS; + case MICROS: + return ChronoUnit.MICROS; + case NANOS: + return ChronoUnit.NANOS; + default: + return null; + } + } + OriginalType originalType = parquetType.getOriginalType(); + if (originalType == OriginalType.TIMESTAMP_MILLIS) { + return ChronoUnit.MILLIS; + } + if (originalType == OriginalType.TIMESTAMP_MICROS) { + return ChronoUnit.MICROS; + } + return null; + } + + private static TimestampData int64ToTimestamp( + boolean utcTimestamp, long value, ChronoUnit unit) { + Instant instant = Instant.EPOCH.plus(value, unit); + if (utcTimestamp) { + return TimestampData.fromInstant(instant); + } + return TimestampData.fromTimestamp(Timestamp.from(instant)); } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java new file mode 100644 index 0000000000000..4b48fdd37460f --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector}, + * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's + * nested-Parquet reader (FLINK-35702). + * + *

The accessors are wrappers over the existing public fields so legacy callers continue to + * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the + * Dremel-style read path is exercised end-to-end by integration tests once + * {@code ParquetSplitReaderUtil} is wired up to {@code NestedColumnReader} in a follow-up PR. + */ +class TestHeapColumnVectorAccessors { + + // ----------------------------------------------------------------------------------------------- + // HeapArrayVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapArrayVectorAccessorsReflectPublicFields() { + HeapIntVector child = new HeapIntVector(4); + HeapArrayVector vector = new HeapArrayVector(2, child); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector replacementChild = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setChild(replacementChild); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(replacementChild, vector.getChild()); + assertEquals(2, vector.getSize()); + + // Backing public fields are kept in sync — preserves backward compatibility. + assertSame(offsets, vector.offsets); + assertSame(lengths, vector.lengths); + assertSame(replacementChild, vector.child); + } + + // ----------------------------------------------------------------------------------------------- + // HeapMapColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + + HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values); + + assertEquals(3, vector.getOffsets().length); + assertEquals(3, vector.getLengths().length); + } + + @Test + void heapMapColumnVectorAccessorsReflectInternalState() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector newKeys = new HeapLongVector(4); + HeapLongVector newValues = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setKeys(newKeys); + vector.setValues(newValues); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(newKeys, vector.getKeys()); + assertSame(newValues, vector.getValues()); + // The Flink-2.1-style ColumnVector accessors return the same underlying child. + assertSame(newKeys, vector.getKeyColumnVector()); + assertSame(newValues, vector.getValueColumnVector()); + assertEquals(2, vector.getSize()); + } + + // ----------------------------------------------------------------------------------------------- + // HeapRowColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() { + HeapIntVector intField = new HeapIntVector(2); + HeapLongVector longField = new HeapLongVector(2); + HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField); + + WritableColumnVector[] originalFields = vector.getFields(); + assertEquals(2, originalFields.length); + assertSame(intField, originalFields[0]); + assertSame(longField, originalFields[1]); + // Backing public field is kept in sync — preserves backward compatibility. + assertSame(originalFields, vector.vectors); + + HeapIntVector replacement = new HeapIntVector(2); + WritableColumnVector[] replacementFields = {replacement, longField}; + vector.setFields(replacementFields); + + assertSame(replacementFields, vector.getFields()); + assertSame(replacementFields, vector.vectors); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java new file mode 100644 index 0000000000000..9d6607d03febb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.flink.table.data.TimestampData; + +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when + * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702). + * + *

The factory is exercised end-to-end by integration tests through + * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece + * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each + * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} + * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values + * using both the values-reader and dictionary code paths. + */ +class TestParquetDataColumnReaderFactory { + + // ----------------------------------------------------------------------------------------------- + // Type dispatch + // ----------------------------------------------------------------------------------------------- + + @Test + void valuesReaderDispatchInt96TimestampUsesInt96Reader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MILLIS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MICROS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt32DoesNotUseTimestampReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + // ----------------------------------------------------------------------------------------------- + // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches) + // ----------------------------------------------------------------------------------------------- + + @Test + void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_123L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMillis), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + assertEquals(0, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + long epochMicros = 1_700_000_000_123_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMicros), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMicros / 1_000L, ts.getMillisecond()); + // 456 microseconds remain → 456_000 nanoseconds within the millisecond + assertEquals(456_000, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + long epochNanos = 1_700_000_000_123_456_789L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochNanos), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochNanos / 1_000_000L, ts.getMillisecond()); + assertEquals(456_789, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMillisFromDictionaryInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(epochMillis), true); + + TimestampData ts = reader.readTimestamp(0); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + } + + // ----------------------------------------------------------------------------------------------- + // Stubs (only the methods exercised by the dispatch + decoding tests above) + // ----------------------------------------------------------------------------------------------- + + /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */ + private static final class StubValuesReader extends ValuesReader { + private final long fixedLong; + + StubValuesReader() { + this(0L); + } + + StubValuesReader(long fixedLong) { + this.fixedLong = fixedLong; + } + + @Override + public long readLong() { + return fixedLong; + } + + @Override + public void skip() { + // unused + } + } + + /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */ + private static final class StubDictionary extends Dictionary { + private final long fixedLong; + + StubDictionary() { + this(0L); + } + + StubDictionary(long fixedLong) { + super(null); + this.fixedLong = fixedLong; + } + + @Override + public Binary decodeToBinary(int id) { + throw new UnsupportedOperationException(); + } + + @Override + public long decodeToLong(int id) { + return fixedLong; + } + + @Override + public int getMaxId() { + return 0; + } + } +} From 08584dabc8c4c4036b9f6d7eaf2f81a43578cfe9 Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Thu, 7 May 2026 20:07:05 -0700 Subject: [PATCH 006/255] feat(flink): Wire Flink 2.1 nested Parquet readers into the Hudi read path (FLINK-35702) (#18700) (cherry picked from commit 47bf4e41342e4f1dab26a7fb1489f278fbd1226c) --- .../format/cow/ParquetSplitReaderUtil.java | 686 +++++++++++------- .../cow/vector/HeapMapColumnVector.java | 19 +- .../cow/vector/reader/NestedColumnReader.java | 22 + .../reader/ParquetColumnarRowSplitReader.java | 34 +- 4 files changed, 472 insertions(+), 289 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index 2bb5be1d9614e..5468dc86a25a6 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,19 +19,18 @@ package org.apache.hudi.table.format.cow; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapDecimalVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; -import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader; import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader; import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader; import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader; +import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader; import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader; -import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader; @@ -64,12 +63,14 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; @@ -77,12 +78,18 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.io.ColumnIO; +import org.apache.parquet.io.GroupColumnIO; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.sql.Date; @@ -90,25 +97,38 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.flink.table.utils.DateTimeUtils.toInternal; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** * Util for generating {@link ParquetColumnarRowSplitReader}. * - *

NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64 - * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that. + *

Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive + * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader}, + * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For + * nested types (ARRAY / MAP / MULTISET / ROW) we build a {@link ParquetField} tree once per + * split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to + * {@link NestedColumnReader}. + * + *

Schema evolution: missing top-level fields are still handled by the caller + * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields + * inside a Row are handled here — {@link #constructField} returns {@code null} for a + * child that isn't physically present, and the corresponding child in the pre-allocated vector + * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can + * passthrough the slot (see {@link NestedColumnReader#readToVector}). */ public class ParquetSplitReaderUtil { - /** - * Util for generating partitioned {@link ParquetColumnarRowSplitReader}. - */ + /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( boolean utcTimestamp, boolean caseSensitive, @@ -182,10 +202,13 @@ private static ColumnVector createVector( return readVector; } - private static ColumnVector createVectorFromConstant( - LogicalType type, - Object value, - int batchSize) { + /** + * Builds a constant-filled column vector for either a partition column (non-null value) or a + * missing-column slot (null value). Used both at the batch-generator level for partition + * injection and at the row-reader level for fields absent from the Parquet file. + */ + public static ColumnVector createVectorFromConstant( + LogicalType type, Object value, int batchSize) { switch (type.getTypeRoot()) { case CHAR: case VARCHAR: @@ -278,6 +301,7 @@ private static ColumnVector createVectorFromConstant( value == null ? null : toInternal((Date) value), batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: HeapTimestampVector tv = new HeapTimestampVector(batchSize); if (value == null) { tv.fillWithNulls(); @@ -286,46 +310,41 @@ private static ColumnVector createVectorFromConstant( } return tv; case ARRAY: - ArrayType arrayType = (ArrayType) type; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize); - if (value == null) { - arrayGroup.fillWithNulls(); - return arrayGroup; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } - } else { - HeapArrayVector arrayVector = new HeapArrayVector(batchSize); - if (value == null) { - arrayVector.fillWithNulls(); - return arrayVector; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } + if (value != null) { + throw new UnsupportedOperationException("Unsupported create array with default value."); } + HeapArrayVector arrayVector = new HeapArrayVector(batchSize); + arrayVector.fillWithNulls(); + return arrayVector; case MAP: - HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); - if (value == null) { - mapVector.fillWithNulls(); - return mapVector; - } else { - throw new UnsupportedOperationException("Unsupported create map with default value."); + case MULTISET: + if (value != null) { + throw new UnsupportedOperationException( + "Unsupported create " + type.getTypeRoot() + " with default value."); } + HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); + mapVector.fillWithNulls(); + return mapVector; case ROW: - HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize); - if (value == null) { - rowVector.fillWithNulls(); - return rowVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create row with default value."); } + RowType rowType = (RowType) type; + WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()]; + for (int i = 0; i < childVectors.length; i++) { + childVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); + } + HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors); + rowVector.fillWithNulls(); + return rowVector; default: throw new UnsupportedOperationException("Unsupported type: " + type); } } - private static List filterDescriptors(int depth, Type type, List columns) throws ParquetRuntimeException { + private static List filterDescriptors( + int depth, Type type, List columns) throws ParquetRuntimeException { List filtered = new ArrayList<>(); for (ColumnDescriptor descriptor : columns) { if (depth >= descriptor.getPath().length) { @@ -339,24 +358,61 @@ private static List filterDescriptors(int depth, Type type, Li return filtered; } + /** + * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the + * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP}, + * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by + * the supplied pre-built {@link ParquetField} tree. + * + * @param field the {@link ParquetField} tree for this column, built by + * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested + * types; ignored for primitives. + */ + public static ColumnReader createColumnReader( + boolean utcTimestamp, + LogicalType fieldType, + Type physicalType, + List descriptors, + PageReadStore pages, + @Nullable ParquetField field) throws IOException { + switch (fieldType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + Preconditions.checkNotNull( + field, "ParquetField must be provided for nested type: %s", fieldType); + return new NestedColumnReader(utcTimestamp, pages, field); + default: + return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages); + } + } + + /** + * Backward-compat entry point kept for callers that don't project nested types and therefore + * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload + * with a null field; nested types now go through that overload directly. + * + * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore, + * ParquetField)} so nested types take the Dremel path. + */ + @Deprecated public static ColumnReader createColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List descriptors, PageReadStore pages) throws IOException { - return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, - pages, 0); + return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null); } - private static ColumnReader createColumnReader( + private static ColumnReader createPrimitiveColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List columns, - PageReadStore pages, - int depth) throws IOException { - List descriptors = filterDescriptors(depth, physicalType, columns); + PageReadStore pages) throws IOException { + List descriptors = filterDescriptors(0, physicalType, columns); ColumnDescriptor descriptor = descriptors.get(0); PageReader pageReader = pages.getPageReader(descriptor); switch (fieldType.getTypeRoot()) { @@ -392,7 +448,9 @@ private static ColumnReader createColumnReader( case INT96: return new TimestampColumnReader(utcTimestamp, descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for TIMESTAMP: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } case DECIMAL: switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { @@ -403,106 +461,23 @@ private static ColumnReader createColumnReader( case BINARY: return new BytesColumnReader(descriptor, pageReader); case FIXED_LEN_BYTE_ARRAY: - return new FixedLenBytesColumnReader( - descriptor, pageReader); + return new FixedLenBytesColumnReader(descriptor, pageReader); default: - throw new AssertionError(); - } - case ARRAY: - ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new ArrayGroupReader(createColumnReader( - utcTimestamp, - arrayType.getElementType(), - elementType, - descriptors, - pages, - elementDepth)); - } else { - return new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - fieldType); - } - case MAP: - MapType mapType = (MapType) fieldType; - ArrayColumnReader keyReader = - new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - new ArrayType(mapType.getKeyType())); - ColumnReader valueReader; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueReader = new ArrayGroupReader(createColumnReader( - utcTimestamp, - mapType.getValueType(), - physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type - descriptors.subList(1, descriptors.size()), // remove the key descriptor - pages, - depth + 2)); // increase the depth by 2, because there's a key_value entry in the path - } else { - valueReader = new ArrayColumnReader( - descriptors.get(1), - pages.getPageReader(descriptors.get(1)), - utcTimestamp, - descriptors.get(1).getPrimitiveType(), - new ArrayType(mapType.getValueType())); + throw new AssertionError( + "Unexpected physical type for DECIMAL: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } - return new MapColumnReader(keyReader, valueReader); - case ROW: - RowType rowType = (RowType) fieldType; - GroupType groupType = physicalType.asGroupType(); - List fieldReaders = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - // schema evolution: read the parquet file with a new extended field name. - int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); - if (fieldIndex < 0) { - fieldReaders.add(new EmptyColumnReader()); - } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - fieldReaders.add( - createColumnReader( - utcTimestamp, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } else { - fieldReaders.add( - createColumnReader( - utcTimestamp, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } - } - } - return new RowColumnReader(fieldReaders); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + /** + * Creates the writable column vector that the reader will write into. The returned vector shape + * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled + * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in + * {@link NestedColumnReader} can pass them through unchanged. + */ public static WritableColumnVector createWritableColumnVector( int batchSize, LogicalType fieldType, @@ -523,40 +498,48 @@ private static WritableColumnVector createWritableColumnVector( switch (fieldType.getTypeRoot()) { case BOOLEAN: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBooleanVector(batchSize); case TINYINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapByteVector(batchSize); case DOUBLE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDoubleVector(batchSize); case FLOAT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.FLOAT, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapFloatVector(batchSize); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapIntVector(batchSize); case BIGINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT64, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapLongVector(batchSize); case SMALLINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapShortVector(batchSize); case CHAR: case VARCHAR: case BINARY: case VARBINARY: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BINARY, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BINARY, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBytesVector(batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: @@ -566,112 +549,64 @@ private static WritableColumnVector createWritableColumnVector( case DECIMAL: checkArgument( (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY - || typeName == PrimitiveType.PrimitiveTypeName.BINARY) + || typeName == PrimitiveType.PrimitiveTypeName.BINARY) && primitiveType.getOriginalType() == OriginalType.DECIMAL, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDecimalVector(batchSize); case ARRAY: ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - elementType, - descriptors, - elementDepth)); - } else { - return new HeapArrayVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - physicalType, - descriptors, - depth)); - } - case MAP: + return new HeapArrayVector( + batchSize, + createWritableColumnVector( + batchSize, arrayType.getElementType(), physicalType, descriptors, depth)); + case MAP: { MapType mapType = (MapType) fieldType; - GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType(); - // the map column has three level paths. - WritableColumnVector keyColumnVector = createWritableColumnVector( + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( batchSize, - new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()), - repeatedType.getType(0), - descriptors, - depth + 2); - WritableColumnVector valueColumnVector; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueColumnVector = new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - mapType.getValueType(), - repeatedType.getType(1).asGroupType(), - descriptors, - depth + 2)); - } else { - valueColumnVector = createWritableColumnVector( - batchSize, - new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()), - repeatedType.getType(1), - descriptors, - depth + 2); - } - return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector); + createWritableColumnVector( + batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2), + createWritableColumnVector( + batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2)); + } + case MULTISET: { + MultisetType multisetType = (MultisetType) fieldType; + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( + batchSize, + createWritableColumnVector( + batchSize, + multisetType.getElementType(), + repeatedType.getType(0), + descriptors, + depth + 2), + createWritableColumnVector( + batchSize, + new IntType(false), + repeatedType.getType(1), + descriptors, + depth + 2)); + } case ROW: RowType rowType = (RowType) fieldType; GroupType groupType = physicalType.asGroupType(); WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()]; for (int i = 0; i < columnVectors.length; i++) { - // schema evolution: read the file with a new extended field name. int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); if (fieldIndex < 0) { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant( - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize); - } else { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); - } + // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled + // vector of the correct shape; NestedColumnReader.readRow will pass it through when the + // matching ParquetField child is null. + columnVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = - createWritableColumnVector( - batchSize, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } else { - columnVectors[i] = - createWritableColumnVector( - batchSize, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } + columnVectors[i] = + createWritableColumnVector( + batchSize, + rowType.getTypeAt(i), + groupType.getType(fieldIndex), + descriptors, + depth + 1); } } return new HeapRowColumnVector(batchSize, columnVectors); @@ -681,56 +616,245 @@ private static WritableColumnVector createWritableColumnVector( } /** - * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. - * - * @return The physical field index or -1 if the field does not exist + * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching + * Parquet's canonical 3-level map encoding. */ - private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { - // get index from fileSchema type, else, return -1 - return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + private static GroupType unwrapMapRepeatedType(Type physicalType) { + return physicalType.asGroupType().getType(0).asGroupType(); } + // ------------------------------------------------------------------------------------------ + // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil) + // + // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children + // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This + // matches the Hudi schema-evolution contract and is the companion to the null-child branch in + // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`. + // ------------------------------------------------------------------------------------------ + /** - * Check whether the given list type is a three-level list type. - *

- * group (LIST) { - * repeated group list { - * element; - * } - * } - * - * @param type list type - * @return true if the list type is a three-level list type + * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed + * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally; + * primitive top-level fields produce {@code null} entries (callers don't need a tree for those). + */ + public static List buildFieldsList( + List children, List fieldNames, MessageColumnIO columnIO) { + List list = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + RowType.RowField child = children.get(i); + if (isNestedType(child.getType())) { + list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i)))); + } else { + list.add(null); + } + } + return list; + } + + private static boolean isNestedType(LogicalType type) { + return type instanceof RowType + || type instanceof ArrayType + || type instanceof MapType + || type instanceof MultisetType; + } + + @Nullable + private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) { + boolean required = columnIO.getType().getRepetition() == REQUIRED; + int repetitionLevel = columnIO.getRepetitionLevel(); + int definitionLevel = columnIO.getDefinitionLevel(); + LogicalType type = rowField.getType(); + String fieldName = rowField.getName(); + if (type instanceof RowType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + RowType rowType = (RowType) type; + List childFields = rowType.getFields(); + List fieldsList = new ArrayList<>(childFields.size()); + for (RowType.RowField childField : childFields) { + // Hudi schema evolution: a logical child may be absent from the Parquet file. In that + // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the + // pre-filled null vector instead of recursing. + ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName()); + if (childIo == null) { + fieldsList.add(null); + } else { + fieldsList.add(constructField(childField, childIo)); + } + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(fieldsList)); + } + + if (type instanceof MapType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MapType mapType = (MapType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof MultisetType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MultisetType multisetType = (MultisetType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", multisetType.getElementType()), + keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof ArrayType) { + ArrayType arrayType = (ArrayType) type; + ColumnIO elementTypeColumnIO; + if (columnIO instanceof GroupColumnIO) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { + while (!Objects.equals(groupColumnIO.getName(), fieldName)) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + elementTypeColumnIO = groupColumnIO; + } else { + if (arrayType.getElementType() instanceof RowType) { + elementTypeColumnIO = groupColumnIO; + } else { + elementTypeColumnIO = groupColumnIO.getChild(0); + } + } + } else if (columnIO instanceof PrimitiveColumnIO) { + elementTypeColumnIO = columnIO; + } else { + throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); + } + + ParquetField elementField = + constructField( + new RowType.RowField("", arrayType.getElementType()), + getArrayElementColumn(elementTypeColumnIO)); + if (repetitionLevel == elementField.getRepetitionLevel()) { + repetitionLevel = columnIO.getParent().getRepetitionLevel(); + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.singletonList(elementField)); + } + + PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO; + return new ParquetPrimitiveField( + type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId()); + } + + /** + * Parquet column names are case-insensitive in Flink's lookup. Matches upstream + * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent. */ - private static boolean isThreeLevelList(Type type) { - if (type.isPrimitive()) { - return false; + public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName); + if (columnIO != null) { + return columnIO; } - GroupType groupType = type.asGroupType(); - OriginalType originalType = groupType.getOriginalType(); - return originalType == OriginalType.LIST - && groupType.getType(0).getName().equals("list"); + throw new FlinkRuntimeException( + "Can not find column io for parquet reader. Column name: " + columnName); } /** - * Construct the error message when primitive type mismatches. - * - * @param primitiveType Primitive type - * @param fieldType Logical field type - * @return The error message + * Case-insensitive column lookup that returns {@code null} when no match is found — the + * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to + * emit null {@link ParquetField} children for fields absent from the Parquet file. */ - private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { - return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name()); + @Nullable + private static ColumnIO lookupColumnByNameOrNull( + GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = groupColumnIO.getChild(columnName); + if (columnIO != null) { + return columnIO; + } + for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) { + if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) { + return groupColumnIO.getChild(i); + } + } + return null; + } + + public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { + while (groupColumnIO.getChildrenCount() == 1) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + return groupColumnIO; + } + + public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { + while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { + columnIO = ((GroupColumnIO) columnIO).getChild(0); + } + + // Three-level list: skip the synthetic `element` / `list` wrapper when present. + if (columnIO instanceof GroupColumnIO + && columnIO.getType().getLogicalTypeAnnotation() == null + && ((GroupColumnIO) columnIO).getChildrenCount() == 1 + && !columnIO.getName().equals("array") + && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { + return ((GroupColumnIO) columnIO).getChild(0); + } + return columnIO; } /** - * Construct the error message when original type mismatches. + * Returns the field index with given physical row type {@code groupType} and field name + * {@code fieldName}. * - * @param originalType Original type - * @param fieldType Logical field type - * @return The error message + * @return the physical field index or -1 if the field does not exist + */ + private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { + return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + } + + private static String getPrimitiveTypeCheckFailureMessage( + PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Primitive type: %s. Field type: %s.", + primitiveType, fieldType.getTypeRoot().name()); + } + + private static String getOriginalTypeCheckFailureMessage( + OriginalType originalType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Original type: %s. Field type: %s.", + originalType, fieldType.getTypeRoot().name()); + } + + /** + * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience + * for callers that need to mirror Hudi's original behaviour where a missing column produces an + * explicit null-valued reader rather than being omitted from the batch. */ - private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) { - return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name()); + public static ColumnReader emptyColumnReader() { + return new EmptyColumnReader(); } } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index a98bdebd707a7..14aad22039e0a 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -20,6 +20,7 @@ import lombok.Getter; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.ColumnarMapData; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; @@ -27,6 +28,14 @@ /** * This class represents a nullable heap map column vector. + * + *

Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from + * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward + * compatibility: the {@code keys} / {@code values} fields are typed + * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing + * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original + * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use + * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}. */ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { @@ -38,10 +47,8 @@ public class HeapMapColumnVector extends AbstractHeapVector // --------------------------------------------------------------------------------------------- // Flink 2.1 Dremel-style state. Populated by {@link - // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port). The - // legacy {@link #getMap(int)} implementation below continues to use {@code ColumnarGroupMapData} - // — wiring it through these offsets/lengths happens in a follow-up PR that switches the read - // path. Left here so the new readers can compile against the additive surface. + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and + // consumed by {@link #getMap(int)}. // --------------------------------------------------------------------------------------------- private long[] offsets; private long[] lengths; @@ -102,6 +109,8 @@ public ColumnVector getValueColumnVector() { @Override public MapData getMap(int rowId) { - return new ColumnarGroupMapData(keys, values, rowId); + long offset = offsets[rowId]; + long length = lengths[rowId]; + return new ColumnarMapData(keys, values, (int) offset, (int) length); } } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 27eab298b3207..6ae1cc9492ecd 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -147,6 +147,28 @@ private Tuple2 readRow( if (rowPosition.getIsNull() != null) { setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); } + + // Hudi-specific: collapse a present row whose every child is null into a null row. The + // legacy RowColumnReader did this so that a SQL value like `row(null, null)` round-trips + // to NULL on read; preserve it here for backward compatibility. Diverges from Flink 2.1, + // which would surface it as Row(null, null). Mirrored by the integration test + // ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + int rowCount = rowPosition.getPositionsCount(); + for (int j = 0; j < rowCount; j++) { + if (heapRowVector.isNullAt(j)) { + continue; + } + boolean allChildrenNull = true; + for (WritableColumnVector child : finalChildrenVectors) { + if (!child.isNullAt(j)) { + allChildrenNull = false; + break; + } + } + if (allChildrenNull) { + heapRowVector.setNullAt(j); + } + } return Tuple2.of(levelDelegation, heapRowVector); } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index 3572b117a6313..1826419db5d44 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -18,7 +18,9 @@ package org.apache.hudi.table.format.cow.vector.reader; +import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.table.data.RowData; @@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,6 +42,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -46,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -53,7 +59,6 @@ import java.util.Map; import java.util.stream.IntStream; -import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; @@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable { private final MessageType requestedSchema; + /** + * {@link ParquetField} tree per top-level requested column, used by + * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List, + * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for + * nested types. Entries are {@code null} for primitive top-level fields. Built once per split. + */ + private final List requestedFields; + /** * The total number of rows this RecordReader will eventually read. The sum of the rows of all * the row groups. @@ -158,6 +171,20 @@ public ParquetColumnarRowSplitReader( checkSchema(); + // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across + // row groups). Only columns with nested logical type get a non-null entry — primitive columns + // still use Hudi's specialized ColumnReaders. + MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema); + List requestedRowFields = new ArrayList<>(requestedTypes.length); + List requestedFieldNames = new ArrayList<>(requestedTypes.length); + for (int i = 0; i < requestedTypes.length; i++) { + String name = requestedSchema.getFieldName(i); + requestedRowFields.add(new RowType.RowField(name, requestedTypes[i])); + requestedFieldNames.add(name); + } + this.requestedFields = ParquetSplitReaderUtil.buildFieldsList( + requestedRowFields, requestedFieldNames, messageColumnIO); + this.writableVectors = createWritableVectors(); ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices); this.columnarBatch = generator.generate(columnVectors); @@ -340,12 +367,13 @@ private void readNextRowGroup() throws IOException { List columns = requestedSchema.getColumns(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = createColumnReader( + columnReaders[i] = ParquetSplitReaderUtil.createColumnReader( utcTimestamp, requestedTypes[i], types.get(i), columns, - pages); + pages, + requestedFields.get(i)); } totalCountLoadedSoFar += pages.getRowCount(); } From f0bf5eb7647dd88ee6a28f201f58a67651e88c88 Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Sat, 9 May 2026 05:16:58 -0700 Subject: [PATCH 007/255] refactor(flink): Remove legacy Parquet nested readers superseded by Flink 2.1 Dremel path (FLINK-35702) (#18701) * refactor(flink): Remove legacy Parquet nested readers superseded by Flink 2.1 Dremel path (FLINK-35702) * Fix flaky IT test (cherry picked from commit 004d159968964d17875adb0ba97a51a206dfe1ff) --- .../hudi/table/ITTestHoodieDataSource.java | 42 +- .../cow/vector/ColumnarGroupArrayData.java | 179 ------- .../cow/vector/ColumnarGroupMapData.java | 63 --- .../cow/vector/ColumnarGroupRowData.java | 138 ----- .../vector/HeapArrayGroupColumnVector.java | 53 -- .../cow/vector/reader/ArrayColumnReader.java | 473 ------------------ .../cow/vector/reader/ArrayGroupReader.java | 44 -- .../cow/vector/reader/MapColumnReader.java | 56 --- .../cow/vector/reader/NestedColumnReader.java | 15 +- .../reader/NestedPrimitiveColumnReader.java | 4 +- .../cow/vector/reader/RowColumnReader.java | 63 --- .../vector/TestHeapColumnVectorAccessors.java | 5 +- 12 files changed, 53 insertions(+), 1082 deletions(-) delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 46b9353e0ec24..5d6a5daa061e5 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -3555,11 +3555,51 @@ private List fetchResultWithExpectedNum(TableEnvironment tEnv, TableResult // and max waiting timeout is 30s tableResult.await(30, TimeUnit.SECONDS); } catch (Throwable e) { - ExceptionUtils.assertThrowable(e, CollectSinkTableFactory.SuccessException.class); + // Acceptable terminal causes: + // 1. SuccessException: the sink reached its expected row count and intentionally + // threw to terminate the streaming job. This is the happy path. + // 2. IOException("Stream is closed!") wrapped as HoodieIOException: a benign + // error-attribution race between the source-side cascading-shutdown path and + // the sink-side SuccessException terminator. When the sink throws + // SuccessException to end the job, the chained source's SplitFetcher can close + // the underlying Hadoop FSDataInputStream while the mailbox is still draining + // a BatchRecords queued earlier; the next row-group read on the now-closed + // stream surfaces an IOException("Stream is closed!"). With + // restart-strategy.fixed-delay.attempts=0 (set in beforeEach to keep tests + // deterministic) that IOException becomes the job's reported failure cause + // instead of the sink's SuccessException, even though the sink has already + // collected the expected rows by then - i.e. the functional outcome is + // unchanged, only the error-attribution differs. Production paths correctly + // fail the job on stream-closed-mid-read (the right behavior for real I/O + // failures), so this tolerance is scoped to the SuccessException-based test + // pattern below and is NOT mirrored in production code. + if (!isAcceptableTerminalFailure(e)) { + throw new AssertionError("Unexpected job failure", e); + } } tEnv.executeSql("DROP TABLE IF EXISTS sink"); return CollectSinkTableFactory.RESULT.values().stream() .flatMap(Collection::stream) .collect(Collectors.toList()); } + + /** + * Whether {@code e} (or any of its causes) is one of the terminal failures that + * {@link #fetchResultWithExpectedNum} is allowed to swallow. See the comment at the call + * site for the rationale. + */ + private static boolean isAcceptableTerminalFailure(Throwable e) { + Throwable cur = e; + while (cur != null) { + if (cur instanceof CollectSinkTableFactory.SuccessException) { + return true; + } + String msg = cur.getMessage(); + if (msg != null && msg.contains("Stream is closed")) { + return true; + } + cur = cur.getCause(); + } + return false; + } } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java deleted file mode 100644 index 4c9275f3b0932..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupArrayData implements ArrayData { - - WritableColumnVector vector; - int rowId; - - public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) { - this.vector = vector; - this.rowId = rowId; - } - - @Override - public int size() { - if (vector == null) { - return 0; - } - - if (vector instanceof HeapRowColumnVector) { - // assume all fields have the same size - if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) { - return 0; - } - return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size(); - } - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean isNullAt(int index) { - if (vector == null) { - return true; - } - - if (vector instanceof HeapRowColumnVector) { - return ((HeapRowColumnVector) vector).vectors == null; - } - - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean getBoolean(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float getFloat(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public StringData getString(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public DecimalData getDecimal(int index, int precision, int scale) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public TimestampData getTimestamp(int index, int precision) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RawValueData getRawValue(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] getBinary(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public ArrayData getArray(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public MapData getMap(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RowData getRow(int index, int numFields) { - return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java deleted file mode 100644 index 69cb6feca13e4..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupMapData implements MapData { - - WritableColumnVector keyVector; - WritableColumnVector valueVector; - int rowId; - - public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) { - this.keyVector = keyVector; - this.valueVector = valueVector; - this.rowId = rowId; - } - - @Override - public int size() { - if (keyVector == null) { - return 0; - } - - if (keyVector instanceof HeapArrayVector) { - return ((HeapArrayVector) keyVector).getArray(rowId).size(); - } - throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector"); - } - - @Override - public ArrayData keyArray() { - return ((HeapArrayVector) keyVector).getArray(rowId); - } - - @Override - public ArrayData valueArray() { - if (valueVector instanceof HeapArrayVector) { - return ((HeapArrayVector) valueVector).getArray(rowId); - } else if (valueVector instanceof HeapArrayGroupColumnVector) { - return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId); - } - throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector"); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java deleted file mode 100644 index 439c1880823f1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.types.RowKind; - -public class ColumnarGroupRowData implements RowData { - - HeapRowColumnVector vector; - int rowId; - int index; - - public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) { - this.vector = vector; - this.rowId = rowId; - this.index = index; - } - - @Override - public int getArity() { - return vector.vectors.length; - } - - @Override - public RowKind getRowKind() { - return RowKind.INSERT; - } - - @Override - public void setRowKind(RowKind rowKind) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public boolean isNullAt(int pos) { - return - vector.vectors[pos].isNullAt(rowId) - || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index); - } - - @Override - public boolean getBoolean(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index); - } - - @Override - public byte getByte(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index); - } - - @Override - public short getShort(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index); - } - - @Override - public int getInt(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index); - } - - @Override - public long getLong(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index); - } - - @Override - public float getFloat(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index); - } - - @Override - public double getDouble(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index); - } - - @Override - public StringData getString(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index); - } - - @Override - public DecimalData getDecimal(int pos, int i1, int i2) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2); - } - - @Override - public TimestampData getTimestamp(int pos, int i1) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1); - } - - @Override - public RawValueData getRawValue(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index); - } - - @Override - public byte[] getBinary(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index); - } - - @Override - public ArrayData getArray(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index); - } - - @Override - public MapData getMap(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index); - } - - @Override - public RowData getRow(int pos, int numFields) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java deleted file mode 100644 index 3d7d8b1f0de0f..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -/** - * This class represents a nullable heap row column vector. - */ -public class HeapArrayGroupColumnVector extends AbstractHeapVector - implements WritableColumnVector, ArrayColumnVector { - - public WritableColumnVector vector; - - public HeapArrayGroupColumnVector(int len) { - super(len); - } - - public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) { - super(len); - this.vector = vector; - } - - @Override - public ArrayData getArray(int rowId) { - return new ColumnarGroupArrayData(vector, rowId); - } - - @Override - public void reset() { - super.reset(); - vector.reset(); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java deleted file mode 100644 index d758f35078d8f..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayVector; -import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; -import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.table.types.logical.ArrayType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Array {@link ColumnReader}. - */ -public class ArrayColumnReader extends BaseVectorizedColumnReader { - - // The value read in last time - private Object lastValue; - - // flag to indicate if there is no data in parquet data page - private boolean eof = false; - - // flag to indicate if it's the first time to read parquet data page with this instance - boolean isFirstRow = true; - - public ArrayColumnReader( - ColumnDescriptor descriptor, - PageReader pageReader, - boolean isUtcTimestamp, - Type type, - LogicalType logicalType) - throws IOException { - super(descriptor, pageReader, isUtcTimestamp, type, logicalType); - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayVector lcv = (HeapArrayVector) vector; - // before readBatch, initial the size of offsets & lengths as the default value, - // the actual size will be assigned in setChildrenInfo() after reading complete. - lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - // Because the length of ListColumnVector.child can't be known now, - // the valueList will save all data for ListColumnVector temporary. - List valueList = new ArrayList<>(); - - LogicalType category = ((ArrayType) logicalType).getElementType(); - - // read the first row in parquet data page, this will be only happened once for this - // instance - if (isFirstRow) { - if (!fetchNextValue(category)) { - return; - } - isFirstRow = false; - } - - int index = collectDataFromParquetPage(readNumber, lcv, valueList, category); - - // Convert valueList to array for the ListColumnVector.child - fillColumnVector(category, lcv, valueList, index); - } - - /** - * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating - * if there is more values to read (true). - * - * @param category - * @return boolean - * @throws IOException - */ - private boolean fetchNextValue(LogicalType category) throws IOException { - int left = readPageIfNeed(); - if (left > 0) { - // get the values of repetition and definitionLevel - readRepetitionAndDefinitionLevels(); - // read the data if it isn't null - if (definitionLevel == maxDefLevel) { - if (isCurrentPageDictionaryEncoded) { - lastValue = dataColumn.readValueDictionaryId(); - } else { - lastValue = readPrimitiveTypedRow(category); - } - } else { - lastValue = null; - } - return true; - } else { - eof = true; - return false; - } - } - - private int readPageIfNeed() throws IOException { - // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); - } - return leftInPage; - } - - // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper - // TODO Reduce the duplicated code - private Object readPrimitiveTypedRow(LogicalType category) { - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dataColumn.readString(); - case BOOLEAN: - return dataColumn.readBoolean(); - case TIME_WITHOUT_TIME_ZONE: - case DATE: - case INTEGER: - return dataColumn.readInteger(); - case TINYINT: - return dataColumn.readTinyInt(); - case SMALLINT: - return dataColumn.readSmallInt(); - case BIGINT: - return dataColumn.readLong(); - case FLOAT: - return dataColumn.readFloat(); - case DOUBLE: - return dataColumn.readDouble(); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dataColumn.readInteger(); - case INT64: - return dataColumn.readLong(); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return dataColumn.readString(); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dataColumn.readTimestamp(); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { - if (dictionaryValue == null) { - return null; - } - - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dictionary.readString(dictionaryValue); - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case INTEGER: - return dictionary.readInteger(dictionaryValue); - case BOOLEAN: - return dictionary.readBoolean(dictionaryValue) ? 1 : 0; - case DOUBLE: - return dictionary.readDouble(dictionaryValue); - case FLOAT: - return dictionary.readFloat(dictionaryValue); - case TINYINT: - return dictionary.readTinyInt(dictionaryValue); - case SMALLINT: - return dictionary.readSmallInt(dictionaryValue); - case BIGINT: - return dictionary.readLong(dictionaryValue); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dictionary.readInteger(dictionaryValue); - case INT64: - return dictionary.readLong(dictionaryValue); - case FIXED_LEN_BYTE_ARRAY: - case BINARY: - return dictionary.readString(dictionaryValue); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dictionary.readTimestamp(dictionaryValue); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - /** - * Collects data from a parquet page and returns the final row index where it stopped. The - * returned index can be equal to or less than total. - * - * @param total maximum number of rows to collect - * @param lcv column vector to do initial setup in data collection time - * @param valueList collection of values that will be fed into the vector later - * @param category - * @return int - * @throws IOException - */ - private int collectDataFromParquetPage( - int total, HeapArrayVector lcv, List valueList, LogicalType category) - throws IOException { - int index = 0; - /* - * Here is a nested loop for collecting all values from a parquet page. - * A column of array type can be considered as a list of lists, so the two loops are as below: - * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.: - * [0, 2, 3] <- index: 0 - * [NULL, 3, 4] <- index: 1 - * - * 2. The inner loop iterates on values within a row (sets all data from parquet data page - * for an element in ListColumnVector), so fetchNextValue returns values one-by-one: - * 0, 2, 3, NULL, 3, 4 - * - * As described below, the repetition level (repetitionLevel != 0) - * can be used to decide when we'll start to read values for the next list. - */ - while (!eof && index < total) { - // add element to ListColumnVector one by one - lcv.offsets[index] = valueList.size(); - /* - * Let's collect all values for a single list. - * Repetition level = 0 means that a new list started there in the parquet page, - * in that case, let's exit from the loop, and start to collect value for a new list. - */ - do { - /* - * Definition level = 0 when a NULL value was returned instead of a list - * (this is not the same as a NULL value in of a list). - */ - if (definitionLevel == 0) { - lcv.setNullAt(index); - } - valueList.add( - isCurrentPageDictionaryEncoded - ? dictionaryDecodeValue(category, (Integer) lastValue) - : lastValue); - } while (fetchNextValue(category) && (repetitionLevel != 0)); - - lcv.lengths[index] = valueList.size() - lcv.offsets[index]; - index++; - } - return index; - } - - /** - * The lengths & offsets will be initialized as default size (1024), it should be set to the - * actual size according to the element number. - */ - private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) { - lcv.setSize(itemNum); - long[] lcvLength = new long[elementNum]; - long[] lcvOffset = new long[elementNum]; - System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum); - System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum); - lcv.lengths = lcvLength; - lcv.offsets = lcvOffset; - } - - private void fillColumnVector( - LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) { - int total = valueList.size(); - setChildrenInfo(lcv, total, elementNum); - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - lcv.child = new HeapBytesVector(total); - ((HeapBytesVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (src == null) { - ((HeapBytesVector) lcv.child).setNullAt(i); - } else { - ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length); - } - } - break; - case BOOLEAN: - lcv.child = new HeapBooleanVector(total); - ((HeapBooleanVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapBooleanVector) lcv.child).setNullAt(i); - } else { - ((HeapBooleanVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TINYINT: - lcv.child = new HeapByteVector(total); - ((HeapByteVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapByteVector) lcv.child).setNullAt(i); - } else { - ((HeapByteVector) lcv.child).vector[i] = - (byte) ((List) valueList).get(i).intValue(); - } - } - break; - case SMALLINT: - lcv.child = new HeapShortVector(total); - ((HeapShortVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapShortVector) lcv.child).setNullAt(i); - } else { - ((HeapShortVector) lcv.child).vector[i] = - (short) ((List) valueList).get(i).intValue(); - } - } - break; - case INTEGER: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - lcv.child = new HeapIntVector(total); - ((HeapIntVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) lcv.child).setNullAt(i); - } else { - ((HeapIntVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case FLOAT: - lcv.child = new HeapFloatVector(total); - ((HeapFloatVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapFloatVector) lcv.child).setNullAt(i); - } else { - ((HeapFloatVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case BIGINT: - lcv.child = new HeapLongVector(total); - ((HeapLongVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) lcv.child).setNullAt(i); - } else { - ((HeapLongVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case DOUBLE: - lcv.child = new HeapDoubleVector(total); - ((HeapDoubleVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapDoubleVector) lcv.child).setNullAt(i); - } else { - ((HeapDoubleVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - lcv.child = new HeapTimestampVector(total); - ((HeapTimestampVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapTimestampVector) lcv.child).setNullAt(i); - } else { - ((HeapTimestampVector) lcv.child) - .setTimestamp(i, ((List) valueList).get(i)); - } - } - break; - case DECIMAL: - PrimitiveType.PrimitiveTypeName primitiveTypeName = - descriptor.getPrimitiveType().getPrimitiveTypeName(); - switch (primitiveTypeName) { - case INT32: - lcv.child = new ParquetDecimalVector(new HeapIntVector(total)); - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).getVector()).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).getVector()) - .setNullAt(i); - } else { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).getVector()) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - case INT64: - lcv.child = new ParquetDecimalVector(new HeapLongVector(total)); - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).getVector()).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).getVector()) - .setNullAt(i); - } else { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).getVector()) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - default: - lcv.child = new ParquetDecimalVector(new HeapBytesVector(total)); - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).getVector()).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (valueList.get(i) == null) { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).getVector()) - .setNullAt(i); - } else { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).getVector()) - .appendBytes(i, src, 0, src.length); - } - } - break; - } - break; - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java deleted file mode 100644 index 437c186a93661..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; - -import java.io.IOException; - -/** - * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}. - */ -public class ArrayGroupReader implements ColumnReader { - - private final ColumnReader fieldReader; - - public ArrayGroupReader(ColumnReader fieldReader) { - this.fieldReader = fieldReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector; - - fieldReader.readToVector(readNumber, rowColumnVector.vector); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java deleted file mode 100644 index ee65dd22c4369..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Map {@link ColumnReader}. - */ -public class MapColumnReader implements ColumnReader { - - private final ArrayColumnReader keyReader; - private final ColumnReader valueReader; - - public MapColumnReader( - ArrayColumnReader keyReader, ColumnReader valueReader) { - this.keyReader = keyReader; - this.valueReader = valueReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector; - AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys()); - keyReader.readToVector(readNumber, mapColumnVector.getKeys()); - valueReader.readToVector(readNumber, mapColumnVector.getValues()); - for (int i = 0; i < keyArrayColumnVector.getLen(); i++) { - if (keyArrayColumnVector.isNullAt(i)) { - mapColumnVector.setNullAt(i); - } - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 6ae1cc9492ecd..b4f04b20c4772 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -114,9 +114,8 @@ private Tuple2 readRow( ParquetField child = children.get(i); if (child == null) { // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot - // vector is expected to be pre-populated with nulls by the caller (lands in a follow-up - // PR that rewires ParquetSplitReaderUtil); keep it as is and skip contributing to the - // level stream. + // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector + // (ROW branch); keep it as is and skip contributing to the level stream. finalChildrenVectors[i] = childrenVectors[i]; continue; } @@ -148,11 +147,11 @@ private Tuple2 readRow( setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); } - // Hudi-specific: collapse a present row whose every child is null into a null row. The - // legacy RowColumnReader did this so that a SQL value like `row(null, null)` round-trips - // to NULL on read; preserve it here for backward compatibility. Diverges from Flink 2.1, - // which would surface it as Row(null, null). Mirrored by the integration test - // ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + // Hudi-specific: collapse a present row whose every child is null into a null row, so that a + // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the + // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely + // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the + // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. int rowCount = rowPosition.getPositionsCount(); for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java index 3f0aefe2af74f..72809db1b2ceb 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -71,8 +71,8 @@ * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- - * reader creation boundary in {@code ParquetSplitReaderUtil} (lands in a follow-up PR), not inside - * this class — keeping it a faithful copy of upstream. + * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it + * a faithful copy of upstream. */ public class NestedPrimitiveColumnReader implements ColumnReader { private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java deleted file mode 100644 index 79b50487f13c1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** - * Row {@link ColumnReader}. - */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector; - WritableColumnVector[] vectors = rowColumnVector.vectors; - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowColumnVector.setNullAt(j); - } - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java index 4b48fdd37460f..7cb62824e8543 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -35,8 +35,9 @@ * *

The accessors are wrappers over the existing public fields so legacy callers continue to * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the - * Dremel-style read path is exercised end-to-end by integration tests once - * {@code ParquetSplitReaderUtil} is wired up to {@code NestedColumnReader} in a follow-up PR. + * Dremel-style read path is exercised end-to-end by integration tests in + * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes / + * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes). */ class TestHeapColumnVectorAccessors { From 276565468ef017609ef0dad82348225b9bc5eaa6 Mon Sep 17 00:00:00 2001 From: Xinli Shang Date: Sat, 9 May 2026 19:10:17 -0700 Subject: [PATCH 008/255] docs: Document muttley package as internal/optional for OSS users (#18394) (cherry picked from commit f8e4b9f3db52fcaab05a37b15f7ea7ce8c8b3355) --- .../org/apache/hudi/sink/muttley/README.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md new file mode 100644 index 0000000000000..04c2e31d0e1e5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md @@ -0,0 +1,25 @@ +# Internal Uber Components (Optional) + +This package contains integration with internal Uber services and is **optional** for Apache Hudi users. + +## Components + +- **FlinkHudiMuttleyClient** - Abstract base class providing HTTP retry logic for Muttley RPC communication +- **AthenaIngestionGateway** - Concrete Muttley RPC client for Uber's Athena Ingestion Gateway service (extends `FlinkHudiMuttleyClient`) +- **FlinkHudiMuttleyException** - Base exception for Muttley errors +- **FlinkHudiMuttleyClientException** - Exception for client-side Muttley errors +- **FlinkHudiMuttleyServerException** - Exception for server-side Muttley errors + +## Usage + +These components are used by `FlinkCheckpointClient` to collect Kafka offset metadata and attach it to Hudi commits as part of Uber's Kafka offset tracking feature. + +**Feature flag**: The feature is controlled by `write.extra.metadata.enabled` (default: `false`). It is disabled by default and has no effect in standard Apache Hudi deployments. + +**For open-source Apache Hudi users**: You can safely ignore this package. If `write.extra.metadata.enabled` is inadvertently set to `true` outside of Uber's infrastructure, Kafka offset collection will be silently skipped and Hudi commits will proceed normally (fail-open behavior). + +## Dependencies + +- Runtime: Uber's internal Muttley RPC framework +- Runtime: Access to Athena Ingestion Gateway service +- Compile-time: Jackson (`jackson-databind`) for JSON serialization of RPC request/response payloads From b85f1cc3e14a3ff89f843b784d4346a44e5e69c2 Mon Sep 17 00:00:00 2001 From: chaoyang Date: Sun, 10 May 2026 11:03:19 +0800 Subject: [PATCH 009/255] perf: Reduce unnecessary FSDataOutputStream#hsync to enhance append performance (#17517) * perf: Reduce unnecessary `FSDataOutputStream#hsync` to enhance append performance 1. Reduce unnecessary `FSDataOutputStream#hsync` to enhance append performance Signed-off-by: TheR1sing3un * feat: flush behavior compatible with the block append mode 1. flush behavior compatible with the block append mode Signed-off-by: TheR1sing3un * fixup: address review - drop syncDuringFlush, expose explicit sync() Following @danny0405's suggestion in the PR review, ensure only commit-level visibility on the production path: - Remove the `withSyncDuringFlush` builder option and the `flush(boolean)` overload on HoodieLogFormatWriter; the production path no longer flushes or hsyncs at appendBlocks. - Expose `Writer#sync()` (flush + hsync) as an explicit API for tests that assert per-append visibility on the underlying file system. - closeStream still calls sync() once before close so a closed writer guarantees data is persisted to DataNodes. - Update tests that previously relied on `withSyncDuringFlush(true)` to call `writer.sync()` explicitly before per-append FileStatus size assertions, and rename the related assertion message to drop the misleading "auto-flushed" wording. --------- Signed-off-by: TheR1sing3un (cherry picked from commit 1fffd70db1cb7b7e0441a09842b0d152dfaafa2e) --- .../hudi/common/table/log/HoodieLogFormat.java | 11 +++++++++++ .../common/table/log/HoodieLogFormatWriter.java | 17 +++++++++++------ .../common/functional/TestHoodieLogFormat.java | 9 ++++++--- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index a1103705c3c29..2c536ec077222 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -85,6 +85,17 @@ interface Writer extends Closeable { AppendResult appendBlocks(List blocks) throws IOException, InterruptedException; long getCurrentSize() throws IOException; + + /** + * Force previously appended blocks to durable storage so that downstream + * readers can observe them before this writer is closed. + * + *

Production code paths typically rely on {@link #close()} for + * commit-level visibility and do not need to call this. It is exposed + * mainly for tests that assert per-append visibility on the underlying + * file system. + */ + void sync() throws IOException; } /** diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 23a69699a43ec..1d6acc8d85ef3 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -179,8 +179,10 @@ public AppendResult appendBlocks(List blocks) throws IOException } sizeWritten += outputStream.size() - startSize; } - // Flush all blocks to disk - flush(); + // No flush/hsync here: append-time visibility is not part of the contract. + // Downstream readers only need commit-level visibility, which is provided + // when the writer is closed (see closeStream) or when callers explicitly + // invoke sync(). AppendResult result = new AppendResult(logFile, startPos, sizeWritten); // roll over if size is past the threshold @@ -236,20 +238,23 @@ public void close() throws IOException { private void closeStream() throws IOException { if (output != null) { - flush(); + // Persist all buffered data to DataNodes before closing so downstream + // readers can observe a fully-written log file at commit-level visibility. + sync(); output.close(); output = null; closed = true; } } - private void flush() throws IOException { + @Override + public void sync() throws IOException { if (output == null) { return; // Presume closed } output.flush(); - // NOTE : the following API call makes sure that the data is flushed to disk on DataNodes (akin to POSIX fsync()) - // See more details here : https://issues.apache.org/jira/browse/HDFS-744 + // NOTE: the following API call makes sure that the data is flushed to disk on DataNodes (akin to POSIX fsync()) + // See more details here: https://issues.apache.org/jira/browse/HDFS-744 output.hsync(); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index cde861f8c5f26..7e3ab4ebbbb86 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -234,8 +234,9 @@ public void testBasicAppend(HoodieLogBlockType dataBlockType) throws IOException long size = writer.getCurrentSize(); assertTrue(size > 0, "We just wrote a block - size should be > 0"); + writer.sync(); assertEquals(size, storage.getPathInfo(writer.getLogFile().getPath()).getLength(), - "Write should be auto-flushed. The size reported by FileStatus and the writer should match"); + "After explicit sync, FileStatus length should match the writer's reported size"); assertEquals(size, result.size()); assertEquals(writer.getLogFile(), result.logFile()); assertEquals(0, result.offset()); @@ -376,8 +377,9 @@ public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOExcept writer.appendBlock(dataBlock); long size2 = writer.getCurrentSize(); assertTrue(size2 > size1, "We just wrote a new block - size2 should be > size1"); + writer.sync(); assertEquals(size2, storage.getPathInfo(writer.getLogFile().getPath()).getLength(), - "Write should be auto-flushed. The size reported by FileStatus and the writer should match"); + "After explicit sync, FileStatus length should match the writer's reported size"); writer.close(); // Close and Open again and append 100 more records @@ -394,8 +396,9 @@ public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOExcept writer.appendBlock(dataBlock); long size3 = writer.getCurrentSize(); assertTrue(size3 > size2, "We just wrote a new block - size3 should be > size2"); + writer.sync(); assertEquals(size3, storage.getPathInfo(writer.getLogFile().getPath()).getLength(), - "Write should be auto-flushed. The size reported by FileStatus and the writer should match"); + "After explicit sync, FileStatus length should match the writer's reported size"); writer.close(); // Cannot get the current size after closing the log From 062e4c6ca5788aaba7ab821e5fc662f4f89f8c1d Mon Sep 17 00:00:00 2001 From: Xinli Shang Date: Sun, 10 May 2026 09:26:06 -0700 Subject: [PATCH 010/255] fix(flink): add Apache license header to muttley/README.md (#18713) PR #18394 added hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md without an Apache license header, causing apache-rat:check to fail on every new build of master ("Too many files with unapproved license: 1"). Prepend the standard Apache 2.0 HTML-comment license header so RAT passes. Verified locally: cd hudi-flink-datasource/hudi-flink mvn -Pflink1.20 -Dscala-2.12 apache-rat:check -> Rat check: Summary over all files. Unapproved: 0 Co-authored-by: Xinli Shang (cherry picked from commit f2f6203aa496cd5e502690cc7fea913acc93dd61) --- .../java/org/apache/hudi/sink/muttley/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md index 04c2e31d0e1e5..1f0387cfdb950 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md @@ -1,3 +1,20 @@ + + # Internal Uber Components (Optional) This package contains integration with internal Uber services and is **optional** for Apache Hudi users. From 53559242345bead1a1dff3aac60d832605b0ce97 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Mon, 11 May 2026 13:44:04 +0800 Subject: [PATCH 011/255] feat: add variant type adapter for Flink (#18702) * Add variant type adapter for Flink * address the review comments (cherry picked from commit 5e72c96fae6117135e8b9d3a405c7d9c06003a65) --- .../client/model/AbstractHoodieRowData.java | 3 +- .../hudi/client/model/BootstrapRowData.java | 6 +- .../row/parquet/ParquetSchemaConverter.java | 47 ++++++++++++++++ .../hudi/util/AvroToRowDataConverters.java | 17 ++++++ .../hudi/util/HoodieSchemaConverter.java | 24 +++++--- .../hudi/util/RowDataToAvroConverters.java | 28 ++++++++++ .../hudi/util/TestHoodieSchemaConverter.java | 3 + ...ITTestVariantCrossEngineCompatibility.java | 4 ++ .../apache/hudi/adapter/DataTypeAdapter.java | 53 ++++++++++++++++++ .../apache/hudi/adapter/DataTypeAdapter.java | 53 ++++++++++++++++++ .../apache/hudi/adapter/DataTypeAdapter.java | 53 ++++++++++++++++++ .../apache/hudi/adapter/DataTypeAdapter.java | 53 ++++++++++++++++++ .../apache/hudi/adapter/DataTypeAdapter.java | 53 ++++++++++++++++++ .../apache/hudi/adapter/DataTypeAdapter.java | 56 +++++++++++++++++++ 14 files changed, 441 insertions(+), 12 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/AbstractHoodieRowData.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/AbstractHoodieRowData.java index 94b13ea32a8a0..d01bf3e34d766 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/AbstractHoodieRowData.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/AbstractHoodieRowData.java @@ -18,6 +18,7 @@ package org.apache.hudi.client.model; +import org.apache.hudi.adapter.DataTypeAdapter; import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.util.ValidationUtils; @@ -169,6 +170,6 @@ protected String getMetaColumnVal(int ordinal) { protected abstract int rebaseOrdinal(int ordinal); public Variant getVariant(int i) { - throw new UnsupportedOperationException("Variant is not supported yet."); + return DataTypeAdapter.getVariant(row, rebaseOrdinal(i)); } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/BootstrapRowData.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/BootstrapRowData.java index 92c0f35a42322..23c4c82f96680 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/BootstrapRowData.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/BootstrapRowData.java @@ -18,6 +18,8 @@ package org.apache.hudi.client.model; +import org.apache.hudi.adapter.DataTypeAdapter; + import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.MapData; @@ -149,7 +151,7 @@ private T getValue(int pos, Function getter) { return getter.apply(pos); } - public Variant getVariant(int i) { - throw new UnsupportedOperationException("Variant is not supported yet."); + public Variant getVariant(int pos) { + return DataTypeAdapter.getVariant(row, pos); } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java index d98497fdc86c8..668098314d48b 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java @@ -18,6 +18,8 @@ package org.apache.hudi.io.storage.row.parquet; +import org.apache.hudi.adapter.DataTypeAdapter; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.collection.Pair; import lombok.extern.slf4j.Slf4j; @@ -155,6 +157,15 @@ public static RowType.RowField convertToRowField(Type parquetType) { new MapType( convertToRowField(keyValueType.getLeft()).getType().copy(true), convertToRowField(keyValueType.getRight()).getType())); + } else if (hasVariantAnnotation(logicalType)) { + if (isShreddedVariant(groupType)) { + throw new UnsupportedOperationException( + "Shredded Variant is not supported in Flink. " + + "The Parquet group '" + groupType.getName() + "' contains a '" + + HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD + + "' field indicating a shredded layout."); + } + dataType = DataTypeAdapter.createVariantType(); } else { dataType = DataTypes.of(new RowType( @@ -190,6 +201,39 @@ public static MessageType convertToParquetMessageType(String name, RowType rowTy return new MessageType(name, types); } + /** + * Checks whether the group carries the Parquet {@code VARIANT} logical type annotation. + * Uses class-name matching so this compiles against parquet-java versions that predate the + * {@code VariantLogicalTypeAnnotation} class (< 1.15.2). + */ + private static boolean hasVariantAnnotation(LogicalTypeAnnotation logicalType) { + // needs to ensure the writer attach the variant annotation in 1.3. + return logicalType != null + && logicalType.getClass().getSimpleName().equals("VariantLogicalTypeAnnotation"); + } + + /** + * Checks whether a variant group contains a {@code typed_value} field, indicating a shredded + * layout. Called only after {@link #hasVariantAnnotation} returns true. + */ + private static boolean isShreddedVariant(GroupType groupType) { + return groupType.containsField(HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD); + } + + /** + * Converts a Variant column to the canonical unshredded Parquet layout: + * a group with required binary {@code metadata} and required binary {@code value}. + */ + private static Type convertVariantToParquetType(String name, Type.Repetition repetition) { + // TODO: add .as(LogicalTypeAnnotation.variantType()) once parquet-java is bumped to 1.16.0 + return Types.buildGroup(repetition) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED) + .named(HoodieSchema.Variant.VARIANT_METADATA_FIELD)) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED) + .named(HoodieSchema.Variant.VARIANT_VALUE_FIELD)) + .named(name); + } + private static Type convertToParquetType( String name, LogicalType type, Type.Repetition repetition) { switch (type.getTypeRoot()) { @@ -304,6 +348,9 @@ private static Type convertToParquetType( .addField(convertToParquetType(field.getName(), field.getType(), field.getType().isNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED))); return builder.named(name); default: + if (DataTypeAdapter.isVariantType(type)) { + return convertVariantToParquetType(name, repetition); + } throw new UnsupportedOperationException("Unsupported type: " + type); } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java index fee24e520382d..d63ce350b487c 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java @@ -18,6 +18,8 @@ package org.apache.hudi.util; +import org.apache.hudi.adapter.DataTypeAdapter; + import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.apache.avro.generic.GenericFixed; @@ -155,6 +157,9 @@ public static AvroToRowDataConverter createConverter(LogicalType type, boolean u case MULTISET: return createMapConverter(type, utcTimezone); default: + if (DataTypeAdapter.isVariantType(type)) { + return createVariantConverter(); + } throw new UnsupportedOperationException("Unsupported type: " + type); } } @@ -212,6 +217,18 @@ private static AvroToRowDataConverter createMapConverter(LogicalType type, boole }; } + /** + * Creates a converter for Flink 2.1+ VARIANT LogicalType. The converter receives an Avro + * GenericRecord carrying metadata/value binary fields and produces a Flink + * {@code BinaryVariant}. + */ + private static AvroToRowDataConverter createVariantConverter() { + return avroObject -> { + IndexedRecord record = (IndexedRecord) avroObject; + return DataTypeAdapter.createVariant(convertToBytes(record.get(1)), convertToBytes(record.get(0))); + }; + } + private static AvroToRowDataConverter createTimestampConverter(int precision, boolean utcTimezone) { final ChronoUnit chronoUnit; if (precision <= 3) { diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java index 0b3a1af7982b7..c49142800b456 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java @@ -19,6 +19,7 @@ package org.apache.hudi.util; +import org.apache.hudi.adapter.DataTypeAdapter; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; @@ -222,6 +223,10 @@ public static HoodieSchema convertToSchema(LogicalType logicalType, String rowNa case RAW: default: + if (DataTypeAdapter.isVariantType(logicalType)) { + schema = HoodieSchema.createVariant(); + break; + } throw new UnsupportedOperationException( "Unsupported type for HoodieSchema conversion: " + logicalType); } @@ -576,23 +581,24 @@ private static DataType convertUnion(HoodieSchema schema) { } /** - * Converts a Variant schema to Flink's ROW type. - * Variant is represented as ROW<`metadata` BYTES, `value` BYTES> in Flink. + * Converts a Variant HoodieSchema to the native Flink {@code VariantType} DataType. + * Requires Flink 2.1+ at runtime; throws {@link UnsupportedOperationException} on older versions. * * @param schema HoodieSchema to convert (must be a VARIANT type) - * @return DataType representing the Variant as a ROW with binary fields + * @return native VariantType DataType + * @throws UnsupportedOperationException if Flink runtime is pre-2.1 or variant is shredded */ private static DataType convertVariant(HoodieSchema schema) { if (schema.getType() != HoodieSchemaType.VARIANT) { throw new IllegalStateException("Expected HoodieSchema.Variant but got: " + schema.getClass()); } - // Variant is stored as a struct with two binary fields: metadata and value. - // Field order follows the Parquet spec and Iceberg convention (metadata first, value second). - return DataTypes.ROW( - DataTypes.FIELD("metadata", DataTypes.BYTES().notNull()), - DataTypes.FIELD("value", DataTypes.BYTES().notNull()) - ).notNull(); + if (((HoodieSchema.Variant) schema).isShredded()) { + throw new UnsupportedOperationException( + "Shredded Variant is not yet supported in Flink. Use unshredded Variant instead."); + } + + return DataTypeAdapter.createVariantType().notNull(); } /** diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java index 9eb2979e19aa0..6bf94b5270747 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java @@ -18,6 +18,7 @@ package org.apache.hudi.util; +import org.apache.hudi.adapter.DataTypeAdapter; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; @@ -239,6 +240,10 @@ public Object convert(HoodieSchema schema, Object object) { break; case RAW: default: + if (DataTypeAdapter.isVariantType(type)) { + converter = createVariantConverter(); + break; + } throw new UnsupportedOperationException("Unsupported type: " + type); } @@ -357,5 +362,28 @@ public Object convert(HoodieSchema schema, Object object) { } }; } + + /** + * Creates a converter for Flink 2.1+ VARIANT LogicalType. The converter receives a Flink + * {@code Variant} object at runtime and extracts the raw metadata/value byte arrays, + * then packs them into an Avro GenericRecord with the Variant schema. + * + *

No shredded-variant check is needed here: {@code HoodieSchemaConverter.convertVariant()} + * already rejects shredded variants before a Flink type or converter is ever constructed, + * and Flink 2.1 itself only supports unshredded variants (FLIP-521). + */ + private static RowDataToAvroConverter createVariantConverter() { + return new RowDataToAvroConverter() { + private static final long serialVersionUID = 1L; + + @Override + public Object convert(HoodieSchema schema, Object object) { + final GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put(HoodieSchema.Variant.VARIANT_METADATA_FIELD, ByteBuffer.wrap(DataTypeAdapter.getVariantMetadata(object))); + record.put(HoodieSchema.Variant.VARIANT_VALUE_FIELD, ByteBuffer.wrap(DataTypeAdapter.getVariantValue(object))); + return record; + } + }; + } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java index e2fd16d7112b0..51b8a15f4c7bc 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java @@ -36,6 +36,7 @@ import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.logical.VarBinaryType; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -692,6 +693,7 @@ public void testBlobInNestedStructures() { } @Test + @Disabled("disabled and reopen the tests for 1.3") public void testVariantTypeConversion() { // Test direct Variant conversion HoodieSchema variantSchema = HoodieSchema.createVariant(); @@ -708,6 +710,7 @@ public void testVariantTypeConversion() { } @Test + @Disabled("disabled and reopen the tests for 1.3") public void testVariantInRecordConversion() { // Test Variant field within a record HoodieSchema recordWithVariant = HoodieSchema.createRecord( diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java index 9aade5503b4cc..03305d478ecd6 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java @@ -27,6 +27,7 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.types.Row; import org.apache.flink.util.CollectionUtil; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -114,6 +115,7 @@ private void verifyFlinkCanReadSparkVariantTable(String tablePath, String tableT } @Test + @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantCOWTable() throws Exception { // Test that Flink can read a COW table with Variant data written by Spark 4.0 Path cowTargetDir = tempDir.resolve("cow"); @@ -123,6 +125,7 @@ public void testFlinkReadSparkVariantCOWTable() throws Exception { } @Test + @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantMORTableWithAvro() throws Exception { // Test that Flink can read a MOR table with AVRO record type and Variant data written by Spark 4.0 Path morAvroTargetDir = tempDir.resolve("mor_avro"); @@ -132,6 +135,7 @@ public void testFlinkReadSparkVariantMORTableWithAvro() throws Exception { } @Test + @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantMORTableWithSpark() throws Exception { // Test that Flink can read a MOR table with SPARK record type and Variant data written by Spark 4.0 Path morSparkTargetDir = tempDir.resolve("mor_spark"); diff --git a/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..79b9e254dd61d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static boolean isVariantType(LogicalType logicalType) { + return false; + } + + public static DataType createVariantType() { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantMetadata(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantValue(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..79b9e254dd61d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static boolean isVariantType(LogicalType logicalType) { + return false; + } + + public static DataType createVariantType() { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantMetadata(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantValue(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..79b9e254dd61d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static boolean isVariantType(LogicalType logicalType) { + return false; + } + + public static DataType createVariantType() { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantMetadata(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantValue(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..79b9e254dd61d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static boolean isVariantType(LogicalType logicalType) { + return false; + } + + public static DataType createVariantType() { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantMetadata(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantValue(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..566ca723648a4 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static boolean isVariantType(LogicalType logicalType) { + return false; + } + + public static DataType createVariantType() { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantMetadata(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } + + public static byte[] getVariantValue(Object obj) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} \ No newline at end of file diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java new file mode 100644 index 0000000000000..6ae4c4a6babae --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.types.variant.BinaryVariant; +import org.apache.flink.types.variant.Variant; + +/** + * Adapter utils to provide {@code DataType} utilities. + */ +public class DataTypeAdapter { + public static Variant getVariant(RowData rowData, int pos) { + return rowData.getVariant(pos); + } + + public static Object createVariant(byte[] value, byte[] metadata) { + return new BinaryVariant(value, metadata); + } + + public static boolean isVariantType(LogicalType logicalType) { + return logicalType.getTypeRoot() == LogicalTypeRoot.VARIANT; + } + + public static DataType createVariantType() { + return DataTypes.VARIANT(); + } + + public static byte[] getVariantMetadata(Object obj) { + return ((BinaryVariant) obj).getMetadata(); + } + + public static byte[] getVariantValue(Object obj) { + return ((BinaryVariant) obj).getValue(); + } +} From a3272bd3293a7b01a66bb1efd602d6d1ed430004 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Tue, 12 May 2026 17:09:39 +0800 Subject: [PATCH 012/255] chore: migrate the flink ITs run to flink2.1 (#18717) * chore: migrate the flink ITs to run with flink2.1 * fix compile errors * fix test failures * fix test failure * address review comments (cherry picked from commit b6bc1658e8f2862583c22425eaa0665ae100f5a3) --- .github/workflows/bot.yml | 30 +++---- azure-pipelines-20230430.yml | 4 +- .../hudi/util/TestHoodieSchemaConverter.java | 23 ++--- .../sink/bootstrap/RLIBootstrapOperator.java | 27 ++++-- .../bootstrap/TestRLIBootstrapOperator.java | 80 ++++++++++++++++++ ...ITTestVariantCrossEngineCompatibility.java | 31 +++---- .../adapter/DataTypeAdapterTestUtils.java | 28 +++++++ .../adapter/DataTypeAdapterTestUtils.java | 28 +++++++ .../adapter/DataTypeAdapterTestUtils.java | 28 +++++++ .../adapter/DataTypeAdapterTestUtils.java | 28 +++++++ .../runtime/kryo/KryoSerializerSnapshot.java | 3 + .../adapter/DataTypeAdapterTestUtils.java | 28 +++++++ .../format/cow/ParquetSplitReaderUtil.java | 83 +++++++++++++++++++ .../runtime/kryo/KryoSerializerSnapshot.java | 3 + .../adapter/DataTypeAdapterTestUtils.java | 32 +++++++ pom.xml | 14 ++-- 16 files changed, 404 insertions(+), 66 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bootstrap/TestRLIBootstrapOperator.java create mode 100644 hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 1e50c64121cbe..06468d0c4fa23 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -67,7 +67,7 @@ jobs: include: - scalaProfile: "scala-2.12" sparkProfile: "spark3.5" - flinkProfile: "flink1.18" + flinkProfile: "flink2.1" steps: - uses: actions/checkout@v5 @@ -126,7 +126,7 @@ jobs: include: - scalaProfile: "scala-2.12" sparkProfile: "spark3.5" - flinkProfile: "flink1.18" + flinkProfile: "flink2.1" steps: - uses: actions/checkout@v5 @@ -178,7 +178,7 @@ jobs: include: - scalaProfile: "scala-2.12" sparkProfile: "spark3.5" - flinkProfile: "flink1.18" + flinkProfile: "flink2.1" env: UT_MODULES: >- @@ -563,7 +563,7 @@ jobs: include: - scalaProfile: "scala-2.12" sparkProfile: "spark3.5" - flinkProfile: "flink1.20" + flinkProfile: "flink2.1" steps: - uses: actions/checkout@v5 @@ -938,7 +938,7 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }} FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }} - if: ${{ endsWith(env.FLINK_PROFILE, '1.20') }} + if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }} run: | mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS @@ -948,9 +948,9 @@ jobs: strategy: matrix: include: - - flinkProfile: "flink1.20" + - flinkProfile: "flink2.1" flinkAvroVersion: "1.11.4" - flinkParquetVersion: '1.13.1' + flinkParquetVersion: '1.15.2' steps: - uses: actions/checkout@v5 - name: Set up JDK 11 @@ -974,7 +974,7 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }} FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }} - if: ${{ endsWith(env.FLINK_PROFILE, '1.20') }} + if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }} run: | mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS @@ -985,19 +985,19 @@ jobs: matrix: include: - scalaProfile: 'scala-2.13' - flinkProfile: 'flink1.20' + flinkProfile: 'flink2.1' sparkProfile: 'spark3.5' sparkRuntime: 'spark3.5.0' - scalaProfile: 'scala-2.12' - flinkProfile: 'flink1.20' + flinkProfile: 'flink2.1' sparkProfile: 'spark3.5' sparkRuntime: 'spark3.5.0' - scalaProfile: 'scala-2.13' - flinkProfile: 'flink1.20' + flinkProfile: 'flink2.1' sparkProfile: 'spark4.0' sparkRuntime: 'spark4.0.0' - scalaProfile: 'scala-2.13' - flinkProfile: 'flink1.20' + flinkProfile: 'flink2.1' sparkProfile: 'spark4.1' sparkRuntime: 'spark4.1.1' @@ -1201,7 +1201,7 @@ jobs: matrix: include: - sparkProfile: 'spark3.5' - flinkProfile: 'flink1.20' + flinkProfile: 'flink2.1' sparkArchive: 'spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz' steps: - uses: actions/checkout@v5 @@ -1297,9 +1297,9 @@ jobs: matrix: include: - scalaProfile: "scala-2.12" - flinkProfile: "flink1.20" + flinkProfile: "flink2.1" flinkAvroVersion: '1.11.4' - flinkParquetVersion: '1.13.1' + flinkParquetVersion: '1.15.2' steps: - uses: actions/checkout@v5 - name: Set up JDK 17 diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 10fab9f349924..3edac6046b1a3 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -14,7 +14,7 @@ # limitations under the License. # NOTE: -# This config file defines how Azure CI runs tests with Spark 3.5 and Flink 1.18 profiles. +# This config file defines how Azure CI runs tests with Spark 3.5 and Flink 2.1 profiles. # PRs will need to keep in sync with master's version to trigger the CI runs. # See scripts/jacoco/README.md for how aggregated code coverage report works # across multiple modules. @@ -101,7 +101,7 @@ parameters: - '!packaging/hudi-utilities-slim-bundle' variables: - BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink1.18' + BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink2.1' PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5 -Dorg.eclipse.jetty.LEVEL=WARN' MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java index 51b8a15f4c7bc..9a4f920b859e4 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java @@ -36,11 +36,12 @@ import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.logical.VarBinaryType; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Arrays; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -693,24 +694,18 @@ public void testBlobInNestedStructures() { } @Test - @Disabled("disabled and reopen the tests for 1.3") public void testVariantTypeConversion() { // Test direct Variant conversion HoodieSchema variantSchema = HoodieSchema.createVariant(); DataType dataType = HoodieSchemaConverter.convertToDataType(variantSchema); assertNotNull(dataType); - // Verify it's a ROW with metadata and value binary fields - RowType rowType = (RowType) dataType.getLogicalType(); - assertEquals(2, rowType.getFieldCount()); - assertEquals("metadata", rowType.getFieldNames().get(0)); - assertEquals("value", rowType.getFieldNames().get(1)); - assertInstanceOf(VarBinaryType.class, rowType.getTypeAt(0)); - assertInstanceOf(VarBinaryType.class, rowType.getTypeAt(1)); + // Verify it's a Variant + assertThat("the return type should be variant", + dataType.getLogicalType().asSummaryString(), is("VARIANT NOT NULL")); } @Test - @Disabled("disabled and reopen the tests for 1.3") public void testVariantInRecordConversion() { // Test Variant field within a record HoodieSchema recordWithVariant = HoodieSchema.createRecord( @@ -727,11 +722,9 @@ public void testVariantInRecordConversion() { assertEquals(2, result.getFieldCount()); assertEquals("data", result.getFieldNames().get(1)); - // Verify variant field is a ROW - RowType variantRowType = (RowType) result.getTypeAt(1); - assertEquals(2, variantRowType.getFieldCount()); - assertEquals("metadata", variantRowType.getFieldNames().get(0)); - assertEquals("value", variantRowType.getFieldNames().get(1)); + // Verify variant field + assertThat("the return type should be variant", + result.getTypeAt(1).asSummaryString(), is("VARIANT NOT NULL")); } @Test diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/RLIBootstrapOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/RLIBootstrapOperator.java index 515b180d3cad0..d0cb16a789e53 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/RLIBootstrapOperator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/RLIBootstrapOperator.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.function.SerializableFunctionUnchecked; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.metadata.HoodieBackedTableMetadata; @@ -52,7 +53,7 @@ public class RLIBootstrapOperator extends AbstractBootstrapOperator { - private transient HoodieBackedTableMetadata metadataTable; + private transient HoodieBackedTableMetadata tableMetadata; private transient long loadedCnt; public RLIBootstrapOperator(Configuration conf) { @@ -63,13 +64,13 @@ public RLIBootstrapOperator(Configuration conf) { public void initializeState(StateInitializationContext context) throws Exception { loadedCnt = 0; HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf); - this.metadataTable = (HoodieBackedTableMetadata) metaClient.getTableFormat().getMetadataFactory().create( + this.tableMetadata = new HoodieBackedTableMetadata( HoodieFlinkEngineContext.DEFAULT, metaClient.getStorage(), StreamerUtil.metadataConfig(conf), conf.get(FlinkOptions.PATH)); // Load RLI records - preLoadRLIRecords(); + preLoadRLIRecords(metaClient.getTableConfig()); } @Override @@ -82,10 +83,20 @@ public void close() throws Exception { // Utilities // ------------------------------------------------------------------------- - private void preLoadRLIRecords() { + private void preLoadRLIRecords(HoodieTableConfig tableConfig) { int taskID = RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()); int parallelism = RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()); + if (!tableMetadata.enabled()) { + if (tableConfig.isMetadataTableAvailable()) { + throw new RuntimeException("Can not initialize the table metadata"); + } + log.info("Skip loading RLI records because table metadata is not initialized, taskId = {}", taskID); + waitForBootstrapReady(taskID); + closeMetadataTable(); + return; + } + log.info("Start loading RLI records from metadata table, taskId = {}, parallelism = {}", taskID, parallelism); SerializableFunctionUnchecked, List> fileSlicesFilter = fileSlices -> { @@ -102,7 +113,7 @@ private void preLoadRLIRecords() { // Each subtask loads buckets assigned to it long startTime = System.currentTimeMillis(); - HoodiePairData rliData = metadataTable.readRecordIndexLocations(fileSlicesFilter); + HoodiePairData rliData = tableMetadata.readRecordIndexLocations(fileSlicesFilter); rliData.forEach(locationPair -> emitIndexRecord(locationPair.getLeft(), locationPair.getRight())); long costMs = System.currentTimeMillis() - startTime; log.info("Finish loading RLI records, total records: {}, cost: {} ms, taskId = {}", loadedCnt, costMs, taskID); @@ -133,13 +144,13 @@ private void emitIndexRecord(String recordKey, HoodieRecordGlobalLocation locati } private void closeMetadataTable() { - if (metadataTable != null) { + if (tableMetadata != null) { try { - metadataTable.close(); + tableMetadata.close(); } catch (Exception e) { log.warn("Failed to close metadata table", e); } - metadataTable = null; + tableMetadata = null; } } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bootstrap/TestRLIBootstrapOperator.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bootstrap/TestRLIBootstrapOperator.java new file mode 100644 index 0000000000000..2e684d0652ae5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bootstrap/TestRLIBootstrapOperator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.bootstrap; + +import org.apache.hudi.client.model.HoodieFlinkInternalRow; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.metadata.MetadataPartitionType; +import org.apache.hudi.util.StreamerUtil; +import org.apache.hudi.utils.TestConfigurations; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link RLIBootstrapOperator}. + */ +public class TestRLIBootstrapOperator { + + @TempDir + File tempFile; + + @Test + void testSkipPreloadForFreshTableWithoutMetadataTable() throws Exception { + Configuration conf = getRLIConf(); + StreamerUtil.initTableIfNotExists(conf); + + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(new RLIBootstrapOperator(conf), 1, 1, 0)) { + harness.open(); + + assertEquals(0, harness.getOutput().size()); + } + } + + @Test + void testFailFastWhenMetadataTableIsMarkedAvailableButCannotBeLoaded() throws Exception { + Configuration conf = getRLIConf(); + HoodieTableMetaClient metaClient = StreamerUtil.initTableIfNotExists(conf); + metaClient.getTableConfig().setMetadataPartitionState(metaClient, MetadataPartitionType.FILES.getPartitionPath(), true); + + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(new RLIBootstrapOperator(conf), 1, 1, 0)) { + RuntimeException error = assertThrows(RuntimeException.class, harness::open); + + assertEquals("Can not initialize the table metadata", error.getMessage()); + } + } + + private Configuration getRLIConf() { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.METADATA_ENABLED, true); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX.name()); + return conf; + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java index 03305d478ecd6..de6e3835d120a 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java @@ -18,6 +18,8 @@ package org.apache.hudi.table; +import org.apache.hudi.adapter.DataTypeAdapter; +import org.apache.hudi.adapter.DataTypeAdapterTestUtils; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.utils.FlinkMiniCluster; @@ -27,7 +29,6 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.types.Row; import org.apache.flink.util.CollectionUtil; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -51,19 +52,16 @@ public class ITTestVariantCrossEngineCompatibility { /** * Helper method to verify that Flink can read Spark 4.0 Variant tables. - * Variant data is represented as ROW in Flink. */ private void verifyFlinkCanReadSparkVariantTable(String tablePath, String tableType, String testDescription) throws Exception { TableEnvironment tableEnv = TestTableEnvs.getBatchTableEnv(); // Create a Hudi table pointing to the Spark-written data - // In Flink, Variant is represented as ROW - // NOTE: value is a reserved keyword String createTableDdl = String.format( "CREATE TABLE variant_table (" + " id INT," + " name STRING," - + " v ROW," + + " v VARIANT," + " ts BIGINT," + " PRIMARY KEY (id) NOT ENFORCED" + ") WITH (" @@ -87,35 +85,32 @@ private void verifyFlinkCanReadSparkVariantTable(String tablePath, String tableT assertEquals("row1", row.getField(1), "Second column should be name=row1"); assertEquals(1000L, row.getField(3), "Fourth column should be ts=1000"); - // Verify the variant column is readable as a ROW with binary fields - Row variantRow = (Row) row.getField(2); - assertNotNull(variantRow, "Variant column should not be null"); - - byte[] metadataBytes = (byte[]) variantRow.getField(0); - byte[] valueBytes = (byte[]) variantRow.getField(1); + // Verify the variant column is readable as a native Flink Variant. + Object variantObject = row.getField(2); + assertNotNull(variantObject, "Variant column should not be null"); + DataTypeAdapterTestUtils.assertAsBinaryVariant(variantObject); // Expected byte values from Spark 4.0 Variant representation: {"updated": true, "new_field": 123} byte[] expectedValueBytes = new byte[]{0x02, 0x02, 0x01, 0x00, 0x01, 0x00, 0x03, 0x04, 0x0C, 0x7B}; byte[] expectedMetadataBytes = new byte[]{0x01, 0x02, 0x00, 0x07, 0x10, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x6E, 0x65, 0x77, 0x5F, 0x66, 0x69, 0x65, 0x6C, 0x64}; - assertArrayEquals(expectedValueBytes, valueBytes, + assertArrayEquals(expectedValueBytes, DataTypeAdapter.getVariantValue(variantObject), String.format("Variant value bytes mismatch (%s). Expected: %s, Got: %s", testDescription, Arrays.toString(StringUtils.encodeHex(expectedValueBytes)), - Arrays.toString(StringUtils.encodeHex(valueBytes)))); + Arrays.toString(StringUtils.encodeHex(DataTypeAdapter.getVariantValue(variantObject))))); - assertArrayEquals(expectedMetadataBytes, metadataBytes, + assertArrayEquals(expectedMetadataBytes, DataTypeAdapter.getVariantMetadata(variantObject), String.format("Variant metadata bytes mismatch (%s). Expected: %s, Got: %s", testDescription, Arrays.toString(StringUtils.encodeHex(expectedMetadataBytes)), - Arrays.toString(StringUtils.encodeHex(metadataBytes)))); + Arrays.toString(StringUtils.encodeHex(DataTypeAdapter.getVariantMetadata(variantObject))))); tableEnv.executeSql("DROP TABLE variant_table"); } @Test - @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantCOWTable() throws Exception { // Test that Flink can read a COW table with Variant data written by Spark 4.0 Path cowTargetDir = tempDir.resolve("cow"); @@ -125,7 +120,6 @@ public void testFlinkReadSparkVariantCOWTable() throws Exception { } @Test - @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantMORTableWithAvro() throws Exception { // Test that Flink can read a MOR table with AVRO record type and Variant data written by Spark 4.0 Path morAvroTargetDir = tempDir.resolve("mor_avro"); @@ -135,7 +129,6 @@ public void testFlinkReadSparkVariantMORTableWithAvro() throws Exception { } @Test - @Disabled("disabled and reopen the tests for 1.3") public void testFlinkReadSparkVariantMORTableWithSpark() throws Exception { // Test that Flink can read a MOR table with SPARK record type and Variant data written by Spark 4.0 Path morSparkTargetDir = tempDir.resolve("mor_spark"); @@ -143,4 +136,4 @@ public void testFlinkReadSparkVariantMORTableWithSpark() throws Exception { String morSparkPath = morSparkTargetDir.resolve("variant_mor_spark").toString(); verifyFlinkCanReadSparkVariantTable(morSparkPath, "MERGE_ON_READ", "MOR table with SPARK record type"); } -} \ No newline at end of file +} diff --git a/hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..ae2e4107d6ea7 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..ae2e4107d6ea7 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..ae2e4107d6ea7 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..ae2e4107d6ea7 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java index 1a256ffe7bba6..aac9a4651aff3 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java @@ -34,6 +34,9 @@ public class KryoSerializerSnapshot implements TypeSerializerSnapshot { private Class type; + public KryoSerializerSnapshot() { + } + public KryoSerializerSnapshot(Class type) { this.type = type; } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..ae2e4107d6ea7 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + throw new UnsupportedOperationException("Variant is not supported yet."); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index bb5d0c55b81de..3e16417ba541e 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -18,6 +18,7 @@ package org.apache.hudi.table.format.cow; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; @@ -320,6 +321,17 @@ private static ColumnVector createVectorFromConstant( } else { throw new UnsupportedOperationException("Unsupported create row with default value."); } + case VARIANT: + HeapRowColumnVector variantVector = new HeapRowColumnVector( + batchSize, + new HeapBytesVector(batchSize), + new HeapBytesVector(batchSize)); + if (value == null) { + variantVector.fillWithNulls(); + return variantVector; + } else { + throw new UnsupportedOperationException("Unsupported create variant with default value."); + } default: throw new UnsupportedOperationException("Unsupported type: " + type); } @@ -498,6 +510,20 @@ private static ColumnReader createColumnReader( } } return new RowColumnReader(fieldReaders); + case VARIANT: + ColumnDescriptor valueDescriptor = getVariantColumnDescriptor( + physicalType, + descriptors, + depth, + HoodieSchema.Variant.VARIANT_VALUE_FIELD); + ColumnDescriptor metadataDescriptor = getVariantColumnDescriptor( + physicalType, + descriptors, + depth, + HoodieSchema.Variant.VARIANT_METADATA_FIELD); + return new RowColumnReader(Arrays.asList( + new BytesColumnReader(valueDescriptor, pages.getPageReader(valueDescriptor)), + new BytesColumnReader(metadataDescriptor, pages.getPageReader(metadataDescriptor)))); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } @@ -675,11 +701,68 @@ private static WritableColumnVector createWritableColumnVector( } } return new HeapRowColumnVector(batchSize, columnVectors); + case VARIANT: + validateVariantType(physicalType); + return new HeapRowColumnVector( + batchSize, + new HeapBytesVector(batchSize), + new HeapBytesVector(batchSize)); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + private static ColumnDescriptor getVariantColumnDescriptor( + Type physicalType, + List descriptors, + int depth, + String fieldName) { + return descriptors.stream() + .filter(descriptor -> descriptor.getPath().length > depth + 1 + && fieldName.equals(descriptor.getPath()[depth + 1])) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "Invalid Variant Parquet schema: missing binary field '" + fieldName + "'.")); + } + + private static void validateVariantType(Type physicalType) { + if (!physicalType.isPrimitive()) { + GroupType groupType = physicalType.asGroupType(); + if (isShreddedVariant(groupType)) { + throw new UnsupportedOperationException( + "Shredded Variant is not supported in Flink. " + + "The Parquet group '" + groupType.getName() + "' contains a '" + + HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD + + "' field indicating a shredded layout."); + } + validateVariantField(groupType, HoodieSchema.Variant.VARIANT_VALUE_FIELD); + validateVariantField(groupType, HoodieSchema.Variant.VARIANT_METADATA_FIELD); + } else { + throw new IllegalArgumentException( + "Type mismatch, expected Variant but got '" + physicalType + "'."); + } + } + + /** + * Checks whether a variant group contains a {@code typed_value} field, indicating a shredded + * layout. + */ + private static boolean isShreddedVariant(GroupType groupType) { + return groupType.containsField(HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD); + } + + private static void validateVariantField(GroupType groupType, String fieldName) { + if (groupType.containsField(fieldName)) { + Type fieldType = groupType.getType(fieldName); + if (fieldType.isPrimitive() + && fieldType.asPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY) { + return; + } + } + throw new IllegalArgumentException( + "Invalid Variant Parquet schema: missing binary field '" + fieldName + "'."); + } + /** * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. * diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java index 1a256ffe7bba6..aac9a4651aff3 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java @@ -34,6 +34,9 @@ public class KryoSerializerSnapshot implements TypeSerializerSnapshot { private Class type; + public KryoSerializerSnapshot() { + } + public KryoSerializerSnapshot(Class type) { this.type = type; } diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java new file mode 100644 index 0000000000000..2727b40b13934 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.adapter; + +import org.apache.flink.types.variant.BinaryVariant; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * Adapter utils. + */ +public class DataTypeAdapterTestUtils { + public static void assertAsBinaryVariant(Object variantObject) { + assertInstanceOf(BinaryVariant.class, variantObject, "Variant column should be a BinaryVariant"); + } +} diff --git a/pom.xml b/pom.xml index cd26cc509f36c..0e556cc76c1cd 100644 --- a/pom.xml +++ b/pom.xml @@ -153,14 +153,14 @@ 1.19.2 1.18.1 1.17.1 - ${flink1.20.version} - hudi-flink1.20.x - 1.20 + ${flink2.1.version} + hudi-flink2.1.x + 2.1 1.11.4 - 1.13.1 + 1.15.2 - 3.3.0-1.20 + 4.0.1-2.0 flink-runtime flink-table-runtime flink-table-planner_2.12 @@ -170,7 +170,7 @@ flink-streaming-java flink-clients flink-connector-kafka - flink-hadoop-compatibility_2.12 + flink-hadoop-compatibility 7.5.3 3.3.4 3.4.3 @@ -2899,6 +2899,7 @@ hudi-flink-datasource/hudi-flink2.1.x + true flink2.1 @@ -2942,7 +2943,6 @@ hudi-flink-datasource/hudi-flink1.20.x - true flink1.20 From 64974eb0e9e9a947ca0e1184d9a1f98865998a14 Mon Sep 17 00:00:00 2001 From: Xinli Shang Date: Sat, 16 May 2026 20:37:56 -0700 Subject: [PATCH 013/255] feat(utilities): add Spark/HoodieStreamer validators for pre-commit validation - Phase 3 (#18405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add Spark streamer validators for phase 3 precommit validation Implements phase 3 of the precommit validation framework by adding: - SparkKafkaOffsetValidator: Validates Kafka offset consistency - SparkValidationContext: Provides Spark-specific validation context - SparkStreamerValidatorUtils: Utility functions for Spark streamer validation - Comprehensive test coverage for all validator components - Integration with StreamSync and HoodiePreCommitValidatorConfig Co-Authored-By: Claude Opus 4.6 * fix: address code review and fix checkstyle violations - Remove unused imports (java.io.IOException, HoodieCommitMetadata, HoodieTestTable, Option) that caused checkstyle build failures - Remove accidentally committed bootstrap_register_only_issue.md - Cache writeStatusRDD before collect() to prevent second DAG evaluation and potential driver OOM - Add comment explaining why validator runs before writeClient.commit(): offset validation is a stronger guard than commitOnErrors and must prevent the commit when data loss is detected - Clarify buildCommitMetadata() produces a pre-commit preview object, not a fully-constructed commit record - Add Javadoc to SparkKafkaOffsetValidator and SparkStreamerValidatorUtils explaining incompatibility with SparkValidatorUtils (different interface and constructor signature) to prevent misconfiguration - Add two-commit integration tests (testSecondCommitMatchingOffsetsPasses, testSecondCommitDataLossDetected) using HoodieTestTable to exercise the real offset comparison path, not just the first-commit skip path * fix: skip non-SparkPreCommitValidator classes in SparkValidatorUtils SparkKafkaOffsetValidator (and similar streaming validators) extend BasePreCommitValidator with a (TypedProperties) constructor, not the (HoodieSparkTable, HoodieEngineContext, HoodieWriteConfig) constructor that SparkValidatorUtils expects. Listing such a validator in hoodie.precommit.validators previously caused a reflection error in the Spark table write path. Add a Class.isAssignableFrom check to filter out classes that don't implement SparkPreCommitValidator before attempting instantiation, with a clear warning pointing users to SparkStreamerValidatorUtils for streaming validators. * ci: trigger CI re-run for flaky trino test * fix: address reviewer comments on pre-commit streaming offset validator - Unpersist cached RDD in finally block to prevent executor memory leak - Let IOException propagate from loadPreviousCommitMetadata instead of silently swallowing it - Filter empty validator class names before Class.forName to handle trailing comma in config - Add write error count to validation message to distinguish write failures from silent data loss * fix: address reviewer follow-up comments on pre-commit streaming validator - Change runValidators to accept List instead of JavaRDD to fix RDD unpersist-before-commit bug; StreamSync now caches the RDD, collects to list for validators, passes RDD to commit, then unpersists - Remove generic catch(Exception) in loadPreviousCommitMetadata so non-IOException failures propagate instead of silently skipping validation - Implement getPreviousCommitInstant() in SparkValidationContext via timeline lookup instead of throwing UnsupportedOperationException - Add Objects::nonNull filter when building writeStats list - Add BasePreCommitValidator assignability guard in SparkStreamerValidatorUtils to warn and skip SparkPreCommitValidator classes (reverse-direction guard) - Eliminate double class loading in SparkValidatorUtils by combining filter+map into a single flatMap; remove unused ReflectionUtils import - Remove trivial constructor Javadoc from SparkKafkaOffsetValidator - Add HoodieTestUtils import in test; remove Spark context boilerplate now that runValidators accepts List directly * fix: guard cache() call when writeStatusRDD is already persisted Calling cache() on an RDD that already has a storage level assigned throws SparkUnsupportedOperationException. The write path may cache the RDD internally before returning it. Track whether we own the cache and only call cache()/unpersist() when the RDD was not already persisted. * Ensure writeStatusRDD is always unpersisted via try/finally * fix: use proper import for StorageLevel instead of fully-qualified class reference 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude Sonnet 4.6 * fix: address final reviewer feedback on pre-commit streaming validator - Move runValidators() inside the try/finally so writeStatusRDD.unpersist() always runs, including on validator exceptions (FAIL policy or HoodieIOException from loadPreviousCommitMetadata). - Use ReflectionUtils.loadClass in SparkValidatorUtils for instantiation, matching SparkStreamerValidatorUtils and the rest of the codebase. - Rename weOwnCache to shouldUnpersist to read in the direction of its actual use (gating unpersist in finally). * fix: only cache writeStatusRDD when pre-commit validators are configured Address danny0405's review comment on PR #18405: skip the .cache()/.unpersist() cycle when no pre-commit validators are configured, since without validators the RDD is consumed exactly once by writeClient.commit() and caching adds no value. Guards both the cache call and the validator collect+run on a single validatorsConfigured boolean derived from hoodie.precommit.validators. * address codope review: V2-then-V1 checkpoint key resolution + V2 test coverage Comment 1 (SparkKafkaOffsetValidator hardcoded V1 key): - StreamingOffsetValidator base class now exposes a no-key constructor that auto-resolves the checkpoint via CheckpointUtils.getCheckpoint(metadata), which prefers V2 and falls back to V1. The explicit-key constructor stays for subclasses that read a custom non-streamer key (e.g. Flink's HOODIE_METADATA_KEY). - SparkKafkaOffsetValidator switches to the no-key constructor. Comment 2 (tests only cover V1 path): - testSecondCommitMatchingOffsetsPasses and testSecondCommitDataLossDetected are now parameterized over both V1 and V2 checkpoint keys. - Added testV2CheckpointKeyOnTableVersionEightFires on a HoodieTableVersion.EIGHT table with V2 keys, asserting the validator fires on data loss. --------- Co-authored-by: Xinli Shang Co-authored-by: Claude Opus 4.6 (cherry picked from commit b934633f066a6efa57e04837b2536baa66b4f2f4) --- .../validator/StreamingOffsetValidator.java | 93 ++++- .../HoodiePreCommitValidatorConfig.java | 8 +- .../TestStreamingOffsetValidator.java | 2 +- .../client/utils/SparkValidatorUtils.java | 25 +- .../client/validator/ValidationContext.java | 14 + .../hudi/utilities/streamer/StreamSync.java | 37 +- .../validator/SparkKafkaOffsetValidator.java | 60 ++++ .../SparkStreamerValidatorUtils.java | 194 +++++++++++ .../validator/SparkValidationContext.java | 139 ++++++++ .../TestSparkKafkaOffsetValidator.java | 322 ++++++++++++++++++ .../TestSparkStreamerValidatorUtils.java | 290 ++++++++++++++++ .../validator/TestSparkValidationContext.java | 156 +++++++++ 12 files changed, 1317 insertions(+), 23 deletions(-) create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkKafkaOffsetValidator.java create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkValidationContext.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkKafkaOffsetValidator.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkStreamerValidatorUtils.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkValidationContext.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java index ce577d84ca018..40e7a4f1635f9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java @@ -20,11 +20,13 @@ package org.apache.hudi.client.validator; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.util.CheckpointUtils; import org.apache.hudi.common.util.CheckpointUtils.CheckpointFormat; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodiePreCommitValidatorConfig; import org.apache.hudi.config.HoodiePreCommitValidatorConfig.ValidationFailurePolicy; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieValidationException; import lombok.extern.slf4j.Slf4j; @@ -50,7 +52,11 @@ * * Subclasses specify: * - Checkpoint format (SPARK_KAFKA, FLINK_KAFKA, etc.) - * - Checkpoint metadata key + * - Checkpoint metadata key (optional — when omitted, the validator auto-resolves the + * active streamer key from commit metadata using + * {@link org.apache.hudi.common.table.checkpoint.CheckpointUtils#getCheckpoint(HoodieCommitMetadata)}, + * which prefers V2 and falls back to V1. Subclasses that read a custom non-streamer key + * (e.g. Flink's HOODIE_METADATA_KEY) must pass it explicitly.) * - Source-specific parsing logic (if needed) * * Configuration: @@ -66,7 +72,26 @@ public abstract class StreamingOffsetValidator extends BasePreCommitValidator { protected final CheckpointFormat checkpointFormat; /** - * Create a streaming offset validator. + * Create a streaming offset validator that auto-resolves the checkpoint key from commit + * metadata using {@link org.apache.hudi.common.table.checkpoint.CheckpointUtils#getCheckpoint(HoodieCommitMetadata)}. + * + *

Use this constructor for streamer pipelines (V1 or V2 checkpoint keys). The validator + * will prefer V2 (table version 8+) and fall back to V1 transparently, so subclasses don't + * need to know which key the writer used.

+ * + * @param config Validator configuration + * @param checkpointFormat Format of the checkpoint string + */ + protected StreamingOffsetValidator(TypedProperties config, + CheckpointFormat checkpointFormat) { + this(config, null, checkpointFormat); + } + + /** + * Create a streaming offset validator with an explicit checkpoint metadata key. + * + *

Use this constructor when the writer stores its checkpoint under a custom key that + * is not the standard streamer V1/V2 key (e.g. Flink's HOODIE_METADATA_KEY).

* * @param config Validator configuration * @param checkpointKey Key to extract checkpoint from extraMetadata @@ -95,10 +120,12 @@ public void validateWithMetadata(ValidationContext context) throws HoodieValidat return; } - // Extract current checkpoint - Option currentCheckpointOpt = context.getExtraMetadata(checkpointKey); + // Extract current checkpoint — either from the explicit key (custom writers like Flink) or + // by auto-resolving from commit metadata (streamer pipelines, V2-then-V1 fallback). + Option currentCheckpointOpt = resolveCheckpoint(context.getCommitMetadata()); if (!currentCheckpointOpt.isPresent()) { - log.warn("Current checkpoint not found with key: {}. Skipping validation.", checkpointKey); + log.warn("Current checkpoint not found (key: {}). Skipping validation.", + checkpointKey == null ? "" : checkpointKey); return; } String currentCheckpoint = currentCheckpointOpt.get(); @@ -110,8 +137,7 @@ public void validateWithMetadata(ValidationContext context) throws HoodieValidat } // Extract previous checkpoint - Option previousCheckpointOpt = context.getPreviousCommitMetadata() - .flatMap(metadata -> Option.ofNullable(metadata.getMetadata(checkpointKey))); + Option previousCheckpointOpt = resolveCheckpoint(context.getPreviousCommitMetadata()); if (!previousCheckpointOpt.isPresent()) { log.info("Previous checkpoint not found. May be first streaming commit. Skipping validation."); @@ -139,6 +165,10 @@ public void validateWithMetadata(ValidationContext context) throws HoodieValidat long recordsWritten = context.getTotalInsertRecordsWritten() + context.getTotalUpdateRecordsWritten(); + // Track write errors so callers can distinguish write-failure deviation (write errors > 0) + // from silent data loss (write errors == 0) when the validator fires. + long writeErrors = context.getTotalWriteErrors(); + // For empty commits (e.g., no new data from source), both offsetDiff and recordsWritten // can be zero. This is a valid scenario — skip validation to avoid false positives. if (offsetDifference == 0 && recordsWritten == 0) { @@ -147,7 +177,7 @@ public void validateWithMetadata(ValidationContext context) throws HoodieValidat } // Validate offset vs record consistency - validateOffsetConsistency(offsetDifference, recordsWritten, + validateOffsetConsistency(offsetDifference, recordsWritten, writeErrors, currentCheckpoint, previousCheckpoint); } @@ -155,12 +185,13 @@ public void validateWithMetadata(ValidationContext context) throws HoodieValidat * Validate that offset difference matches record count within tolerance. * * @param offsetDiff Expected records based on offset difference - * @param recordsWritten Actual records written + * @param recordsWritten Actual records written (inserts + updates) + * @param writeErrors Records that failed to write (tracked in write status errors) * @param currentCheckpoint Current checkpoint string (for error messages) * @param previousCheckpoint Previous checkpoint string (for error messages) * @throws HoodieValidationException if validation fails and policy is FAIL */ - protected void validateOffsetConsistency(long offsetDiff, long recordsWritten, + protected void validateOffsetConsistency(long offsetDiff, long recordsWritten, long writeErrors, String currentCheckpoint, String previousCheckpoint) throws HoodieValidationException { @@ -169,10 +200,13 @@ protected void validateOffsetConsistency(long offsetDiff, long recordsWritten, if (deviation > tolerancePercentage) { String errorMsg = String.format( "Streaming offset validation failed. " - + "Offset difference: %d, Records written: %d, Deviation: %.2f%%, Tolerance: %.2f%%. " - + "This may indicate data loss or filtering. " + + "Offset difference: %d, Records written: %d, Write errors: %d, Deviation: %.2f%%, Tolerance: %.2f%%. " + + "%s" + "Previous checkpoint: %s, Current checkpoint: %s", - offsetDiff, recordsWritten, deviation, tolerancePercentage, + offsetDiff, recordsWritten, writeErrors, deviation, tolerancePercentage, + writeErrors > 0 + ? "Non-zero write errors suggest records failed to write rather than silent data loss. " + : "This may indicate data loss or filtering. ", previousCheckpoint, currentCheckpoint); if (failurePolicy == ValidationFailurePolicy.WARN_LOG) { @@ -181,8 +215,8 @@ protected void validateOffsetConsistency(long offsetDiff, long recordsWritten, throw new HoodieValidationException(errorMsg); } } else { - log.info("Offset validation passed. Offset diff: {}, Records: {}, Deviation: {}% (within {}%)", - offsetDiff, recordsWritten, String.format("%.2f", deviation), tolerancePercentage); + log.info("Offset validation passed. Offset diff: {}, Records: {}, Write errors: {}, Deviation: {}% (within {}%)", + offsetDiff, recordsWritten, writeErrors, String.format("%.2f", deviation), tolerancePercentage); } } @@ -210,4 +244,33 @@ private double calculateDeviation(long offsetDiff, long recordsWritten) { long difference = Math.abs(offsetDiff - recordsWritten); return (100.0 * difference) / offsetDiff; } + + /** + * Resolve the checkpoint string from commit metadata. + * + *

When the validator was constructed with an explicit {@code checkpointKey}, that key + * is read directly. Otherwise, {@link org.apache.hudi.common.table.checkpoint.CheckpointUtils#getCheckpoint(HoodieCommitMetadata)} + * is used to locate the active streamer checkpoint (V2 first, V1 fallback), so callers + * don't need to know which key the writer used.

+ * + * @param commitMetadataOpt Optional commit metadata containing extraMetadata + * @return Optional checkpoint string (empty if metadata is absent or no checkpoint key matches) + */ + private Option resolveCheckpoint(Option commitMetadataOpt) { + if (!commitMetadataOpt.isPresent()) { + return Option.empty(); + } + HoodieCommitMetadata metadata = commitMetadataOpt.get(); + if (checkpointKey != null) { + return Option.ofNullable(metadata.getMetadata(checkpointKey)); + } + try { + return Option.ofNullable( + org.apache.hudi.common.table.checkpoint.CheckpointUtils.getCheckpoint(metadata) + .getCheckpointKey()); + } catch (HoodieException e) { + // No V1 or V2 streamer checkpoint key present in extraMetadata. + return Option.empty(); + } + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java index f85cc44120d4e..169494b7244ac 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java @@ -43,7 +43,10 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig { .key("hoodie.precommit.validators") .defaultValue("") .markAdvanced() - .withDocumentation("Comma separated list of class names that can be invoked to validate commit"); + .withDocumentation("Comma separated list of class names that can be invoked to validate commit. " + + "Available streaming offset validators: " + + "org.apache.hudi.sink.validator.FlinkKafkaOffsetValidator (Flink Kafka), " + + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator (Spark/HoodieStreamer Kafka)"); public static final String VALIDATOR_TABLE_VARIABLE = ""; public static final ConfigProperty EQUALITY_SQL_QUERIES = ConfigProperty @@ -71,7 +74,8 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig { .markAdvanced() .withDocumentation("Tolerance percentage for streaming offset validation " + "(used by org.apache.hudi.client.validator.StreamingOffsetValidator " - + "and org.apache.hudi.sink.validator.FlinkKafkaOffsetValidator). " + + "and org.apache.hudi.sink.validator.FlinkKafkaOffsetValidator " + + "and org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator). " + "The validator compares the offset difference (expected records from source) " + "with actual records written. If the deviation exceeds this percentage, " + "the commit is rejected or warned depending on the validation failure policy. " diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/validator/TestStreamingOffsetValidator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/validator/TestStreamingOffsetValidator.java index 818bc2087a3c1..bc402e5ee5a0e 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/validator/TestStreamingOffsetValidator.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/validator/TestStreamingOffsetValidator.java @@ -62,7 +62,7 @@ public MockOffsetValidator(TypedProperties config) { // Expose protected method for testing public void testValidateOffsetConsistency(long offsetDiff, long recordsWritten, String current, String previous) { - validateOffsetConsistency(offsetDiff, recordsWritten, current, previous); + validateOffsetConsistency(offsetDiff, recordsWritten, 0L, current, previous); } // Expose validateWithMetadata for testing diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java index c7804631d3f65..40ee7d8cefff9 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java @@ -84,9 +84,28 @@ public static void runValidators(HoodieWriteConfig config, Dataset beforeState = getRecordsFromCommittedFiles(sqlContext, partitionsModified, table, afterState.schema()); Stream validators = Arrays.stream(config.getPreCommitValidators().split(",")) - .map(validatorClass -> ((SparkPreCommitValidator) ReflectionUtils.loadClass(validatorClass, - new Class[] {HoodieSparkTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, - table, context, config))); + .map(String::trim) + .filter(s -> !s.isEmpty()) + .flatMap(validatorClass -> { + try { + Class clazz = Class.forName(validatorClass); + if (!SparkPreCommitValidator.class.isAssignableFrom(clazz)) { + LOG.warn("Skipping validator {} — it does not implement SparkPreCommitValidator. " + + "If this is a streaming offset validator (e.g. SparkKafkaOffsetValidator), " + + "it will be invoked by SparkStreamerValidatorUtils instead.", validatorClass); + return Stream.empty(); + } + SparkPreCommitValidator validator = (SparkPreCommitValidator) ReflectionUtils.loadClass( + validatorClass, + new Class[] {HoodieSparkTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, + table, context, config); + return Stream.of(validator); + } catch (ClassNotFoundException e) { + throw new HoodieValidationException("Cannot find validator class: " + validatorClass, e); + } catch (ReflectiveOperationException e) { + throw new HoodieValidationException("Failed to instantiate validator: " + validatorClass, e); + } + }); boolean allSuccess = validators.map(v -> runValidatorAsync(v, writeMetadata, beforeState, afterState, instantTime)).map(CompletableFuture::join) .reduce(true, Boolean::logicalAnd); diff --git a/hudi-common/src/main/java/org/apache/hudi/client/validator/ValidationContext.java b/hudi-common/src/main/java/org/apache/hudi/client/validator/ValidationContext.java index 30fdbb3ba3b4a..b85218e587e87 100644 --- a/hudi-common/src/main/java/org/apache/hudi/client/validator/ValidationContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/client/validator/ValidationContext.java @@ -169,6 +169,20 @@ default long getTotalUpdateRecordsWritten() { .orElse(0L); } + /** + * Calculate total write errors in the current commit. + * Records that failed to write are tracked in {@link org.apache.hudi.common.model.HoodieWriteStat#getTotalWriteErrors()}. + * A non-zero error count alongside a deviation in offset validation indicates write failures + * rather than silent data loss — useful context for distinguishing the two failure modes. + * + * @return Total count of records that failed to write + */ + default long getTotalWriteErrors() { + return getWriteStats() + .map(stats -> stats.stream().mapToLong(HoodieWriteStat::getTotalWriteErrors).sum()) + .orElse(0L); + } + /** * Check if this is the first commit (no previous commits exist). * Derived from {@link #getPreviousCommitInstant()}. diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 600891c85dff6..2b1406c7deab5 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -74,6 +74,7 @@ import org.apache.hudi.config.HoodieErrorTableConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodiePayloadConfig; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.data.HoodieJavaRDD; @@ -115,6 +116,7 @@ import org.apache.hudi.utilities.sources.InputBatch; import org.apache.hudi.utilities.sources.Source; import org.apache.hudi.utilities.streamer.HoodieStreamer.Config; +import org.apache.hudi.utilities.streamer.validator.SparkStreamerValidatorUtils; import org.apache.hudi.utilities.transform.Transformer; import com.codahale.metrics.Timer; @@ -128,6 +130,7 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; +import org.apache.spark.storage.StorageLevel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -874,8 +877,38 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood totalSuccessfulRecords); String commitActionType = CommitUtils.getCommitActionType(cfg.operation, HoodieTableType.valueOf(cfg.tableType)); - boolean success = writeClient.commit(instantTime, writeStatusRDD, Option.of(checkpointCommitMetadata), commitActionType, partitionToReplacedFileIds, Option.empty(), - Option.of(writeStatusValidator)); + // Cache the RDD only when pre-commit validators are configured. Validators collect the RDD + // before commit, so without caching the same DAG would re-evaluate inside writeClient.commit(). + // When no validators are configured, commit consumes the RDD once and caching adds no value. + // shouldUnpersist is true only when we created the cache here (validators present and storage + // level was NONE), so the finally block knows to release it. + boolean validatorsConfigured = !StringUtils.isNullOrEmpty(props.getString( + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.defaultValue())); + boolean shouldUnpersist = validatorsConfigured && writeStatusRDD.getStorageLevel().equals(StorageLevel.NONE()); + if (shouldUnpersist) { + writeStatusRDD.cache(); + } + boolean success; + try { + if (validatorsConfigured) { + List writeStatuses = writeStatusRDD.collect(); + + // Run pre-commit streaming offset validators (if configured). + // Placement before writeClient.commit() is intentional: offset validation is a stronger + // guard than commitOnErrors — if offset deviation indicates potential data loss, the commit + // must be prevented regardless of the commitOnErrors policy. + SparkStreamerValidatorUtils.runValidators(props, instantTime, writeStatuses, + checkpointCommitMetadata, metaClient); + } + + success = writeClient.commit(instantTime, writeStatusRDD, Option.of(checkpointCommitMetadata), commitActionType, partitionToReplacedFileIds, Option.empty(), + Option.of(writeStatusValidator)); + } finally { + if (shouldUnpersist) { + writeStatusRDD.unpersist(); + } + } releaseResourcesInvoked = true; if (success) { LOG.info("Commit " + instantTime + " successful!"); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkKafkaOffsetValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkKafkaOffsetValidator.java new file mode 100644 index 0000000000000..589e09e96de89 --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkKafkaOffsetValidator.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.client.validator.StreamingOffsetValidator; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.util.CheckpointUtils.CheckpointFormat; + +/** + * Spark/HoodieStreamer-specific Kafka offset validator. + * + *

Validates that the number of records written matches the Kafka offset difference + * between the current and previous HoodieStreamer checkpoints. The active checkpoint key + * (V1 {@code deltastreamer.checkpoint.key} or V2 {@code streamer.checkpoint.key.v2}) is + * resolved at validation time via {@link org.apache.hudi.common.table.checkpoint.CheckpointUtils#getCheckpoint}, + * so this validator works against tables written with either checkpoint key version.

+ * + *

Configuration: + *

    + *
  • {@code hoodie.precommit.validators}: Include + * {@code org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator}
    • + *
  • {@code hoodie.precommit.validators.streaming.offset.tolerance.percentage}: + * Acceptable deviation (default: 0.0 = strict)
    • + *
  • {@code hoodie.precommit.validators.failure.policy}: + * FAIL (default) or WARN_LOG
    • + *

+ * + *

This validator is primarily intended for append-only ingestion from Kafka via HoodieStreamer. + * For upsert workloads with deduplication, configure a higher tolerance or use WARN_LOG.

+ * + *

Important: This class extends {@link org.apache.hudi.client.validator.BasePreCommitValidator} + * and is invoked by {@link SparkStreamerValidatorUtils}, NOT by {@code SparkValidatorUtils} + * (which expects {@code SparkPreCommitValidator} with a different constructor signature). + * Listing this class in {@code hoodie.precommit.validators} while also using the standard + * Spark table write-path validators will cause an instantiation failure in {@code SparkValidatorUtils}. + * Use this validator exclusively with HoodieStreamer pipelines.

+ */ +public class SparkKafkaOffsetValidator extends StreamingOffsetValidator { + + public SparkKafkaOffsetValidator(TypedProperties config) { + super(config, CheckpointFormat.SPARK_KAFKA); + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java new file mode 100644 index 0000000000000..474215ec0f3c1 --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.validator.BasePreCommitValidator; +import org.apache.hudi.client.validator.ValidationContext; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieValidationException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Utility for running pre-commit validators in the HoodieStreamer commit flow. + * + *

Instantiates and executes validators configured via + * {@code hoodie.precommit.validators}. Each validator must extend + * {@link BasePreCommitValidator} and have a constructor that accepts + * {@link TypedProperties}.

+ * + *

Called from {@code StreamSync.writeToSinkAndDoMetaSync()} before + * the commit is finalized.

+ * + *

Note on validator compatibility: This utility uses a different instantiation + * mechanism than {@code SparkValidatorUtils} (used by the Spark table write path). + * {@code SparkValidatorUtils} expects validators implementing {@code SparkPreCommitValidator} + * with a {@code (HoodieSparkTable, HoodieEngineContext, HoodieWriteConfig)} constructor. + * Validators registered here (e.g. {@link SparkKafkaOffsetValidator}) extend + * {@link BasePreCommitValidator} with a {@code (TypedProperties)} constructor and + * are NOT compatible with {@code SparkValidatorUtils}. Do not mix them under the same + * {@code hoodie.precommit.validators} config if both paths are active.

+ */ +public class SparkStreamerValidatorUtils { + + private static final Logger LOG = LoggerFactory.getLogger(SparkStreamerValidatorUtils.class); + + /** + * Run all configured pre-commit validators. + * + *

The caller is responsible for caching and unpersisting the source RDD if needed. + * This method accepts pre-collected write statuses to avoid a second DAG evaluation — + * the caller should cache the RDD, collect to this list, call this method, then pass + * the same RDD to {@code writeClient.commit()}, and unpersist after commit completes.

+ * + * @param props Configuration properties containing validator class names + * @param instantTime Commit instant time + * @param writeStatuses Pre-collected write statuses from Spark write operations + * @param checkpointCommitMetadata Extra metadata being committed (contains checkpoint info) + * @param metaClient Table meta client for timeline access and previous commit lookup + * @throws HoodieValidationException if any validator fails with FAIL policy + */ + public static void runValidators(TypedProperties props, + String instantTime, + List writeStatuses, + Map checkpointCommitMetadata, + HoodieTableMetaClient metaClient) { + String validatorClassNames = props.getString( + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.defaultValue()); + + if (StringUtils.isNullOrEmpty(validatorClassNames)) { + return; + } + + HoodieCommitMetadata currentMetadata = buildCommitMetadata(writeStatuses, checkpointCommitMetadata); + List writeStats = writeStatuses.stream() + .map(WriteStatus::getStat) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + Option previousCommitMetadata = loadPreviousCommitMetadata(metaClient); + + ValidationContext context = new SparkValidationContext( + instantTime, + Option.of(currentMetadata), + Option.of(writeStats), + previousCommitMetadata, + metaClient); + + List classNames = Arrays.stream(validatorClassNames.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + + for (String className : classNames) { + try { + Class clazz = Class.forName(className); + if (!BasePreCommitValidator.class.isAssignableFrom(clazz)) { + LOG.warn("Skipping validator {} in HoodieStreamer path — it does not extend BasePreCommitValidator. " + + "If this is a SparkPreCommitValidator (e.g. SqlQueryEqualityPreCommitValidator), " + + "it must be invoked via SparkValidatorUtils in the standard Spark write path instead.", className); + continue; + } + BasePreCommitValidator validator = (BasePreCommitValidator) + ReflectionUtils.loadClass(className, new Class[] {TypedProperties.class}, props); + LOG.info("Running pre-commit validator: {} for instant: {}", className, instantTime); + validator.validateWithMetadata(context); + LOG.info("Pre-commit validator {} passed for instant: {}", className, instantTime); + } catch (HoodieValidationException e) { + LOG.error("Pre-commit validator {} failed for instant: {}", className, instantTime, e); + throw e; + } catch (Exception e) { + LOG.error("Failed to instantiate or run validator: {}", className, e); + throw new HoodieValidationException( + "Failed to run pre-commit validator: " + className, e); + } + } + } + + /** + * Build a pre-commit snapshot of {@link HoodieCommitMetadata} from write statuses and extra metadata. + * + *

This is intentionally a partial/preview object used only for validation — it contains + * write stats and checkpoint extra-metadata, but omits fields that are not available before the + * commit (e.g. schema, operation type). Validators should treat this as a read-only snapshot + * of what will be committed, not a fully-constructed commit record.

+ */ + private static HoodieCommitMetadata buildCommitMetadata( + List writeStatuses, Map extraMetadata) { + HoodieCommitMetadata metadata = new HoodieCommitMetadata(); + + // Add write stats + for (WriteStatus status : writeStatuses) { + HoodieWriteStat stat = status.getStat(); + if (stat != null) { + metadata.addWriteStat(stat.getPartitionPath(), stat); + } + } + + // Add extra metadata (includes checkpoint info like deltastreamer.checkpoint.key) + if (extraMetadata != null) { + extraMetadata.forEach(metadata::addMetadata); + } + + return metadata; + } + + /** + * Load the previous completed commit metadata from the timeline. + */ + private static Option loadPreviousCommitMetadata(HoodieTableMetaClient metaClient) { + try { + HoodieTimeline completedTimeline = metaClient.reloadActiveTimeline() + .getWriteTimeline() + .filterCompletedInstants(); + Option lastInstant = completedTimeline.lastInstant(); + if (lastInstant.isPresent()) { + return Option.of(completedTimeline.readCommitMetadata(lastInstant.get())); + } + } catch (IOException e) { + throw new HoodieIOException("Failed to load previous commit metadata", e); + } + return Option.empty(); + } + + private SparkStreamerValidatorUtils() { + // Utility class + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkValidationContext.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkValidationContext.java new file mode 100644 index 0000000000000..1d46e13aeedbd --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkValidationContext.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.client.validator.ValidationContext; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; + +import java.util.List; + +/** + * Spark/HoodieStreamer implementation of {@link ValidationContext}. + * + *

Constructed from data available in {@code StreamSync.writeToSinkAndDoMetaSync()} + * before the commit is finalized. Provides validators with access to commit metadata, + * write statistics, and previous commit information for streaming offset validation.

+ * + *

Unlike Flink's implementation, Spark can optionally provide active timeline access + * via {@link HoodieTableMetaClient} for richer validation patterns.

+ */ +public class SparkValidationContext implements ValidationContext { + + private final String instantTime; + private final Option commitMetadata; + private final Option> writeStats; + private final Option previousCommitMetadata; + private final HoodieTableMetaClient metaClient; + + /** + * Create a Spark validation context with full timeline access. + * + * @param instantTime Current commit instant time + * @param commitMetadata Current commit metadata (with extraMetadata including checkpoints) + * @param writeStats Write statistics from write operations + * @param previousCommitMetadata Metadata from the previous completed commit + * @param metaClient Table meta client for timeline access (may be null for testing) + */ + public SparkValidationContext(String instantTime, + Option commitMetadata, + Option> writeStats, + Option previousCommitMetadata, + HoodieTableMetaClient metaClient) { + this.instantTime = instantTime; + this.commitMetadata = commitMetadata; + this.writeStats = writeStats; + this.previousCommitMetadata = previousCommitMetadata; + this.metaClient = metaClient; + } + + /** + * Create a Spark validation context without timeline access (for testing). + * + * @param instantTime Current commit instant time + * @param commitMetadata Current commit metadata (with extraMetadata including checkpoints) + * @param writeStats Write statistics from write operations + * @param previousCommitMetadata Metadata from the previous completed commit + */ + public SparkValidationContext(String instantTime, + Option commitMetadata, + Option> writeStats, + Option previousCommitMetadata) { + this(instantTime, commitMetadata, writeStats, previousCommitMetadata, null); + } + + @Override + public String getInstantTime() { + return instantTime; + } + + @Override + public Option getCommitMetadata() { + return commitMetadata; + } + + @Override + public Option> getWriteStats() { + return writeStats; + } + + /** + * Get the active timeline. Available when metaClient is provided. + * + * @throws UnsupportedOperationException if metaClient was not provided + */ + @Override + public HoodieActiveTimeline getActiveTimeline() { + if (metaClient == null) { + throw new UnsupportedOperationException( + "Active timeline is not available without HoodieTableMetaClient."); + } + return metaClient.getActiveTimeline(); + } + + /** + * Get the previous completed commit instant by querying the timeline. + * Returns {@link Option#empty()} if this is the first commit or metaClient is unavailable. + */ + @Override + public Option getPreviousCommitInstant() { + if (metaClient == null) { + return Option.empty(); + } + return metaClient.getActiveTimeline() + .getWriteTimeline() + .filterCompletedInstants() + .lastInstant(); + } + + @Override + public boolean isFirstCommit() { + return !previousCommitMetadata.isPresent(); + } + + @Override + public Option getPreviousCommitMetadata() { + return previousCommitMetadata; + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkKafkaOffsetValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkKafkaOffsetValidator.java new file mode 100644 index 0000000000000..d109aa3246f64 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkKafkaOffsetValidator.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; +import org.apache.hudi.exception.HoodieValidationException; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link SparkKafkaOffsetValidator}. + */ +public class TestSparkKafkaOffsetValidator { + + // ========== Helper methods ========== + + private static TypedProperties defaultConfig() { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.STREAMING_OFFSET_TOLERANCE_PERCENTAGE.key(), "0.0"); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "FAIL"); + return props; + } + + private static TypedProperties configWithTolerance(double tolerance) { + TypedProperties props = defaultConfig(); + props.setProperty(HoodiePreCommitValidatorConfig.STREAMING_OFFSET_TOLERANCE_PERCENTAGE.key(), + String.valueOf(tolerance)); + return props; + } + + private static TypedProperties configWithWarnPolicy() { + TypedProperties props = defaultConfig(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "WARN_LOG"); + return props; + } + + /** + * Build a Spark Kafka checkpoint string. + * Format: topic,partition:offset,partition:offset,... + */ + private static String buildSparkKafkaCheckpoint(String topic, int[] partitions, long[] offsets) { + StringBuilder sb = new StringBuilder(); + sb.append(topic); + for (int i = 0; i < partitions.length; i++) { + sb.append(",").append(partitions[i]).append(":").append(offsets[i]); + } + return sb.toString(); + } + + private static HoodieCommitMetadata buildMetadata(String checkpointValue) { + HoodieCommitMetadata metadata = new HoodieCommitMetadata(); + if (checkpointValue != null) { + metadata.addMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, checkpointValue); + } + return metadata; + } + + private static List buildWriteStats(long numInserts, long numUpdates) { + HoodieWriteStat stat = new HoodieWriteStat(); + stat.setNumInserts(numInserts); + stat.setNumUpdateWrites(numUpdates); + stat.setPartitionPath("partition1"); + return Collections.singletonList(stat); + } + + private static SparkValidationContext buildContext( + String instantTime, + HoodieCommitMetadata currentMetadata, + List writeStats, + HoodieCommitMetadata previousMetadata) { + return new SparkValidationContext( + instantTime, + Option.of(currentMetadata), + Option.of(writeStats), + previousMetadata != null ? Option.of(previousMetadata) : Option.empty()); + } + + // ========== Tests ========== + + @Test + public void testExactMatchPasses() { + // Previous: partition 0 at offset 100, partition 1 at offset 200 + // Current: partition 0 at offset 200, partition 1 at offset 300 + // Diff = (200-100) + (300-200) = 200. Records written = 200. + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0, 1}, new long[]{100, 200}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0, 1}, new long[]{200, 300}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(200, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testDataLossDetected() { + // Diff = 1000 but only 500 records written -> 50% deviation + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(500, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertThrows(HoodieValidationException.class, () -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testWithinTolerancePasses() { + // Diff = 1000, records = 950 -> 5% deviation, tolerance = 10% + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(950, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(configWithTolerance(10.0)); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testWarnPolicyDoesNotThrow() { + // Data loss but WARN_LOG policy + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(0, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(configWithWarnPolicy()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testSkipsFirstCommit() { + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + // No previous commit + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(buildMetadata(currCheckpoint)), + Option.of(buildWriteStats(500, 0)), + Option.empty()); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testSkipsWhenNoCheckpointKey() { + // Current metadata has no checkpoint key + HoodieCommitMetadata currentMeta = new HoodieCommitMetadata(); + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{100}); + + SparkValidationContext ctx = buildContext("20260320120000000", + currentMeta, + buildWriteStats(500, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testMultiPartitionValidation() { + // 4 partitions, each advancing by 250 = total diff 1000 + String prevCheckpoint = buildSparkKafkaCheckpoint("events", + new int[]{0, 1, 2, 3}, new long[]{0, 0, 0, 0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", + new int[]{0, 1, 2, 3}, new long[]{250, 250, 250, 250}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(800, 200), // 800 inserts + 200 updates = 1000 + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testEmptyCommitSkipsValidation() { + // Both offsets same and no records written + String checkpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{100}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(checkpoint), + buildWriteStats(0, 0), + buildMetadata(checkpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testPreviousCheckpointMissingSkipsValidation() { + // Previous metadata exists but has no checkpoint key + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(500, 0), + prevMeta); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testOvercountingDetected() { + // More records written than offset diff + // Diff = 100, records = 200 -> |100-200|/100 = 100% deviation + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{100}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(200, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertThrows(HoodieValidationException.class, () -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testExactToleranceBoundaryPasses() { + // Diff = 1000, records = 900 -> 10% deviation, tolerance = 10% + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(900, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(configWithTolerance(10.0)); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testJustOverToleranceFails() { + // Diff = 1000, records = 899 -> 10.1% deviation, tolerance = 10% + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(899, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(configWithTolerance(10.0)); + assertThrows(HoodieValidationException.class, () -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testOnlyInsertsNoUpdates() { + // Pure insert workload + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0, 1}, new long[]{0, 0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0, 1}, new long[]{500, 500}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(1000, 0), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testUpdatesCountedInRecordTotal() { + // Diff = 1000. 600 inserts + 400 updates = 1000 total + String prevCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{0}); + String currCheckpoint = buildSparkKafkaCheckpoint("events", new int[]{0}, new long[]{1000}); + + SparkValidationContext ctx = buildContext("20260320120000000", + buildMetadata(currCheckpoint), + buildWriteStats(600, 400), + buildMetadata(prevCheckpoint)); + + SparkKafkaOffsetValidator validator = new SparkKafkaOffsetValidator(defaultConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkStreamerValidatorUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkStreamerValidatorUtils.java new file mode 100644 index 0000000000000..69d5d228dab60 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkStreamerValidatorUtils.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.testutils.HoodieTestTable; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; +import org.apache.hudi.exception.HoodieValidationException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link SparkStreamerValidatorUtils}. + * + *

Tests cover orchestration logic (class loading, config passing, error handling) + * as well as end-to-end offset validation using a two-commit timeline to verify + * the real comparison path is exercised.

+ */ +public class TestSparkStreamerValidatorUtils { + + @TempDir + Path tempDir; + + private static TypedProperties propsWithValidator(String validatorClassName) { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), validatorClassName); + props.setProperty(HoodiePreCommitValidatorConfig.STREAMING_OFFSET_TOLERANCE_PERCENTAGE.key(), "0.0"); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "FAIL"); + return props; + } + + private static WriteStatus buildWriteStatus(String partitionPath, long numInserts, long numUpdates) { + HoodieWriteStat stat = new HoodieWriteStat(); + stat.setPartitionPath(partitionPath); + stat.setNumInserts(numInserts); + stat.setNumUpdateWrites(numUpdates); + + WriteStatus ws = new WriteStatus(false, 0.0); + ws.setStat(stat); + return ws; + } + + private HoodieTableMetaClient createMetaClient() throws IOException { + return HoodieTestUtils.init(tempDir.toAbsolutePath().toString()); + } + + private HoodieTableMetaClient createMetaClient(HoodieTableVersion version) throws IOException { + return HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), HoodieTableType.COPY_ON_WRITE, version); + } + + // ========== Tests ========== + + @Test + public void testNoValidatorsConfigured() throws IOException { + TypedProperties props = new TypedProperties(); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, + new HashMap<>(), createMetaClient())); + } + + @Test + public void testEmptyValidatorString() throws IOException { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), ""); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, + new HashMap<>(), createMetaClient())); + } + + @Test + public void testValidValidatorFirstCommitPasses() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + Map extraMeta = new HashMap<>(); + extraMeta.put(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:100"); + + // First commit (no previous metadata on timeline) — validator should skip and pass + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, createMetaClient())); + } + + @Test + public void testInvalidValidatorClassThrows() throws IOException { + TypedProperties props = propsWithValidator("com.nonexistent.FakeValidator"); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertThrows(HoodieValidationException.class, + () -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, new HashMap<>(), createMetaClient())); + } + + @Test + public void testMultipleValidators() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator," + + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + Map extraMeta = new HashMap<>(); + extraMeta.put(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:100"); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, createMetaClient())); + } + + @Test + public void testValidatorWithWhitespaceInClassNames() throws IOException { + TypedProperties props = propsWithValidator( + " org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator , "); + + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, new HashMap<>(), createMetaClient())); + } + + @Test + public void testNullExtraMetadataHandled() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, null, createMetaClient())); + } + + @Test + public void testMultipleWriteStatusesAggregated() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + List writeStatuses = new ArrayList<>(); + writeStatuses.add(buildWriteStatus("p1", 60, 0)); + writeStatuses.add(buildWriteStatus("p2", 40, 0)); + + Map extraMeta = new HashMap<>(); + extraMeta.put(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:100"); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, createMetaClient())); + } + + @Test + public void testEmptyWriteStatuses() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + List writeStatuses = Collections.emptyList(); + Map extraMeta = new HashMap<>(); + extraMeta.put(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:100"); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, createMetaClient())); + } + + @Test + public void testValidationExceptionPreservedAcrossValidators() throws IOException { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator," + + "com.nonexistent.FakeValidator"); + + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, new HashMap<>(), createMetaClient())); + assertTrue(ex.getMessage().contains("FakeValidator")); + } + + @ParameterizedTest + @ValueSource(strings = { + StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, + StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2 + }) + public void testSecondCommitMatchingOffsetsPasses(String checkpointKey) throws Exception { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + // Create table with a previous committed instant: offset 0 -> 500 + HoodieTableMetaClient metaClient = createMetaClient(); + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + prevMeta.addMetadata(checkpointKey, "events,0:500"); + HoodieTestTable.of(metaClient).addCommit("20260320110000000", Option.of(prevMeta)); + + // Second commit: offset 500 -> 600, 100 records written — matches diff exactly + Map extraMeta = new HashMap<>(); + extraMeta.put(checkpointKey, "events,0:600"); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 100, 0)); + + assertDoesNotThrow(() -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, metaClient)); + } + + @ParameterizedTest + @ValueSource(strings = { + StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, + StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2 + }) + public void testSecondCommitDataLossDetected(String checkpointKey) throws Exception { + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + // Create table with a previous committed instant: offset 0 -> 1000 + HoodieTableMetaClient metaClient = createMetaClient(); + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + prevMeta.addMetadata(checkpointKey, "events,0:1000"); + HoodieTestTable.of(metaClient).addCommit("20260320110000000", Option.of(prevMeta)); + + // Second commit: offset 1000 -> 2000 (diff=1000) but only 500 records written — data loss + Map extraMeta = new HashMap<>(); + extraMeta.put(checkpointKey, "events,0:2000"); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 500, 0)); + + assertThrows(HoodieValidationException.class, + () -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, metaClient)); + } + + @Test + public void testV2CheckpointKeyOnTableVersionEightFires() throws Exception { + // Verifies the validator actually fires on a writeTableVersion=8 table that uses the + // V2 checkpoint key — i.e. the auto-resolution in StreamingOffsetValidator picks up V2 + // and runs the comparison instead of silently skipping. + TypedProperties props = propsWithValidator( + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator"); + + HoodieTableMetaClient metaClient = createMetaClient(HoodieTableVersion.EIGHT); + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + prevMeta.addMetadata(StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2, "events,0:1000"); + HoodieTestTable.of(metaClient).addCommit("20260320110000000", Option.of(prevMeta)); + + // Offset diff = 1000 but only 200 records written — must fail + Map extraMeta = new HashMap<>(); + extraMeta.put(StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2, "events,0:2000"); + List writeStatuses = Collections.singletonList(buildWriteStatus("p1", 200, 0)); + + assertThrows(HoodieValidationException.class, + () -> SparkStreamerValidatorUtils.runValidators( + props, "20260320120000000", writeStatuses, extraMeta, metaClient)); + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkValidationContext.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkValidationContext.java new file mode 100644 index 0000000000000..7f94262e98c43 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkValidationContext.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.util.Option; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link SparkValidationContext}. + */ +public class TestSparkValidationContext { + + private static HoodieWriteStat buildStat(long inserts, long updates) { + HoodieWriteStat stat = new HoodieWriteStat(); + stat.setNumInserts(inserts); + stat.setNumUpdateWrites(updates); + stat.setPartitionPath("partition1"); + return stat; + } + + @Test + public void testBasicProperties() { + HoodieCommitMetadata metadata = new HoodieCommitMetadata(); + metadata.addMetadata("key1", "value1"); + List writeStats = Collections.singletonList(buildStat(100, 50)); + + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(metadata), + Option.of(writeStats), + Option.empty()); + + assertEquals("20260320120000000", ctx.getInstantTime()); + assertTrue(ctx.getCommitMetadata().isPresent()); + assertTrue(ctx.getWriteStats().isPresent()); + assertEquals(1, ctx.getWriteStats().get().size()); + } + + @Test + public void testRecordCounting() { + List writeStats = Arrays.asList( + buildStat(100, 50), // partition1: 100 inserts, 50 updates + buildStat(200, 30)); // partition2: 200 inserts, 30 updates + + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(new HoodieCommitMetadata()), + Option.of(writeStats), + Option.empty()); + + assertEquals(300, ctx.getTotalInsertRecordsWritten()); + assertEquals(80, ctx.getTotalUpdateRecordsWritten()); + assertEquals(380, ctx.getTotalRecordsWritten()); + } + + @Test + public void testFirstCommitDetection() { + // No previous commit metadata -> first commit + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(new HoodieCommitMetadata()), + Option.of(Collections.emptyList()), + Option.empty()); + + assertTrue(ctx.isFirstCommit()); + } + + @Test + public void testNotFirstCommitWhenPreviousExists() { + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(new HoodieCommitMetadata()), + Option.of(Collections.emptyList()), + Option.of(prevMeta)); + + assertFalse(ctx.isFirstCommit()); + } + + @Test + public void testExtraMetadataAccess() { + HoodieCommitMetadata metadata = new HoodieCommitMetadata(); + metadata.addMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:1000"); + metadata.addMetadata("custom.key", "custom_value"); + + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(metadata), + Option.of(Collections.emptyList()), + Option.empty()); + + assertEquals("events,0:1000", + ctx.getExtraMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1).get()); + assertEquals("custom_value", ctx.getExtraMetadata("custom.key").get()); + assertFalse(ctx.getExtraMetadata("nonexistent.key").isPresent()); + } + + @Test + public void testPreviousCommitMetadataAccess() { + HoodieCommitMetadata prevMeta = new HoodieCommitMetadata(); + prevMeta.addMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1, "events,0:500"); + + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(new HoodieCommitMetadata()), + Option.of(Collections.emptyList()), + Option.of(prevMeta)); + + assertTrue(ctx.getPreviousCommitMetadata().isPresent()); + assertEquals("events,0:500", + ctx.getPreviousCommitMetadata().get().getMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1)); + } + + @Test + public void testEmptyWriteStats() { + SparkValidationContext ctx = new SparkValidationContext( + "20260320120000000", + Option.of(new HoodieCommitMetadata()), + Option.empty(), + Option.empty()); + + assertEquals(0, ctx.getTotalRecordsWritten()); + assertEquals(0, ctx.getTotalInsertRecordsWritten()); + assertEquals(0, ctx.getTotalUpdateRecordsWritten()); + } +} From 9023d94478d5c68cf8269e08fc07bc3f9a22b425 Mon Sep 17 00:00:00 2001 From: Xinli Shang Date: Sun, 17 May 2026 20:15:40 -0700 Subject: [PATCH 014/255] [MINOR] Fix typos in comments and assertion messages (#18763) Corrects a handful of long-standing typos in code comments and test assertion messages across the codebase. No functional changes. - atleast -> at least (8 occurrences) - commited -> committed (1 occurrence in comment) - existance -> existence (1 occurrence in exception message) - transfering -> transferring (2 occurrences) - succesfully -> successfully (1 occurrence in comment) Co-authored-by: Xinli Shang (cherry picked from commit cd2c8b831c8b7c8f46b1f175bdb323329f60c8ab) --- .../org/apache/hudi/client/utils/LazyIterableIterator.java | 2 +- .../apache/hudi/metadata/HoodieBackedTableMetadataWriter.java | 2 +- .../org/apache/hudi/client/TestCoalescingPartitioner.java | 2 +- .../test/java/org/apache/hudi/table/TestHoodieSparkTable.java | 2 +- .../apache/hudi/table/action/compact/CompactionTestBase.java | 4 ++-- .../generator/GenericRecordPartialPayloadGenerator.java | 2 +- .../apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java | 2 +- .../parquet/Spark41LegacyHoodieParquetFileFormat.scala | 2 +- .../execution/datasources/parquet/Spark41ParquetReader.scala | 2 +- .../deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java | 2 +- scripts/pr_compliance.py | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LazyIterableIterator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LazyIterableIterator.java index b921c6ddfc813..64a92ee1ae8ca 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LazyIterableIterator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LazyIterableIterator.java @@ -28,7 +28,7 @@ * Provide a way to obtain a inputItr of type O (output), out of an inputItr of type I (input) *

* Things to remember: - Assumes Spark calls hasNext() to check for elements, before calling next() to obtain them - - * Assumes hasNext() gets called atleast once. - Concrete Implementation is responsible for calling inputIterator.next() + * Assumes hasNext() gets called at least once. - Concrete Implementation is responsible for calling inputIterator.next() * and doing the processing in computeNext() */ public abstract class LazyIterableIterator implements Iterable, Iterator { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index db30f301d97a5..5d09762909be7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -2026,7 +2026,7 @@ protected Pair, List> tagRecordsWith // scheduling of INDEX only initializes the file group and not add commit // so if there are no committed file slices, look for inflight slices if (isNonGlobalRLI) { - // For isNonGlobalRLI, new partitions added to the data table will cause new filegroups that are not yet commited + // For isNonGlobalRLI, new partitions added to the data table will cause new filegroups that are not yet committed // therefore, we always need to look for inflight filegroups fileSlices = getPartitionLatestFileSlicesIncludingInflight(metadataMetaClient, Option.ofNullable(fsView), partitionPath); } else if (fileSlices.isEmpty()) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCoalescingPartitioner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCoalescingPartitioner.java index 0f3b972574ce9..9f99742e0c0e4 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCoalescingPartitioner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCoalescingPartitioner.java @@ -123,7 +123,7 @@ public void testCoalescingPartitionerWithRDD(int inputNumPartitions, int targetP }, true).collect(); assertEquals(targetPartitions, countsPerPartition.size()); - // lets validate that atleast we have 50% of data in each spark partition compared to ideal scenario (we can't assume hash of strings will evenly distribute). + // lets validate that at least we have 50% of data in each spark partition compared to ideal scenario (we can't assume hash of strings will evenly distribute). countsPerPartition.forEach(pair -> { int numElements = pair.getValue(); int idealExpectedCount = inputNumPartitions / targetPartitions; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java index e50bb9e5ecb71..9808de708374d 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java @@ -107,7 +107,7 @@ public void testDeleteFailureDuringMarkerReconciliation(DeleteFailureType failur // lets create the data file. so that we can validate later. localStorage.create(storagePath); } catch (IOException e) { - throw new HoodieException("Failed to check data file existance " + fileName); + throw new HoodieException("Failed to check data file existence " + fileName); } }); HoodieTable hoodieTable = HoodieSparkTable.create(writeConfig, getEngineContext(), metaClient); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java index ae6b60ccf679a..677ece46d801b 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java @@ -104,7 +104,7 @@ protected void validateDeltaCommit(String latestDeltaCommit, final Map 0, - "Expect atleast one log file to be present where the latest delta commit was written"); + "Expect at least one log file to be present where the latest delta commit was written"); assertFalse(fileSlice.getBaseFile().isPresent(), "Expect no data-file to be present"); } else { assertTrue(fileSlice.getBaseInstantTime().compareTo(latestDeltaCommit) <= 0, @@ -210,7 +210,7 @@ protected void executeCompaction(String compactionInstantTime, SparkRDDWriteClie if (hasDeltaCommitAfterPendingCompaction) { assertFalse(fileSliceList.stream().anyMatch(fs -> fs.getLogFiles().count() == 0), - "Verify all file-slices have atleast one log-file"); + "Verify all file-slices have at least one log-file"); } else { assertFalse(fileSliceList.stream().anyMatch(fs -> fs.getLogFiles().count() > 0), "Verify all file-slices have no log-files"); diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java index 999f83de66c08..b42ad4396c1be 100644 --- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java +++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java @@ -63,7 +63,7 @@ public boolean validate(GenericRecord record) { return validate((Object) record); } - // Atleast 1 entry should be null + // At least 1 entry should be null private boolean validate(Object object) { if (object == null) { return true; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java index a33ee7c5fb4fd..ad6ae17673835 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java @@ -98,7 +98,7 @@ public void testAvroFileSinkWriter() throws IOException { DeltaWriteStats deltaWriteStats = fileSinkWriter.getDeltaWriteStats(); FileSystem fs = HadoopFSUtils.getFs(basePath, jsc.hadoopConfiguration()); FileStatus[] fileStatuses = fs.listStatus(new Path(deltaWriteStats.getFilePath())); - // Atleast 1 file was written + // At least 1 file was written assertEquals(1, fileStatuses.length); // File length should be greater than 0 assertTrue(fileStatuses[0].getLen() > 0); diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala index 8dff79c1e07b8..91b085b693aaa 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala @@ -187,7 +187,7 @@ class Spark41LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu } // When there are vectorized reads, we can avoid - // 1. opening the file twice by transfering the SeekableInputStream + // 1. opening the file twice by transferring the SeekableInputStream // 2. reading the footer twice by reading all row groups in advance and filter row groups // according to filters that require push down val openedFooter = ParquetFooterReader.openFileAndReadFooter(sharedConf, file, enableVectorizedReader) diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala index f5ccd7b54176f..d15520106e23c 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala @@ -108,7 +108,7 @@ class Spark41ParquetReader(enableVectorizedReader: Boolean, partitionSchema, internalSchemaOpt) // When there are vectorized reads, we can avoid - // 1. opening the file twice by transfering the SeekableInputStream + // 1. opening the file twice by transferring the SeekableInputStream // 2. reading the footer twice by reading all row groups in advance and filter row groups // according to filters that require push down val originalFooter = diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java index fbd8726242a6a..5e09ac9237941 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java @@ -419,7 +419,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, }); Future backfillJobFuture = service.submit(() -> { try { - // trigger backfill atleast after 1 requested entry is added to timeline from continuous job. If not, there is a chance that backfill will complete even before + // trigger backfill at least after 1 requested entry is added to timeline from continuous job. If not, there is a chance that backfill will complete even before // continuous job starts. awaitCondition(new GetCommitsAfterInstant(tableBasePath, lastSuccessfulCommit)); backfillJob.sync(); diff --git a/scripts/pr_compliance.py b/scripts/pr_compliance.py index aafeb9c2a0d0d..5bf1a23b69e19 100644 --- a/scripts/pr_compliance.py +++ b/scripts/pr_compliance.py @@ -46,7 +46,7 @@ class Outcomes: #parsing the next section NEXTSECTION = 2 - #parsing has concluded succesfully, exit with no error + #parsing has concluded successfully, exit with no error SUCCESS = 3 From 41717512f0c00ec5dd1d5a2c8b91210f244220a5 Mon Sep 17 00:00:00 2001 From: Krishen <22875197+kbuci@users.noreply.github.com> Date: Mon, 18 May 2026 21:26:41 -0700 Subject: [PATCH 015/255] fix(flink): enforce Parquet VARIANT annotation in Flink schema conversion for unshredded variant (#18539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(flink): write/read unshredded variant to Flink parquet file writers/readers using Flink's Variant type --------- Co-authored-by: Krishen Bhan <“bkrishen@uber.com”> Co-authored-by: Cursor (cherry picked from commit 71aa121035cf912770f6ad4bf157a945ae3a8409) --- .../row/parquet/ParquetSchemaConverter.java | 31 +++- .../hudi/util/HoodieSchemaConverter.java | 4 + .../parquet/TestParquetSchemaConverter.java | 148 ++++++++++++++++++ .../hudi/util/TestHoodieSchemaConverter.java | 34 +++- ...ITTestVariantCrossEngineCompatibility.java | 3 +- .../apache/hudi/adapter/DataTypeAdapter.java | 20 ++- .../apache/hudi/adapter/DataTypeAdapter.java | 20 ++- .../apache/hudi/adapter/DataTypeAdapter.java | 20 ++- .../apache/hudi/adapter/DataTypeAdapter.java | 20 ++- .../apache/hudi/adapter/DataTypeAdapter.java | 20 ++- .../apache/hudi/adapter/DataTypeAdapter.java | 35 +++++ 11 files changed, 323 insertions(+), 32 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java index 668098314d48b..dd19100f8ccd7 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetSchemaConverter.java @@ -49,6 +49,17 @@ /** * Schema converter converts Parquet schema to and from Flink internal types. * + *

On reads, this converter performs best-effort physical type mapping. It detects the + * Parquet {@code VARIANT} annotation and will reject shredded variants. Blob and Vector types + * cannot be distinguished from ordinary binary columns via Parquet schema alone. + * + *

On writes, this converter maps Flink {@code VariantType} to the canonical unshredded Parquet + * layout (group with binary metadata + value fields). The VARIANT logical type annotation is + * resolved by {@link DataTypeAdapter#variantParquetAnnotation()} — on Flink 2.1+ with + * parquet-java 1.16.0+ the annotation is attached automatically; on pre-2.1 Flink or with + * parquet < 1.16.0 the write throws {@link UnsupportedOperationException} because writing + * variant data without the annotation would produce files that no reader can identify as variant. + * *

Reference org.apache.flink.formats.parquet.utils.ParquetSchemaConverter to support timestamp of INT64 8 bytes. */ @Slf4j @@ -158,6 +169,9 @@ public static RowType.RowField convertToRowField(Type parquetType) { convertToRowField(keyValueType.getLeft()).getType().copy(true), convertToRowField(keyValueType.getRight()).getType())); } else if (hasVariantAnnotation(logicalType)) { + // Fires for files written with parquet-java that carry the VARIANT annotation. + // The reader infers the Flink RowType from the Parquet footer via convertToRowType(), + // so this annotation detection is the primary mechanism for recognizing Variant columns. if (isShreddedVariant(groupType)) { throw new UnsupportedOperationException( "Shredded Variant is not supported in Flink. " @@ -223,10 +237,25 @@ private static boolean isShreddedVariant(GroupType groupType) { /** * Converts a Variant column to the canonical unshredded Parquet layout: * a group with required binary {@code metadata} and required binary {@code value}. + * + *

No shredded-variant guard is needed here: Flink 2.1's {@code VariantType} is a single + * atomic {@code LogicalTypeRoot.VARIANT} with no shredding representation (FLIP-521 scopes + * shredding out), so a shredded variant can never arrive as a Flink LogicalType. + * + *

Delegates to {@link DataTypeAdapter#variantParquetAnnotation()} for the VARIANT logical + * type annotation. On Flink < 2.1 this throws (variant writes are unsupported). On Flink 2.1+ + * with parquet-java < 1.16.0 this also throws, because writing variant data without the + * annotation would produce files that no reader can identify as variant. */ private static Type convertVariantToParquetType(String name, Type.Repetition repetition) { - // TODO: add .as(LogicalTypeAnnotation.variantType()) once parquet-java is bumped to 1.16.0 + LogicalTypeAnnotation annotation = DataTypeAdapter.variantParquetAnnotation() + .orElseThrow(() -> new UnsupportedOperationException( + "Cannot write Variant columns: parquet-java 1.16.0+ is required to emit the VARIANT " + + "logical type annotation. Without the annotation, readers cannot identify the " + + "column as Variant. Current parquet-java version does not support " + + "LogicalTypeAnnotation.variantType().")); return Types.buildGroup(repetition) + .as(annotation) .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED) .named(HoodieSchema.Variant.VARIANT_METADATA_FIELD)) .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java index c49142800b456..c413101dde461 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java @@ -70,6 +70,10 @@ public static HoodieSchema convertToSchema(LogicalType logicalType) { *

The "{rowName}." is used as the nested row type name prefix in order to generate * the right schema. Nested record types that only differ by type name are still compatible. * + *

On Flink 2.1+, {@code LogicalTypeRoot.VARIANT} is detected via string comparison + * (to avoid compile-time dependency) and mapped to {@link HoodieSchema#createVariant()}. + * Pre-2.1 Flink does not support Variant. + * * @param logicalType Flink logical type * @param rowName the record name * @return HoodieSchema matching this logical type diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java index 318f2034d1753..403b840684965 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java @@ -18,6 +18,10 @@ package org.apache.hudi.io.storage.row.parquet; +import org.apache.hudi.adapter.DataTypeAdapter; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.util.Option; + import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.ArrayType; @@ -27,13 +31,19 @@ import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.SmallIntType; import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.types.logical.TinyIntType; import org.apache.flink.table.types.logical.VarCharType; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -41,6 +51,9 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test cases for {@link ParquetSchemaConverter}. @@ -216,4 +229,139 @@ void testConvertTimestampTypes() { + "}\n"; assertThat(messageType.toString(), is(expected)); } + + /** + * A Parquet group with metadata + value binary fields but NO VARIANT annotation must be + * treated as a plain ROW. Only the Parquet {@code VARIANT} annotation triggers variant + * detection in this converter; unannotated groups are never guessed as variant. + */ + @Test + void testVariantPhysicalLayoutTreatedAsRow() { + MessageType variantParquet = new MessageType( + "test", + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, + Type.Repetition.REQUIRED).named("id"), + Types.buildGroup(Type.Repetition.REQUIRED) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.REQUIRED).named("metadata")) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.REQUIRED).named("value")) + .named("data")); + + RowType rowType = ParquetSchemaConverter.convertToRowType(variantParquet); + assertEquals(2, rowType.getFieldCount()); + assertEquals("ROW", rowType.getTypeAt(1).getTypeRoot().name()); + } + + /** + * Unannotated group with metadata + value + typed_value (3 fields) is treated as a generic + * ROW when no annotation or schema hint is present. + */ + @Test + void testUnannotatedShreddedGroupTreatedAsRow() { + MessageType shreddedNoAnnotation = new MessageType( + "test", + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, + Type.Repetition.REQUIRED).named("id"), + Types.buildGroup(Type.Repetition.REQUIRED) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.REQUIRED).named("metadata")) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.REQUIRED).named("value")) + .addField(Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, + Type.Repetition.OPTIONAL).named("typed_value")) + .named("data")); + + RowType rowType = ParquetSchemaConverter.convertToRowType(shreddedNoAnnotation); + assertEquals(2, rowType.getFieldCount()); + assertEquals("ROW", rowType.getTypeAt(1).getTypeRoot().name()); + } + + /** + * On Flink 2.1+ with parquet 1.16.0+, converting a RowType containing a Variant column to a + * Parquet MessageType should produce a group with the VARIANT annotation and required binary + * {@code metadata} and {@code value} fields. + * On pre-2.1 Flink this test is skipped since VariantType does not exist. + * On parquet < 1.16.0 the write is expected to fail (annotation unavailable). + */ + @Test + void testVariantWritePathProducesCorrectLayout() { + LogicalType variantType; + try { + variantType = DataTypeAdapter.createVariantType().getLogicalType(); + } catch (UnsupportedOperationException e) { + // Pre-2.1 Flink: VariantType doesn't exist, skip + return; + } + + RowType rowType = RowType.of( + new LogicalType[]{new IntType(), variantType}, + new String[]{"id", "data"}); + + if (!DataTypeAdapter.variantParquetAnnotation().isPresent()) { + // parquet < 1.16.0: write must fail because annotation is unavailable + UnsupportedOperationException ex = org.junit.jupiter.api.Assertions.assertThrows( + UnsupportedOperationException.class, + () -> ParquetSchemaConverter.convertToParquetMessageType("test", rowType)); + assertTrue(ex.getMessage().contains("parquet-java 1.16.0+"), + "Error message should mention parquet version requirement"); + return; + } + + // parquet 1.16.0+: write succeeds with annotation + MessageType messageType = ParquetSchemaConverter.convertToParquetMessageType("test", rowType); + assertEquals(2, messageType.getFieldCount()); + + Type variantField = messageType.getType("data"); + assertTrue(variantField instanceof GroupType, "Variant column should be a Parquet group"); + GroupType variantGroup = (GroupType) variantField; + assertEquals(2, variantGroup.getFieldCount()); + assertEquals(HoodieSchema.Variant.VARIANT_METADATA_FIELD, variantGroup.getType(0).getName()); + assertEquals(HoodieSchema.Variant.VARIANT_VALUE_FIELD, variantGroup.getType(1).getName()); + assertTrue(variantGroup.getType(0).isPrimitive()); + assertTrue(variantGroup.getType(1).isPrimitive()); + assertEquals(PrimitiveType.PrimitiveTypeName.BINARY, + variantGroup.getType(0).asPrimitiveType().getPrimitiveTypeName()); + assertEquals(PrimitiveType.PrimitiveTypeName.BINARY, + variantGroup.getType(1).asPrimitiveType().getPrimitiveTypeName()); + assertNotNull(variantGroup.getLogicalTypeAnnotation(), + "Variant group must carry the VARIANT annotation"); + } + + /** + * Verifies that writing a Variant column fails with a clear error when parquet-java on the + * classpath does not support the VARIANT annotation (< 1.16.0). On pre-2.1 Flink the adapter + * throws directly; on Flink 2.1+ with parquet < 1.16.0 the write path throws. + */ + @Test + void testVariantWriteFailsWithoutAnnotation() { + Option annotationOpt; + try { + annotationOpt = DataTypeAdapter.variantParquetAnnotation(); + } catch (UnsupportedOperationException e) { + // Pre-2.1 Flink: expected to throw from the adapter + assertTrue(e.getMessage().contains("VARIANT type is only supported in Flink 2.1+")); + return; + } + + if (annotationOpt.isPresent()) { + // parquet 1.16.0+: annotation is available, write succeeds — nothing to test here + return; + } + + // Flink 2.1 + parquet < 1.16.0: annotation is null, write must fail + LogicalType variantType = DataTypeAdapter.createVariantType().getLogicalType(); + RowType rowType = RowType.of( + new LogicalType[]{new IntType(), variantType}, + new String[]{"id", "data"}); + + UnsupportedOperationException ex = org.junit.jupiter.api.Assertions.assertThrows( + UnsupportedOperationException.class, + () -> ParquetSchemaConverter.convertToParquetMessageType("test", rowType)); + assertTrue(ex.getMessage().contains("parquet-java 1.16.0+"), + "Error message should mention the parquet version requirement"); + assertTrue(ex.getMessage().contains("VARIANT"), + "Error message should mention VARIANT"); + } + } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java index 9a4f920b859e4..9f6ad8f53bf73 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java @@ -695,19 +695,16 @@ public void testBlobInNestedStructures() { @Test public void testVariantTypeConversion() { - // Test direct Variant conversion HoodieSchema variantSchema = HoodieSchema.createVariant(); DataType dataType = HoodieSchemaConverter.convertToDataType(variantSchema); assertNotNull(dataType); - // Verify it's a Variant assertThat("the return type should be variant", dataType.getLogicalType().asSummaryString(), is("VARIANT NOT NULL")); } @Test public void testVariantInRecordConversion() { - // Test Variant field within a record HoodieSchema recordWithVariant = HoodieSchema.createRecord( "test_record", null, @@ -722,11 +719,40 @@ public void testVariantInRecordConversion() { assertEquals(2, result.getFieldCount()); assertEquals("data", result.getFieldNames().get(1)); - // Verify variant field assertThat("the return type should be variant", result.getTypeAt(1).asSummaryString(), is("VARIANT NOT NULL")); } + @Test + public void testVariantInArrayConversion() { + HoodieSchema arrayOfVariant = HoodieSchema.createArray(HoodieSchema.createVariant()); + DataType dataType = HoodieSchemaConverter.convertToDataType(arrayOfVariant); + assertNotNull(dataType); + assertInstanceOf(ArrayType.class, dataType.getLogicalType()); + LogicalType elementType = ((ArrayType) dataType.getLogicalType()).getElementType(); + assertEquals("VARIANT", elementType.getTypeRoot().name()); + } + + @Test + public void testVariantInMapConversion() { + HoodieSchema mapOfVariant = HoodieSchema.createMap(HoodieSchema.createVariant()); + DataType dataType = HoodieSchemaConverter.convertToDataType(mapOfVariant); + assertNotNull(dataType); + assertInstanceOf(MapType.class, dataType.getLogicalType()); + LogicalType valueType = ((MapType) dataType.getLogicalType()).getValueType(); + assertEquals("VARIANT", valueType.getTypeRoot().name()); + } + + @Test + public void testShreddedVariantConversionThrows() { + HoodieSchema.Variant shredded = HoodieSchema.createVariantShredded( + HoodieSchema.create(HoodieSchemaType.STRING)); + UnsupportedOperationException ex = assertThrows( + UnsupportedOperationException.class, + () -> HoodieSchemaConverter.convertToDataType(shredded)); + assertTrue(ex.getMessage().contains("Shredded Variant is not yet supported in Flink")); + } + @Test public void testBlobStructureValidation() { // Positive case: Create ROW matching BLOB structure diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java index de6e3835d120a..e7f9ed3766f1a 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVariantCrossEngineCompatibility.java @@ -41,6 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; + /** * Integration test for cross-engine compatibility - verifying that Flink can read Variant tables written by Spark 4.0. */ @@ -56,7 +57,6 @@ public class ITTestVariantCrossEngineCompatibility { private void verifyFlinkCanReadSparkVariantTable(String tablePath, String tableType, String testDescription) throws Exception { TableEnvironment tableEnv = TestTableEnvs.getBatchTableEnv(); - // Create a Hudi table pointing to the Spark-written data String createTableDdl = String.format( "CREATE TABLE variant_table (" + " id INT," @@ -73,7 +73,6 @@ private void verifyFlinkCanReadSparkVariantTable(String tablePath, String tableT tableEnv.executeSql(createTableDdl); - // Query the table to verify Flink can read the data TableResult result = tableEnv.executeSql("SELECT id, name, v, ts FROM variant_table ORDER BY id"); List rows = CollectionUtil.iteratorToList(result.collect()); diff --git a/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 79b9e254dd61d..e8e31b341a180 100644 --- a/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -22,17 +22,27 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + private static final String VARIANT_UNSUPPORTED_MSG = + "VARIANT type is only supported in Flink 2.1+. " + + "Please upgrade your Flink version to use Variant columns."; + + public static Option variantParquetAnnotation() { + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); + } + public static Variant getVariant(RowData rowData, int pos) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static Object createVariant(byte[] value, byte[] metadata) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static boolean isVariantType(LogicalType logicalType) { @@ -40,14 +50,14 @@ public static boolean isVariantType(LogicalType logicalType) { } public static DataType createVariantType() { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantMetadata(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantValue(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 79b9e254dd61d..e8e31b341a180 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -22,17 +22,27 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + private static final String VARIANT_UNSUPPORTED_MSG = + "VARIANT type is only supported in Flink 2.1+. " + + "Please upgrade your Flink version to use Variant columns."; + + public static Option variantParquetAnnotation() { + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); + } + public static Variant getVariant(RowData rowData, int pos) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static Object createVariant(byte[] value, byte[] metadata) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static boolean isVariantType(LogicalType logicalType) { @@ -40,14 +50,14 @@ public static boolean isVariantType(LogicalType logicalType) { } public static DataType createVariantType() { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantMetadata(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantValue(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } } diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 79b9e254dd61d..e8e31b341a180 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -22,17 +22,27 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + private static final String VARIANT_UNSUPPORTED_MSG = + "VARIANT type is only supported in Flink 2.1+. " + + "Please upgrade your Flink version to use Variant columns."; + + public static Option variantParquetAnnotation() { + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); + } + public static Variant getVariant(RowData rowData, int pos) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static Object createVariant(byte[] value, byte[] metadata) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static boolean isVariantType(LogicalType logicalType) { @@ -40,14 +50,14 @@ public static boolean isVariantType(LogicalType logicalType) { } public static DataType createVariantType() { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantMetadata(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantValue(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 79b9e254dd61d..e8e31b341a180 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -22,17 +22,27 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + private static final String VARIANT_UNSUPPORTED_MSG = + "VARIANT type is only supported in Flink 2.1+. " + + "Please upgrade your Flink version to use Variant columns."; + + public static Option variantParquetAnnotation() { + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); + } + public static Variant getVariant(RowData rowData, int pos) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static Object createVariant(byte[] value, byte[] metadata) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static boolean isVariantType(LogicalType logicalType) { @@ -40,14 +50,14 @@ public static boolean isVariantType(LogicalType logicalType) { } public static DataType createVariantType() { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantMetadata(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantValue(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 566ca723648a4..18db48f0fed1f 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -22,17 +22,27 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + private static final String VARIANT_UNSUPPORTED_MSG = + "VARIANT type is only supported in Flink 2.1+. " + + "Please upgrade your Flink version to use Variant columns."; + + public static Option variantParquetAnnotation() { + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); + } + public static Variant getVariant(RowData rowData, int pos) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static Object createVariant(byte[] value, byte[] metadata) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static boolean isVariantType(LogicalType logicalType) { @@ -40,14 +50,14 @@ public static boolean isVariantType(LogicalType logicalType) { } public static DataType createVariantType() { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantMetadata(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } public static byte[] getVariantValue(Object obj) { - throw new UnsupportedOperationException("Variant is not supported yet."); + throw new UnsupportedOperationException(VARIANT_UNSUPPORTED_MSG); } } \ No newline at end of file diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java index 6ae4c4a6babae..77d6f708ece31 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java @@ -25,11 +25,46 @@ import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.types.variant.BinaryVariant; import org.apache.flink.types.variant.Variant; +import org.apache.hudi.common.util.Option; +import org.apache.parquet.schema.LogicalTypeAnnotation; + +import java.lang.reflect.Method; /** * Adapter utils to provide {@code DataType} utilities. */ public class DataTypeAdapter { + + /** + * The Parquet Variant binary format specification version passed to + * {@code LogicalTypeAnnotation.variantType(byte)}. Version 1 is the initial spec + * defined by the Parquet Variant proposal (parquet-format 2.11.0 / parquet-java 1.16.0). + */ + private static final byte VARIANT_SPEC_VERSION = 1; + + /** + * Cached VARIANT annotation resolved via reflection. Empty if parquet-java + * on the classpath predates {@code LogicalTypeAnnotation.variantType()} (< 1.16.0). + */ + private static final Option VARIANT_ANNOTATION = resolveVariantAnnotation(); + + private static Option resolveVariantAnnotation() { + try { + Method factory = LogicalTypeAnnotation.class.getMethod("variantType", byte.class); + return Option.of((LogicalTypeAnnotation) factory.invoke(null, VARIANT_SPEC_VERSION)); + } catch (Exception e) { + return Option.empty(); + } + } + + /** + * Returns the Parquet VARIANT {@link LogicalTypeAnnotation} if parquet-java 1.16.0+ is on the + * classpath, or empty if the annotation class is unavailable. + */ + public static Option variantParquetAnnotation() { + return VARIANT_ANNOTATION; + } + public static Variant getVariant(RowData rowData, int pos) { return rowData.getVariant(pos); } From dbfafbcf5738ce7c34c844492145530359c7a8ea Mon Sep 17 00:00:00 2001 From: Mahsood Ebrahim Date: Mon, 18 May 2026 23:43:55 -0700 Subject: [PATCH 016/255] feat(spark): add restore_to_instant stored procedure (#18696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(spark): add restore_to_instant stored procedure Adds a Spark SQL stored procedure that performs a point-in-time table restore to any instant on the active timeline, with optional post-restore file-existence audit. Unlike rollback_to_savepoint, no savepoint is required at the target instant. Centralizes the MDT pre-check that was inlined in restoreToSavepoint into a new BaseHoodieWriteClient.shouldDeleteMdtBeforeRestore helper, and extends restoreToInstant to invoke it. The helper also catches the penultimate-compaction case (target at or before the second-most-recent MDT compaction) which the previous restoreToSavepoint inline check missed. IO/permission failures now surface as HoodieException instead of being silently swallowed. The audit returns a tri-state result (PASSED / FAILED / INCONCLUSIVE) so transient cloud-storage timeouts are distinguishable from real audit failures; an audit_only mode lets users re-audit a previously completed restore by passing its restore_instant_time. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(spark): address PR review: public MDT guard, variable rename, test coverage Add a public deleteMetadataTableIfNecessaryBeforeRestore method to BaseHoodieWriteClient so callers that drive restore via restoreToInstant directly (e.g. the restore_to_instant procedure) can pre-check and pre-emptively delete the MDT before calling restoreToInstant. This closes the coverage gap identified in review: the procedure previously had no MDT protection after shouldDeleteMdtBeforeRestore was removed from restoreToInstant. The procedure now calls client.deleteMetadataTableIfNecessaryBeforeRestore before restoreToInstant and passes the returned boolean as initialMetadataTableIfNecessary, ensuring the penultimate/oldest compaction and timeline-start checks fire on the procedure path. Rename internal Scala variable restoreInstantTime -> startRestoreTimeArg for consistency with the start_restore_time parameter name. Strengthen testRestoreToInstantSkipsMdtCheckWhenMetadataDisabled: the test now verifies (a) deleteMetadataTableIfNecessaryBeforeRestore returns false and deletes the MDT for a target at/before the oldest compaction, and (b) restoreToInstant(target, false) then proceeds without invoking the guard. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(spark): address PR review round 2 — remove penultimate check, rename public API Remove the penultimate-compaction check from shouldDeleteMdtBeforeRestore. The check fired too aggressively: it deleted the MDT even when the MDT could restore successfully, and then the fresh re-bootstrap failed for record_index-enabled tables (testRLIWithMDTCleaning) or triggered an inline MDT compaction during bootstrap that broke the isMetadataTableRecreatedDuringRestore detection (testRestoreToSavepointDeletesMdtWhenTargetIsBeforePenultimateCompaction). Only the oldest-compaction and timeline-start checks remain, matching the original pre-PR behaviour. Add a catch clause for HoodieException (e.g. TableNotFoundException from a partially initialized MDT directory) so a corrupt-but-present MDT is treated as absent rather than hard-failing the restore. Rename deleteMetadataTableIfNecessaryBeforeRestore -> deleteMdtIfNecessaryBeforeRestore and flip the return value so that true = MDT was deleted (callers negate with ! when passing to restoreToInstant). Update RestoreToInstantProcedure and the corresponding Java test accordingly. Restore the @Deprecated annotation on the two-arg rollback overload that was accidentally dropped in a prior commit. Remove testRestoreToSavepointDeletesMdtWhenTargetIsBeforePenultimateCompaction which tested the now-removed penultimate check. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: mahsoode Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit e406e5d092d090926803946445ede3e6ac261af0) --- .../hudi/client/BaseHoodieWriteClient.java | 120 ++++--- .../TestSavepointRestoreCopyOnWrite.java | 89 +++++ .../command/procedures/HoodieProcedures.scala | 1 + .../RestoreToInstantProcedure.scala | 309 ++++++++++++++++++ .../hudi/procedure/TestRestoreProcedure.scala | 258 +++++++++++++++ 5 files changed, 739 insertions(+), 38 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RestoreToInstantProcedure.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRestoreProcedure.scala diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index df06df2fbbba9..76ffe51a256d2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -92,6 +92,7 @@ import org.apache.hudi.metadata.HoodieTableMetadataWriter; import org.apache.hudi.metadata.MetadataPartitionType; import org.apache.hudi.metrics.HoodieMetrics; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; @@ -846,44 +847,11 @@ public void restoreToSavepoint() { */ public void restoreToSavepoint(String savepointTime) { boolean initializeMetadataTableIfNecessary = config.isMetadataTableEnabled(); - if (initializeMetadataTableIfNecessary) { - try { - // Delete metadata table directly when users trigger savepoint rollback if mdt existed and if the savePointTime is beforeTimelineStarts - // or before the oldest compaction on MDT. - // We cannot restore to before the oldest compaction on MDT as we don't have the basefiles before that time. - HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder() - .setConf(storageConf.newInstance()) - .setBasePath(getMetadataTableBasePath(config.getBasePath())).build(); - Option oldestMdtCompaction = mdtMetaClient.getCommitTimeline().filterCompletedInstants().firstInstant(); - boolean deleteMDT = false; - if (oldestMdtCompaction.isPresent()) { - if (LESSER_THAN_OR_EQUALS.test(savepointTime, oldestMdtCompaction.get().requestedTime())) { - log.warn("Deleting MDT during restore to {} as the savepoint is older than oldest compaction {} on MDT", - savepointTime, oldestMdtCompaction.get().requestedTime()); - deleteMDT = true; - } - } - - // The instant required to sync rollback to MDT has been archived and the mdt syncing will be failed - // So that we need to delete the whole MDT here. - if (!deleteMDT) { - HoodieInstant syncedInstant = mdtMetaClient.createNewInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, savepointTime); - if (mdtMetaClient.getCommitsTimeline().isBeforeTimelineStarts(syncedInstant.requestedTime())) { - log.warn("Deleting MDT during restore to {} as the savepoint is older than the MDT timeline {}", - savepointTime, mdtMetaClient.getCommitsTimeline().firstInstant().get().requestedTime()); - deleteMDT = true; - } - } - - if (deleteMDT) { - HoodieTableMetadataUtil.deleteMetadataTable(config.getBasePath(), context); - // rollbackToSavepoint action will try to bootstrap MDT at first but sync to MDT will fail at the current scenario. - // so that we need to disable metadata initialized here. - initializeMetadataTableIfNecessary = false; - } - } catch (Exception e) { - // Metadata directory does not exist - } + if (initializeMetadataTableIfNecessary && shouldDeleteMdtBeforeRestore(savepointTime)) { + HoodieTableMetadataUtil.deleteMetadataTable(config.getBasePath(), context); + // rollbackToSavepoint action will try to bootstrap MDT at first but sync to MDT will fail at the current scenario. + // so that we need to disable metadata initialized here. + initializeMetadataTableIfNecessary = false; } HoodieTable table = initTable(WriteOperationType.UNKNOWN, Option.empty(), initializeMetadataTableIfNecessary); @@ -894,6 +862,82 @@ public void restoreToSavepoint(String savepointTime) { SavepointHelpers.validateSavepointRestore(table, savepointTime); } + /** + * Decides whether the metadata table (MDT) must be deleted before restoring the data table to + * {@code targetInstant}. Returns true when restoring would leave the MDT in an inconsistent + * state, specifically when any of the following holds: + *

    + *
  1. The target is at or before the oldest completed compaction. We cannot restore to before + * the oldest compaction because we don't have base files before that time.
    2. + *
  2. The target is before the MDT timeline start (the relevant history was archived away).
    3. + *
+ * Returns false when the MDT directory does not exist or is not readable (nothing to delete or + * worry about). Wraps genuine IO failures ({@link IOException}) in a {@link HoodieException} + * so permission / network errors surface to the caller. + */ + protected boolean shouldDeleteMdtBeforeRestore(String targetInstant) { + String mdtBasePath = getMetadataTableBasePath(config.getBasePath()); + try { + // Cheap existence check first to avoid constructing an MDT meta client when there is no MDT. + if (!storage.exists(new StoragePath(mdtBasePath))) { + return false; + } + HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder() + .setConf(storageConf.newInstance()) + .setBasePath(mdtBasePath).build(); + List completedCompactions = mdtMetaClient.getCommitTimeline() + .filterCompletedInstants().getInstants(); + Option oldestMdtCompaction = completedCompactions.isEmpty() + ? Option.empty() : Option.of(completedCompactions.get(0)); + if (oldestMdtCompaction.isPresent() + && LESSER_THAN_OR_EQUALS.test(targetInstant, oldestMdtCompaction.get().requestedTime())) { + log.warn("Deleting MDT before restore to {}: target is at or before oldest MDT compaction {}", + targetInstant, oldestMdtCompaction.get().requestedTime()); + return true; + } + if (mdtMetaClient.getCommitsTimeline().isBeforeTimelineStarts(targetInstant)) { + log.warn("Deleting MDT before restore to {}: target is before MDT timeline start", targetInstant); + return true; + } + return false; + } catch (IOException e) { + throw new HoodieException( + "Failed to inspect MDT at " + mdtBasePath + " before restore to " + targetInstant + + " - refusing to silently proceed without an MDT integrity check.", e); + } catch (HoodieException e) { + // MDT directory exists but is not usable (e.g. TableNotFoundException from a partially + // initialized MDT). Treat as absent: no deletion needed, let the restore proceed. + log.warn("MDT at {} is present but could not be read ({}); skipping pre-check.", + mdtBasePath, e.getMessage()); + return false; + } + } + + /** + * Deletes the metadata table (MDT) if it would be left in an inconsistent state by a restore + * to {@code targetInstant}, and returns whether the MDT was actually deleted. + * + *

Callers that drive restore via {@link #restoreToInstant} directly (e.g. the + * {@code restore_to_instant} stored procedure) should call this method before invoking + * {@code restoreToInstant} and suppress MDT initialization when it returns {@code true}: + * + * 

{@code
+   * boolean mdtDeleted = client.deleteMdtIfNecessaryBeforeRestore(targetInstant);
+   * client.restoreToInstant(targetInstant, !mdtDeleted && enableMetadata);
+   * }
+ * + * @param targetInstant the instant the data table will be restored to + * @return {@code true} if the MDT was deleted (caller must not re-initialize it); + * {@code false} otherwise (MDT either did not need deletion or does not exist) + */ + public boolean deleteMdtIfNecessaryBeforeRestore(String targetInstant) { + if (shouldDeleteMdtBeforeRestore(targetInstant)) { + HoodieTableMetadataUtil.deleteMetadataTable(config.getBasePath(), context); + return true; + } + return false; + } + @Deprecated public boolean rollback(final String commitInstantTime) throws HoodieRollbackException { HoodieTable table = initTable(WriteOperationType.UNKNOWN, Option.empty()); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java index 0be971630c346..5c3c832260736 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java @@ -29,6 +29,8 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.storage.HoodieStorageUtils; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.testutils.HoodieClientTestBase; @@ -41,6 +43,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -217,4 +220,90 @@ void testCleaningRollbackInstants(boolean commitRollback) throws Exception { assertRowNumberEqualsTo(20); } } + + /** + * Two-assertion test for the MDT pre-check guard: + *
    + *
  1. {@code deleteMdtIfNecessaryBeforeRestore(firstCommit)} returns {@code true} (and + * deletes the MDT) when the target is at or before the oldest MDT compaction, + * confirming the guard logic fires for this table setup.
    2. + *
  2. {@code restoreToInstant(firstCommit, false)} does NOT invoke the pre-check, so a + * caller that explicitly opts out of MDT integration is never surprised by an implicit + * MDT deletion.
    3. + *
+ */ + @Test + void testRestoreToInstantSkipsMdtCheckWhenMetadataDisabled() throws Exception { + HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY) + .withRollbackUsingMarkers(true) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withMaxNumDeltaCommitsBeforeCompaction(4) + .build()) + .build(); + try (SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig)) { + String firstCommit = null; + String prevInstant = HoodieTimeline.INIT_INSTANT_TS; + final int numRecords = 10; + // 5 inserts so the MDT goes through one compaction with maxDeltaCommits=4. + for (int i = 1; i <= 5; i++) { + String newCommitTime = WriteClientTestUtils.createNewInstantTime(); + insertBatch(hoodieWriteConfig, client, newCommitTime, prevInstant, numRecords, SparkRDDWriteClient::insert, + false, true, numRecords, numRecords * i, 1, Option.empty(), INSTANT_GENERATOR); + prevInstant = newCommitTime; + if (i == 1) { + firstCommit = newCommitTime; + } + } + assertRowNumberEqualsTo(50); + + String mdtBasePath = HoodieTableMetadata.getMetadataTableBasePath(hoodieWriteConfig.getBasePath()); + assertTrue(HoodieStorageUtils.getStorage(mdtBasePath, storageConf).exists(new StoragePath(mdtBasePath)), + "MDT directory should exist before any pre-check"); + + // Assertion 1: deleteMdtIfNecessaryBeforeRestore detects that firstCommit is at or before + // the oldest MDT compaction, deletes the MDT, and returns true. + boolean mdtDeleted = client.deleteMdtIfNecessaryBeforeRestore( + Objects.requireNonNull(firstCommit, "first commit should not be null")); + assertTrue(mdtDeleted, + "deleteMdtIfNecessaryBeforeRestore should return true when target is at or before the oldest MDT compaction"); + assertFalse(HoodieStorageUtils.getStorage(mdtBasePath, storageConf).exists(new StoragePath(mdtBasePath)), + "MDT directory should have been deleted by deleteMdtIfNecessaryBeforeRestore"); + + // Assertion 2: restoreToInstant with initialMetadataTableIfNecessary=false does NOT invoke + // the pre-check — the MDT (now absent) is not touched. The restore proceeds without MDT. + client.restoreToInstant(firstCommit, false); + assertRowNumberEqualsTo(numRecords); + } + } + + /** + * Regression coverage for the {@code restoreToSavepoint} refactor. After replacing the inline + * MDT pre-check with a call to the centralized helper, the common case (target after the + * oldest MDT compaction, no MDT delete needed) must still work end-to-end. + */ + @Test + void testRestoreToSavepointStillWorksAfterRefactor() throws Exception { + HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY) + .withRollbackUsingMarkers(true) + .build(); + try (SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig)) { + String savepointCommit = null; + String prevInstant = HoodieTimeline.INIT_INSTANT_TS; + final int numRecords = 10; + for (int i = 1; i <= 4; i++) { + String newCommitTime = WriteClientTestUtils.createNewInstantTime(); + insertBatch(hoodieWriteConfig, client, newCommitTime, prevInstant, numRecords, SparkRDDWriteClient::insert, + false, true, numRecords, numRecords * i, 1, Option.empty(), INSTANT_GENERATOR); + prevInstant = newCommitTime; + if (i == 2) { + savepointCommit = newCommitTime; + client.savepoint("user1", "Savepoint for 2nd commit"); + } + } + assertRowNumberEqualsTo(40); + client.restoreToSavepoint(Objects.requireNonNull(savepointCommit, "restore commit should not be null")); + assertRowNumberEqualsTo(20); + } + } } diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala index 4319ce77c1c2a..027c9f8c2882d 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala @@ -39,6 +39,7 @@ object HoodieProcedures { ,(DeleteSavepointProcedure.NAME, DeleteSavepointProcedure.builder) ,(RollbackToSavepointProcedure.NAME, RollbackToSavepointProcedure.builder) ,(RollbackToInstantTimeProcedure.NAME, RollbackToInstantTimeProcedure.builder) + ,(RestoreToInstantProcedure.NAME, RestoreToInstantProcedure.builder) ,(RunClusteringProcedure.NAME, RunClusteringProcedure.builder) ,(ShowClusteringProcedure.NAME, ShowClusteringProcedure.builder) ,(ShowCommitsProcedure.NAME, ShowCommitsProcedure.builder) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RestoreToInstantProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RestoreToInstantProcedure.scala new file mode 100644 index 0000000000000..ca5a2fc924ae7 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RestoreToInstantProcedure.scala @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.command.procedures + +import org.apache.hudi.HoodieCLIUtils +import org.apache.hudi.avro.model.HoodieRestoreMetadata +import org.apache.hudi.client.SparkRDDWriteClient +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.fs.ConsistencyGuardConfig +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.timeline.HoodieInstant +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.hadoop.fs.HadoopFSUtils +import org.apache.hudi.storage.StoragePath + +import org.apache.hadoop.fs.Path +import org.apache.spark.internal.Logging +import org.apache.spark.sql.Row +import org.apache.spark.sql.hudi.command.procedures.RestoreToInstantProcedure._ +import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} + +import java.util.function.Supplier + +import scala.collection.JavaConverters._ + +/** + * Stored procedure to perform a full point-in-time table restore to a given instant. + * + * Unlike [[RollbackToSavepointProcedure]] (which requires a savepoint at the target instant), + * this procedure calls restoreToInstant() directly and works on any arbitrary instant on the + * active timeline. + * + * Parameters: + * - table / path: identifies the Hudi table (one must be provided) + * - instant_time: target commit to restore to (required when audit_only=false; must be omitted + * when audit_only=true) + * - start_restore_time: the restore operation's own timeline timestamp (the start_restore_time + * value returned by a prior restore_to_instant call). Required when + * audit_only=true; must be omitted otherwise. + * - enable_metadata: whether the metadata table is enabled (default: true) + * - rollback_parallelism: Spark parallelism for rollback and audit operations (default: 4) + * - enable_consistency_guard: enable consistency guard for file existence checks (default: false) + * - audit_post_restore: after restoring, verify that all successfully-deleted files are absent (default: false) + * - audit_only: skip the restore and only audit a previously completed restore instant (default: false) + * + * Output columns: + * - restore_result: true if restore succeeded; null if audit_only=true + * - start_restore_time: the restore operation's own timeline timestamp; null if audit_only=true. + * Pass this value as start_restore_time to re-run the audit later. + * - time_taken_in_millis: restore duration; null if audit_only=true + * - instants_rolled_back: number of commits rolled back; null if audit_only=true + * - audit_result: one of "PASSED" / "FAILED" / "INCONCLUSIVE" when an audit ran; null otherwise. + * INCONCLUSIVE means at least one file existence check threw an IOException + * (e.g. transient cloud-storage timeout) — re-run audit_only=true to retry. + */ +class RestoreToInstantProcedure extends BaseProcedure with ProcedureBuilder with Logging { + + private val PARAMETERS = Array[ProcedureParameter]( + ProcedureParameter.optional(0, "table", DataTypes.StringType), + ProcedureParameter.optional(1, "instant_time", DataTypes.StringType), + ProcedureParameter.optional(2, "enable_metadata", DataTypes.BooleanType, true), + ProcedureParameter.optional(3, "rollback_parallelism", DataTypes.IntegerType, 4), + ProcedureParameter.optional(4, "enable_consistency_guard", DataTypes.BooleanType, false), + ProcedureParameter.optional(5, "audit_post_restore", DataTypes.BooleanType, false), + ProcedureParameter.optional(6, "audit_only", DataTypes.BooleanType, false), + ProcedureParameter.optional(7, "path", DataTypes.StringType), + ProcedureParameter.optional(8, "start_restore_time", DataTypes.StringType) + ) + + private val OUTPUT_TYPE = new StructType(Array[StructField]( + StructField("restore_result", DataTypes.BooleanType, nullable = true, Metadata.empty), + StructField("start_restore_time", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("time_taken_in_millis", DataTypes.LongType, nullable = true, Metadata.empty), + StructField("instants_rolled_back", DataTypes.LongType, nullable = true, Metadata.empty), + StructField("audit_result", DataTypes.StringType, nullable = true, Metadata.empty) + )) + + def parameters: Array[ProcedureParameter] = PARAMETERS + + def outputType: StructType = OUTPUT_TYPE + + override def call(args: ProcedureArgs): Seq[Row] = { + super.checkArgs(PARAMETERS, args) + + val tableName = getArgValueOrDefault(args, PARAMETERS(0)) + val instantTime = getArgValueOrDefault(args, PARAMETERS(1)) + val enableMetadata = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Boolean] + val rollbackParallelism = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Int] + val enableConsistencyGuard = getArgValueOrDefault(args, PARAMETERS(4)).get.asInstanceOf[Boolean] + val shouldAuditPostRestore = getArgValueOrDefault(args, PARAMETERS(5)).get.asInstanceOf[Boolean] + val auditOnly = getArgValueOrDefault(args, PARAMETERS(6)).get.asInstanceOf[Boolean] + val tablePath = getArgValueOrDefault(args, PARAMETERS(7)) + val startRestoreTimeArg = getArgValueOrDefault(args, PARAMETERS(8)) + + // Cross-validation: each of (instant_time, start_restore_time) has one unambiguous meaning. + if (!auditOnly && instantTime.isEmpty) { + throw new HoodieException("instant_time is required when audit_only=false.") + } + if (auditOnly && startRestoreTimeArg.isEmpty) { + throw new HoodieException( + "start_restore_time is required when audit_only=true. " + + "Pass the start_restore_time value from a prior restore_to_instant call.") + } + if (!auditOnly && startRestoreTimeArg.isDefined) { + throw new HoodieException("start_restore_time may only be specified when audit_only=true.") + } + if (auditOnly && instantTime.isDefined) { + throw new HoodieException( + "instant_time may only be specified when audit_only=false. " + + "Use start_restore_time to identify a previously executed restore.") + } + if (auditOnly && shouldAuditPostRestore) { + logWarning("Both audit_only and audit_post_restore are set. Only audit_only will be honored.") + } + + val basePath = getBasePath(tableName, tablePath) + + val confs = Map( + HoodieMetadataConfig.ENABLE.key() -> enableMetadata.toString, + HoodieWriteConfig.ROLLBACK_PARALLELISM_VALUE.key() -> rollbackParallelism.toString, + HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.key() -> "false" + ) ++ (if (enableConsistencyGuard) Map(ConsistencyGuardConfig.ENABLE.key() -> "true") else Map.empty) + + val metaClient = createMetaClient(jsc, basePath) + + // Nullable boxed types so Row can hold null for audit_only runs + var restoreResult: java.lang.Boolean = null + var startRestoreTime: String = null + var timeTakenInMillis: java.lang.Long = null + var instantsRolledBack: java.lang.Long = null + + if (!auditOnly) { + val targetInstant = instantTime.get.asInstanceOf[String] + var client: SparkRDDWriteClient[_] = null + try { + client = HoodieCLIUtils.createHoodieWriteClient(sparkSession, basePath, confs, + tableName.asInstanceOf[Option[String]]) + // Pre-check: if the target instant is at or before the oldest MDT compaction or before the + // MDT timeline start, pre-emptively delete the MDT so restoreToInstant does not leave it + // inconsistent. deleteMdtIfNecessaryBeforeRestore returns true when the MDT was deleted + // (caller must not re-initialize it), false otherwise. + val mdtDeleted = if (enableMetadata) { + client.deleteMdtIfNecessaryBeforeRestore(targetInstant) + } else false + val restoreMetadata = client.restoreToInstant(targetInstant, !mdtDeleted && enableMetadata) + restoreResult = true + startRestoreTime = restoreMetadata.getStartRestoreTime + timeTakenInMillis = restoreMetadata.getTimeTakenInMillis + instantsRolledBack = restoreMetadata.getInstantsToRollback.size().toLong + } finally { + if (client != null) { + client.close() + } + } + if (tableName.isDefined) { + spark.catalog.refreshTable(tableName.get.asInstanceOf[String]) + } + // getActiveTimeline is lazily cached; reload so the new .restore instant is visible to the audit. + if (shouldAuditPostRestore) { + metaClient.reloadActiveTimeline() + } + } + + var auditResult: String = null + if (auditOnly || shouldAuditPostRestore) { + val restoreInstant: HoodieInstant = if (auditOnly) { + val ts = startRestoreTimeArg.get.asInstanceOf[String] + val instants = metaClient.getActiveTimeline.getRestoreTimeline.filterCompletedInstants + .getInstants.asScala + instants.find(_.requestedTime().equals(ts)).getOrElse( + throw new HoodieException(s"No completed restore instant found for $ts. " + + "Pass the start_restore_time from a prior restore_to_instant call as start_restore_time.") + ) + } else { + // Use startRestoreTime captured from the restore metadata — more deterministic than + // lastInstant() since another concurrent restore could otherwise land in between. + val ts = startRestoreTime + val instants = metaClient.getActiveTimeline.getRestoreTimeline.filterCompletedInstants + .getInstants.asScala + instants.find(_.requestedTime().equals(ts)).getOrElse( + throw new HoodieException(s"No completed restore instant found for $ts after restore.") + ) + } + auditResult = auditPostRestore(metaClient, basePath, restoreInstant, rollbackParallelism) + } + + Seq(Row(restoreResult, startRestoreTime, timeTakenInMillis, instantsRolledBack, auditResult)) + } + + /** + * Verifies that all files expected to have been deleted by a restore operation are actually + * absent from storage. Returns one of "PASSED", "FAILED", or "INCONCLUSIVE": + * - PASSED: every file expected to be absent is in fact absent. + * - FAILED: at least one file is still present after restore. + * - INCONCLUSIVE: no file was confirmed present, but at least one existence check threw + * an IOException (e.g. transient cloud-storage timeout). Re-run with + * audit_only=true to retry. + */ + private def auditPostRestore( + metaClient: HoodieTableMetaClient, + basePath: String, + restoreInstant: HoodieInstant, + rollbackParallelism: Int): String = { + try { + val restoreMetadata: HoodieRestoreMetadata = + metaClient.getActiveTimeline.readRestoreMetadata(restoreInstant) + + val filesToCheck = new java.util.ArrayList[String]() + restoreMetadata.getHoodieRestoreMetadata.values.asScala.foreach { rollbackList => + rollbackList.asScala.foreach { rollback => + rollback.getPartitionMetadata.asScala.foreach { case (partition, pm) => + val partitionPathStr = basePath + Path.SEPARATOR + partition + val partitionStoragePath = new StoragePath(partitionPathStr) + if (!metaClient.getStorage.exists(partitionStoragePath)) { + logInfo(s"Partition path $partitionStoragePath does not exist. Skipping audit for its files.") + } else { + // Use .toString() to preserve the full absolute path. Using .getName() would strip + // the path to just the filename, making the subsequent FS.exists() check vacuously pass. + // Only check getSuccessDeleteFiles: these are the files the restore intended to delete + // and whose absence we can meaningfully verify. Files in getFailedDeleteFiles already + // failed to be deleted and are expected to still be present. + pm.getSuccessDeleteFiles.asScala.foreach(f => + filesToCheck.add(new Path(partitionPathStr, f).toString)) + } + } + } + } + + if (filesToCheck.isEmpty) { + logInfo(s"No files to audit for restore instant ${restoreInstant.requestedTime()}") + "PASSED" + } else { + // HadoopFSUtils.getStorageConfWithCopy returns a Serializable StorageConfiguration; the + // closure captures only storageConf, so Spark's ClosureCleaner can serialize it cleanly. + // HadoopFSUtils.getFs internally calls prepareHadoopConf, preserving HOODIE_ENV_* (e.g. + // S3A) injection at cloud DCs. + val storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration()) + val outcomes = jsc.parallelize(filesToCheck, Math.max(1, rollbackParallelism)) + .map { pathStr => + val p = new Path(pathStr) + try { + val exists = HadoopFSUtils.getFs(p, storageConf.unwrap()).exists(p) + val status: AuditFileStatus = if (exists) Present else Absent + (pathStr, status) + } catch { + case e: Exception => (pathStr, IndeterminateError(e.getMessage)) + } + } + .collect() + .asScala + .toArray + + val present = outcomes.collect { case (p, Present) => p } + val indeterminate = outcomes.collect { case (p, IndeterminateError(msg)) => (p, msg) } + if (present.nonEmpty) { + logError(s"Restore audit FAILED for instant ${restoreInstant.requestedTime()}: " + + s"${present.length} file(s) still present, e.g. ${present.take(5).mkString(", ")}") + "FAILED" + } else if (indeterminate.nonEmpty) { + logWarning(s"Restore audit INCONCLUSIVE for instant ${restoreInstant.requestedTime()}: " + + s"${indeterminate.length} file(s) had IO errors, e.g. ${indeterminate.take(5).mkString(", ")}") + "INCONCLUSIVE" + } else { + logInfo(s"Restore audit PASSED for instant ${restoreInstant.requestedTime()}: " + + s"${outcomes.length} file(s) confirmed absent.") + "PASSED" + } + } + } catch { + case e: Exception => + logError(s"Exception during restore audit for instant ${restoreInstant.requestedTime()}", e) + "INCONCLUSIVE" + } + } + + override def build: Procedure = new RestoreToInstantProcedure() +} + +object RestoreToInstantProcedure { + val NAME: String = "restore_to_instant" + + def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] { + override def get(): RestoreToInstantProcedure = new RestoreToInstantProcedure() + } + + // ADT for per-file audit outcomes. Defined at object level (NOT inside auditPostRestore) so the + // generated bytecode does not embed a synthetic outer-class reference; otherwise Spark's + // ClosureCleaner cannot serialize the closure that returns these values from executors. + private sealed trait AuditFileStatus extends Serializable + private case object Absent extends AuditFileStatus + private case object Present extends AuditFileStatus + private case class IndeterminateError(message: String) extends AuditFileStatus +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRestoreProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRestoreProcedure.scala new file mode 100644 index 0000000000000..7674fa6abaa8f --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRestoreProcedure.scala @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +class TestRestoreProcedure extends HoodieSparkProcedureTestBase { + + private def createTableAndInsertData(tableName: String, tablePath: String, tableType: String = "cow"): Array[String] = { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | location '$tablePath' + | tblproperties ( + | primaryKey = 'id', + | preCombineField = 'ts', + | type = '$tableType' + | ) + """.stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10.0, 1000") + spark.sql(s"insert into $tableName select 2, 'a2', 20.0, 1500") + spark.sql(s"insert into $tableName select 3, 'a3', 30.0, 2000") + spark.sql(s"insert into $tableName select 4, 'a4', 40.0, 2500") + spark.sql(s"call show_commits(table => '$tableName')").collect() + .map(_.getString(0)).sorted + } + + test("Test restore_to_instant basic CoW") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + assertResult(4)(commits.length) + + // restore to after the 2nd commit — only commits(0) and commits(1) should remain + val result = spark.sql( + s"call restore_to_instant(table => '$tableName', instant_time => '${commits(1)}')" + ).collect() + + assertResult(1)(result.length) + assertResult(true)(result(0).getBoolean(0)) // restore_result + assert(result(0).getString(1) != null) // start_restore_time (dynamic timestamp) + assert(result(0).getLong(2) >= 0) // time_taken_in_millis + assert(result(0).getLong(3) >= 0L) // instants_rolled_back + assertResult(true)(result(0).isNullAt(4)) // audit_result = null (audit not requested) + + // verify data reverted to 2 records + val count = spark.sql(s"select count(*) from $tableName").collect()(0).getLong(0) + assertResult(2)(count) + } + } + + test("Test restore_to_instant basic MoR") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath, tableType = "mor") + assertResult(4)(commits.length) + + val result = spark.sql( + s"call restore_to_instant(table => '$tableName', instant_time => '${commits(1)}')" + ).collect() + + assertResult(1)(result.length) + assertResult(true)(result(0).getBoolean(0)) + assert(result(0).getString(1) != null) + assert(result(0).getLong(2) >= 0) + assert(result(0).getLong(3) >= 0L) + assertResult(true)(result(0).isNullAt(4)) + + val count = spark.sql(s"select count(*) from $tableName").collect()(0).getLong(0) + assertResult(2)(count) + } + } + + test("Test restore_to_instant using path parameter") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + assertResult(4)(commits.length) + + val result = spark.sql( + s"call restore_to_instant(path => '$tablePath', instant_time => '${commits(1)}')" + ).collect() + + assertResult(1)(result.length) + assertResult(true)(result(0).getBoolean(0)) + assert(result(0).isNullAt(4)) + + val count = spark.sql(s"select count(*) from $tableName").collect()(0).getLong(0) + assertResult(2)(count) + } + } + + test("Test restore_to_instant with audit_post_restore") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + assertResult(4)(commits.length) + + val result = spark.sql( + s"""call restore_to_instant( + | table => '$tableName', + | instant_time => '${commits(1)}', + | audit_post_restore => true + |)""".stripMargin + ).collect() + + assertResult(1)(result.length) + assertResult(true)(result(0).getBoolean(0)) // restore_result + assert(result(0).getString(1) != null) // start_restore_time + // audit_result should be "PASSED": all rolled-back files are absent + assertResult(false)(result(0).isNullAt(4)) + assertResult("PASSED")(result(0).getString(4)) + + val count = spark.sql(s"select count(*) from $tableName").collect()(0).getLong(0) + assertResult(2)(count) + } + } + + test("Test restore_to_instant audit_only mode") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + assertResult(4)(commits.length) + + // Round 1: perform the restore and capture the restore operation's own timeline timestamp + val restoreRows = spark.sql( + s"call restore_to_instant(table => '$tableName', instant_time => '${commits(1)}')" + ).collect() + assertResult(true)(restoreRows(0).getBoolean(0)) + // start_restore_time is the restore instant's timeline timestamp — NOT commits(1) + val restoreInstantTs = restoreRows(0).getString(1) + assert(restoreInstantTs != null) + assert(restoreInstantTs != commits(1)) + + // Round 2: audit_only using start_restore_time (the original target commit is not needed) + val auditRows = spark.sql( + s"""call restore_to_instant( + | table => '$tableName', + | audit_only => true, + | start_restore_time => '$restoreInstantTs' + |)""".stripMargin + ).collect() + + assertResult(1)(auditRows.length) + assertResult(true)(auditRows(0).isNullAt(0)) // restore_result = null (no restore performed) + assertResult(true)(auditRows(0).isNullAt(1)) // start_restore_time = null + assertResult(true)(auditRows(0).isNullAt(2)) // time_taken_in_millis = null + assertResult(true)(auditRows(0).isNullAt(3)) // instants_rolled_back = null + assertResult(false)(auditRows(0).isNullAt(4)) // audit_result is present + assertResult("PASSED")(auditRows(0).getString(4)) + } + } + + test("Test restore_to_instant audit_only with non-existent restore instant throws") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + createTableAndInsertData(tableName, tablePath) + + // start_restore_time pointing at a timestamp that has never been a restore instant. + assertThrows[Exception] { + spark.sql( + s"""call restore_to_instant( + | table => '$tableName', + | audit_only => true, + | start_restore_time => '19700101000000000' + |)""".stripMargin + ).collect() + } + } + } + + test("Test restore_to_instant cross-validation: audit_only=true requires start_restore_time") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + createTableAndInsertData(tableName, tablePath) + + assertThrows[Exception] { + spark.sql( + s"call restore_to_instant(table => '$tableName', audit_only => true)" + ).collect() + } + } + } + + test("Test restore_to_instant cross-validation: audit_only=false rejects start_restore_time") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + + assertThrows[Exception] { + spark.sql( + s"""call restore_to_instant( + | table => '$tableName', + | instant_time => '${commits(1)}', + | start_restore_time => '20990101000000000' + |)""".stripMargin + ).collect() + } + } + } + + test("Test restore_to_instant cross-validation: audit_only=true rejects instant_time") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + val commits = createTableAndInsertData(tableName, tablePath) + + assertThrows[Exception] { + spark.sql( + s"""call restore_to_instant( + | table => '$tableName', + | audit_only => true, + | instant_time => '${commits(1)}', + | start_restore_time => '20990101000000000' + |)""".stripMargin + ).collect() + } + } + } + + test("Test restore_to_instant cross-validation: audit_only=false requires instant_time") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + "/" + tableName + createTableAndInsertData(tableName, tablePath) + + assertThrows[Exception] { + spark.sql(s"call restore_to_instant(table => '$tableName')").collect() + } + } + } +} From 8f74fdaf2f5245210e620bea62a95a8fc1dc5343 Mon Sep 17 00:00:00 2001 From: Mahsood Ebrahim Date: Tue, 19 May 2026 14:44:27 +0800 Subject: [PATCH 017/255] =?UTF-8?q?feat(spark):=20add=20show=5Finflight=5F?= =?UTF-8?q?commits=20and=20cleanup=5Fstale=5Finflight=5Fcom=E2=80=A6=20(#1?= =?UTF-8?q?8709)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(spark): add show_inflight_commits and cleanup_stale_inflight_commits stored procedures Two new CALL procedures so operators can inspect and remediate stale inflight commits via SQL instead of using hudi-cli. show_inflight_commits(table, min_age_minutes?) lists REQUESTED+INFLIGHT instants from the active timeline. cleanup_stale_inflight_commits(table, allowed_inflight_interval_minutes?, include_ingestion_commits?, dry_run?) rolls back stale write-timeline inflights older than the threshold (default 180 min). COMPACTION, LOG_COMPACTION, and CLUSTERING route to their dedicated table.rollbackInflight* methods (HoodieSparkTable is lazy-init on first such instant); other write actions go through client.rollback(). include_ingestion_commits and dry_run both default to false; dry_run emits rollback_status=NULL and skips write-client construction. A single-method utility HoodieTimelineCleanupUtil (inflightWriteCommitsOlderThan) is added in hudi-spark-common. Tested with 4 show + 9 cleanup unit tests covering empty/threshold/ ingestion-gating/dry_run paths plus COMPACTION, CLUSTERING, partitioned COW, and MOR delta_commit. checkstyle + scalastyle clean. * address review comments on show_inflight_commits / cleanup_stale_inflight_commits - Rename HoodieTimelineCleanupUtil.inflightWriteCommitsOlderThan param `mins` -> `ageMinutes` for clarity. - Replace Duration.ofMinutes(...).getSeconds() * 1000 with .toMillis() in HoodieTimelineCleanupUtil and ShowInflightCommitsProcedure. - Add inflight-state recheck in CleanupStaleInflightCommitsProcedure before client.rollback(): reloads the active timeline and skips the rollback if the instant is no longer INFLIGHT/REQUESTED, preventing destructive rollback of a commit that completed concurrently after detection. --------- Co-authored-by: mahsoode (cherry picked from commit d94b2e2c3162fefd26a7c86301228f6c2c74ceef) --- .../hudi/HoodieTimelineCleanupUtil.java | 59 +++ ...CleanupStaleInflightCommitsProcedure.scala | 224 ++++++++++ .../command/procedures/HoodieProcedures.scala | 2 + .../ShowInflightCommitsProcedure.scala | 113 +++++ ...CleanupStaleInflightCommitsProcedure.scala | 413 ++++++++++++++++++ .../TestShowInflightCommitsProcedure.scala | 151 +++++++ 6 files changed, 962 insertions(+) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java new file mode 100644 index 0000000000000..f8583d5d5d2a7 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi; + +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; +import org.apache.hudi.common.table.timeline.HoodieTimeline; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Date; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class HoodieTimelineCleanupUtil { + private static final Logger LOG = LoggerFactory.getLogger(HoodieTimelineCleanupUtil.class); + + public static List inflightWriteCommitsOlderThan(HoodieTableMetaClient metaClient, long ageMinutes, boolean includeIngestionCommits) { + long goBackMs = Duration.ofMinutes(ageMinutes).toMillis(); + String oldestAllowedTimestamp = HoodieInstantTimeGenerator.formatDate(new Date(System.currentTimeMillis() - goBackMs)); + + Stream inflightInstants = metaClient + .reloadActiveTimeline() + .getWriteTimeline() + .filterInflightsAndRequested() + .findInstantsBefore(oldestAllowedTimestamp) + .getInstants().stream(); + + if (!includeIngestionCommits) { + Predicate ingestionCommitsFilter = + (x) -> x.getAction().equals(HoodieTimeline.COMMIT_ACTION) || x.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION); + inflightInstants = inflightInstants.filter(ingestionCommitsFilter.negate()); + } + List inflightInstantsList = inflightInstants.collect(Collectors.toList()); + LOG.info("Inflight commits older than {} minutes: {}", ageMinutes, inflightInstantsList); + return inflightInstantsList; + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala new file mode 100644 index 0000000000000..66e31302e45bb --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.command.procedures + +import org.apache.hudi.{HoodieCLIUtils, HoodieTimelineCleanupUtil} +import org.apache.hudi.client.SparkRDDWriteClient +import org.apache.hudi.common.HoodiePendingRollbackInfo +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.timeline.HoodieTimeline +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.hadoop.fs.HadoopFSUtils +import org.apache.hudi.table.HoodieSparkTable + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} + +import java.util.function.Supplier + +import scala.collection.JavaConverters._ + +/** + * Spark SQL stored procedure to roll back stale inflight write commits older than a configurable + * age threshold. + * + * SAFETY WARNING: Unlike rollback_to_instant_time which targets a single named instant, this + * procedure rolls back a SET of instants matched by age. A misconfigured threshold can erase + * recent in-progress writes when include_ingestion_commits=true. Operators should call + * show_inflight_commits first to preview the pending instants, and use dry_run => true (see + * below) when in doubt about which instants will be processed. + * + * This procedure targets the write timeline (COMMIT, DELTA_COMMIT, COMPACTION, LOG_COMPACTION, + * REPLACE_COMMIT, CLUSTERING actions). Stale rollback, clean, or restore inflights visible in + * show_inflight_commits are NOT covered by this procedure. + * + * Compaction, log-compaction, and clustering inflights are handled via the targeted table + * methods (table.rollbackInflightCompaction / rollbackInflightLogCompaction / + * rollbackInflightClustering) — the supporting state (HoodieSparkTable, table-service client, + * pending-rollback lookup) is constructed lazily on the first such instant and reused. + * + * Parameters: + * - table: Required. Catalog name of the Hudi table. + * - allowed_inflight_interval_minutes: Optional (default 180). Instants older than this many + * minutes are considered stale and eligible for rollback. + * - include_ingestion_commits: Optional (default false). DANGEROUS when true: enabling + * this allows the procedure to roll back COMMIT_ACTION and + * DELTA_COMMIT_ACTION inflights, which means it can drop + * in-progress ingestion data. The default false is the + * safe choice; true is for operators recovering from a + * known-stuck ingestion job. + * - dry_run: Optional (default false). When true, the matched-instant + * set is resolved exactly as in normal mode but no rollback + * calls are issued. Each returned row carries + * rollback_status = NULL meaning "matched but not acted + * upon". Re-run with dry_run => false to act. + * + * Output columns (one row per processed instant): + * - instant_time: The instant's requested timestamp. + * - action: The action type. + * - rollback_status: true if the rollback succeeded, false if the rollback failed, + * NULL if dry_run was true (matched but not actioned). + * + * Example usage: + * {{{ + * -- Clean stale table-service inflights (default 180-min threshold) + * CALL cleanup_stale_inflight_commits(table => 'my_table'); + * + * -- Preview what would be processed without acting + * CALL cleanup_stale_inflight_commits(table => 'my_table', dry_run => true); + * + * -- Clean stale inflights older than 1 hour, including ingestion commits (DANGEROUS) + * CALL cleanup_stale_inflight_commits( + * table => 'my_table', + * allowed_inflight_interval_minutes => 60, + * include_ingestion_commits => true + * ); + * }}} + * + * For inflight commit types not covered by this procedure (clean, restore, rollback inflights), + * use hudi-cli's `repair rollback` command. + */ +class CleanupStaleInflightCommitsProcedure extends BaseProcedure with ProcedureBuilder with Logging { + + private val PARAMETERS = Array[ProcedureParameter]( + ProcedureParameter.required(0, "table", DataTypes.StringType), + ProcedureParameter.optional(1, "allowed_inflight_interval_minutes", DataTypes.IntegerType, 180), + ProcedureParameter.optional(2, "include_ingestion_commits", DataTypes.BooleanType, false), + ProcedureParameter.optional(3, "dry_run", DataTypes.BooleanType, false) + ) + + private val OUTPUT_TYPE = new StructType(Array[StructField]( + StructField("instant_time", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("action", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("rollback_status", DataTypes.BooleanType, nullable = true, Metadata.empty) + )) + + override def parameters: Array[ProcedureParameter] = PARAMETERS + + override def outputType: StructType = OUTPUT_TYPE + + override def build: Procedure = new CleanupStaleInflightCommitsProcedure() + + override def call(args: ProcedureArgs): Seq[Row] = { + super.checkArgs(PARAMETERS, args) + + val tableName = getArgValueOrDefault(args, PARAMETERS(0)) + val allowedMinutes = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Int] + val includeIngestionCommits = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Boolean] + val dryRun = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Boolean] + + val basePath = getBasePath(tableName) + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration)) + .setBasePath(basePath) + .build + + val staleInflights = HoodieTimelineCleanupUtil + .inflightWriteCommitsOlderThan(metaClient, allowedMinutes.toLong, includeIngestionCommits) + + if (staleInflights.isEmpty) { + Seq.empty[Row] + } else if (dryRun) { + // Dry-run: do not open the table for write — emit preview rows with NULL rollback_status + // to mean "matched but not actioned". Re-run with dry_run => false to act. + staleInflights.asScala.map { instant => + Row(instant.requestedTime, instant.getAction, null) + }.toSeq + } else { + // Pass ROLLBACK_USING_MARKERS_ENABLE=false via the createHoodieWriteClient confs Map. + // Inflight commits may not have marker files, so timeline-based rollback is required. + // The user-specified confs win over defaults / table config / session conf — see + // HoodieCLIUtils.scala "Priority: defaults < catalog props < table config < sparkSession conf < specified conf". + val confs = Map(HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.key() -> "false") + var client: SparkRDDWriteClient[_] = null + try { + client = HoodieCLIUtils.createHoodieWriteClient(sparkSession, basePath, confs, + tableName.asInstanceOf[scala.Option[String]]) + + // Lazy state for compaction / log-compaction / clustering branches. + // For matched sets that contain no such instants, none of these are constructed. + lazy val tsClient = client.getTableServiceClient + lazy val table = HoodieSparkTable.create(client.getConfig, client.getEngineContext) + lazy val getPendingRollbackInstantFunc: java.util.function.Function[String, HOption[HoodiePendingRollbackInfo]] = + new java.util.function.Function[String, HOption[HoodiePendingRollbackInfo]] { + override def apply(commitToRollback: String): HOption[HoodiePendingRollbackInfo] = { + tsClient.getPendingRollbackInfo(table.getMetaClient, commitToRollback, false) + } + } + + val rows = staleInflights.asScala.map { instant => + val status: java.lang.Boolean = try { + val result: Boolean = instant.getAction match { + case HoodieTimeline.COMPACTION_ACTION => + table.rollbackInflightCompaction(instant, getPendingRollbackInstantFunc, client.getTransactionManager) + true + case HoodieTimeline.LOG_COMPACTION_ACTION => + table.rollbackInflightLogCompaction(instant, getPendingRollbackInstantFunc, client.getTransactionManager) + true + case HoodieTimeline.CLUSTERING_ACTION => + table.rollbackInflightClustering(instant, getPendingRollbackInstantFunc, client.getTransactionManager) + true + case _ => + // Recheck that the instant is still inflight before calling client.rollback(), + // which searches getCommitsTimeline() (completed + pending). Without this guard, + // a concurrent writer that completes the commit after detection could have its + // now-completed commit rolled back destructively. The table.rollbackInflight* + // branches above fail loudly via revertInstantFromInflightToRequested if the + // inflight is gone; this branch performs an equivalent safety check. + val stillInflight = metaClient.reloadActiveTimeline() + .filterInflightsAndRequested() + .containsInstant(instant.requestedTime) + if (!stillInflight) { + logWarning(s"Instant ${instant.requestedTime} is no longer inflight; " + + "skipping rollback to avoid rolling back a completed commit") + false + } else { + client.rollback(instant.requestedTime) + } + } + java.lang.Boolean.valueOf(result) + } catch { + case e: Exception => + logError(s"Failed to rollback inflight instant ${instant.requestedTime}", e) + java.lang.Boolean.FALSE + } + Row(instant.requestedTime, instant.getAction, status) + }.toSeq + + // Refresh catalog after all rollbacks, inside try — consistent with RollbackToInstantTimeProcedure. + // Not placed in finally to avoid refreshing on client-creation failures. + if (tableName.isDefined) { + spark.catalog.refreshTable(tableName.get.asInstanceOf[String]) + } + rows + } finally { + if (client != null) client.close() + } + } + } +} + +object CleanupStaleInflightCommitsProcedure { + val NAME = "cleanup_stale_inflight_commits" + + def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] { + override def get() = new CleanupStaleInflightCommitsProcedure() + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala index 027c9f8c2882d..fd084ec0bf6e8 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala @@ -94,6 +94,8 @@ object HoodieProcedures { ,(ShowTablePropertiesProcedure.NAME, ShowTablePropertiesProcedure.builder) ,(HelpProcedure.NAME, HelpProcedure.builder) ,(ArchiveCommitsProcedure.NAME, ArchiveCommitsProcedure.builder) + ,(ShowInflightCommitsProcedure.NAME, ShowInflightCommitsProcedure.builder) + ,(CleanupStaleInflightCommitsProcedure.NAME, CleanupStaleInflightCommitsProcedure.builder) ,(RunTTLProcedure.NAME, RunTTLProcedure.builder) ,(DropPartitionProcedure.NAME, DropPartitionProcedure.builder) ,(TruncateTableProcedure.NAME, TruncateTableProcedure.builder) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala new file mode 100644 index 0000000000000..3b501bdb25a17 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.command.procedures + +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator +import org.apache.hudi.hadoop.fs.HadoopFSUtils + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} + +import java.time.Duration +import java.util.Date +import java.util.function.Supplier + +import scala.collection.JavaConverters._ + +/** + * Spark SQL stored procedure to list all pending inflight and requested instants on a Hudi table. + * + * Unlike cleanup_stale_inflight_commits (which targets the write timeline only), this procedure + * queries the full active timeline and therefore includes inflight rollback, clean, restore, and + * indexing instants in addition to write-action instants. A stale rollback or clean inflight + * visible here cannot be cleaned by cleanup_stale_inflight_commits. + * + * Parameters: + * - table: Required. Catalog name of the Hudi table. + * - min_age_minutes: Optional (default 0). When > 0, only instants older than this many minutes + * are returned. When 0 (default), all pending instants are returned. + * + * Output columns (one row per pending instant): + * - instant_time: The instant's requested timestamp. + * - action: The action type (commit, delta_commit, compaction, replace, rollback, clean, etc.). + * - state: The instant's state (REQUESTED or INFLIGHT). + * + * Example usage: + * {{{ + * -- Show all pending inflights + * CALL show_inflight_commits(table => 'my_table'); + * + * -- Show only inflights older than 2 hours + * CALL show_inflight_commits(table => 'my_table', min_age_minutes => 120); + * }}} + */ +class ShowInflightCommitsProcedure extends BaseProcedure with ProcedureBuilder { + + private val PARAMETERS = Array[ProcedureParameter]( + ProcedureParameter.required(0, "table", DataTypes.StringType), + ProcedureParameter.optional(1, "min_age_minutes", DataTypes.IntegerType, 0) + ) + + private val OUTPUT_TYPE = new StructType(Array[StructField]( + StructField("instant_time", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("action", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("state", DataTypes.StringType, nullable = true, Metadata.empty) + )) + + override def parameters: Array[ProcedureParameter] = PARAMETERS + + override def outputType: StructType = OUTPUT_TYPE + + override def build: Procedure = new ShowInflightCommitsProcedure() + + override def call(args: ProcedureArgs): Seq[Row] = { + super.checkArgs(PARAMETERS, args) + + val tableName = getArgValueOrDefault(args, PARAMETERS(0)) + val minAgeMinutes = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Int] + + val basePath = getBasePath(tableName) + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration)) + .setBasePath(basePath) + .build + + val baseTimeline = metaClient.reloadActiveTimeline().filterInflightsAndRequested() + + val timeline = if (minAgeMinutes > 0) { + val goBackMs = Duration.ofMinutes(minAgeMinutes).toMillis + val cutoff = HoodieInstantTimeGenerator.formatDate(new Date(System.currentTimeMillis() - goBackMs)) + baseTimeline.findInstantsBefore(cutoff) + } else { + baseTimeline + } + + timeline.getInstants.asScala.map { instant => + Row(instant.requestedTime, instant.getAction, instant.getState.name()) + }.toSeq + } +} + +object ShowInflightCommitsProcedure { + val NAME = "show_inflight_commits" + + def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] { + override def get() = new ShowInflightCommitsProcedure() + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala new file mode 100644 index 0000000000000..f18a173e29ad1 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.hadoop.fs.HadoopFSUtils + +import java.text.SimpleDateFormat +import java.util.Date + +import scala.collection.JavaConverters._ + +class TestCleanupStaleInflightCommitsProcedure extends HoodieSparkProcedureTestBase { + + /** + * Creates a table DDL without inserting data. Tests that manipulate inflight instants must NOT + * insert data before injecting the inflight, because BaseRollbackActionExecutor + * .validateRollbackCommitSequence throws HoodieRollbackException when committed instants exist + * after the injected (old) timestamp and no heartbeat exists for the injected instant. + * With no prior inserts, commitTimeline.empty() = true and the guard is bypassed. + */ + private def createEmptyTable(tableName: String, tablePath: String): Unit = { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | ts long + | ) using hudi + | location '$tablePath' + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | preCombineField = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + } + + private def createEmptyPartitionedTable(tableName: String, tablePath: String, tableType: String): Unit = { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | ts long, + | part int + | ) using hudi + | partitioned by (part) + | location '$tablePath' + | tblproperties ( + | primaryKey = 'id', + | type = '$tableType', + | preCombineField = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + } + + test("Test cleanup_stale_inflight_commits returns empty when no stale inflights exist") { + withTempDir { tmp => + val tableName = generateTableName + createEmptyTable(tableName, tmp.getCanonicalPath) + spark.sql(s"insert into $tableName values(1, 'a1', 1000)") + spark.sql(s"insert into $tableName values(2, 'a2', 2000)") + + val result = spark.sql(s"call cleanup_stale_inflight_commits(table => '$tableName')").collect() + assertResult(0)(result.length) + } + } + + test("Test cleanup_stale_inflight_commits rolls back stale REPLACE_COMMIT_ACTION inflight") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + // No inserts before injecting — see createEmptyTable docstring + createEmptyTable(tableName, tablePath) + + val staleTs = "20200101120000" + // Must use REPLACE_COMMIT_ACTION: inflightWriteCommitsOlderThan with + // include_ingestion_commits=false (default) filters out COMMIT_ACTION and DELTA_COMMIT_ACTION. + // REPLACE_COMMIT_ACTION is included in both getWriteTimeline() and getCommitsTimeline(), + // so client.rollback() finds it and returns true. + injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs) + + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1)) + assertResult(true)(result(0).getBoolean(2)) + + // Verify the instant is gone from the active timeline + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val remaining = metaClient.reloadActiveTimeline().filterInflightsAndRequested() + .getInstants.asScala + assert(!remaining.exists(_.requestedTime == staleTs), + s"Stale instant $staleTs should have been removed from the timeline after rollback") + } + } + + test("Test cleanup_stale_inflight_commits respects allowed_inflight_interval_minutes threshold") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + // No inserts before injecting — see createEmptyTable docstring + createEmptyTable(tableName, tablePath) + + val staleTs = "20200101120000" + val freshTs = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + + injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs) + injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, freshTs) + + // 60-minute threshold: only the stale instant qualifies; fresh instant is too recent + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 60)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + } + } + + test("Test cleanup_stale_inflight_commits cleans COMMIT_ACTION with include_ingestion_commits=true") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + // No inserts before injecting — see createEmptyTable docstring + createEmptyTable(tableName, tablePath) + + val staleTs = "20200101120000" + // COMMIT_ACTION is filtered out with the default include_ingestion_commits=false, + // but included when include_ingestion_commits=true. + // client.rollback returns true for COMMIT_ACTION since getCommitsTimeline() includes it. + injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, staleTs) + + // Default (include_ingestion_commits=false): should not find COMMIT_ACTION inflight + val defaultResult = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1)").collect() + assertResult(0)(defaultResult.length) + + // With include_ingestion_commits=true: should find and process COMMIT_ACTION inflight + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1, " + + s"include_ingestion_commits => true)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + assertResult(HoodieTimeline.COMMIT_ACTION)(result(0).getString(1)) + assertResult(true)(result(0).getBoolean(2)) + } + } + + test("Test cleanup_stale_inflight_commits dry_run lists matched instants without rolling back") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + createEmptyTable(tableName, tablePath) + + val staleTs = "20200101120000" + injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs) + + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1, " + + s"dry_run => true)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1)) + // dry_run: rollback_status is NULL meaning "matched but not actioned" + assert(result(0).isNullAt(2), + "Expected rollback_status=NULL in dry_run mode, but got non-null value") + + // Verify the instant is STILL on the active timeline (dry_run did not act) + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val remaining = metaClient.reloadActiveTimeline().filterInflightsAndRequested() + .getInstants.asScala + assert(remaining.exists(_.requestedTime == staleTs), + s"dry_run should not have rolled back $staleTs; expected to find it still on the timeline") + } + } + + test("Test cleanup_stale_inflight_commits rolls back stale COMPACTION_ACTION inflight via table.rollbackInflightCompaction") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + // Compaction is MOR-only. Use the real schedule + run + delete-commit pattern from + // TestRunRollbackInflightTableServiceProcedure to construct a valid compaction inflight. + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | tblproperties ( + | primaryKey = 'id', + | type = 'mor', + | preCombineField = 'ts' + | ) + | partitioned by(ts) + | location '$tablePath' + """.stripMargin) + withSQLConf( + "hoodie.parquet.max.file.size" -> "10000", + "hoodie.compact.inline" -> "false", + "hoodie.compact.schedule.inline" -> "false", + // Prevent auto-clean from creating a clean instant after compaction completion, + // which would shift getReverseOrderedInstants.findFirst() away from the compaction commit. + "hoodie.clean.automatic" -> "false" + ) { + spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)") + spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000)") + spark.sql(s"insert into $tableName values(3, 'a3', 10, 1000)") + spark.sql(s"insert into $tableName values(4, 'a4', 10, 1000)") + spark.sql(s"update $tableName set price = 11 where id = 1") + + spark.sql(s"call run_compaction(op => 'schedule', table => '$tableName')") + spark.sql(s"call run_compaction(op => 'run', table => '$tableName')") + + // Delete the completed compaction commit file so the inflight remains + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val compactionInstant = metaClient.getActiveTimeline.getReverseOrderedInstants.findFirst().get() + metaClient.getActiveTimeline.deleteInstantFileIfExists(compactionInstant) + val compactionInstantTime = compactionInstant.requestedTime + + // Confirm the compaction inflight is actually present before we call cleanup. + // If this assertion fires, the test setup (schedule+run+delete) didn't produce the expected + // state and the rest of the test is moot — fail with a clear diagnostic instead of an empty result. + val reloadedTimeline = metaClient.reloadActiveTimeline() + val compactionInflightPresent = reloadedTimeline.getWriteTimeline.filterInflightsAndRequested.getInstants.asScala + .exists(i => i.getAction == HoodieTimeline.COMPACTION_ACTION && i.requestedTime == compactionInstantTime) + assert(compactionInflightPresent, + s"Setup failure: compaction inflight at $compactionInstantTime not present after deleting completed commit. " + + s"Active timeline: ${reloadedTimeline.getInstants.asScala.map(i => s"${i.requestedTime}/${i.getAction}/${i.getState}").mkString(", ")}") + + // Sleep so the second-precision cutoff timestamp is strictly newer than the inflight's timestamp + Thread.sleep(2000) + + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 0)").collect() + + val compactionRow = result.find(r => r.getString(0) == compactionInstantTime) + assert(compactionRow.isDefined, + s"Expected compaction inflight $compactionInstantTime in result; got ${result.map(r => s"${r.getString(0)}/${r.getString(1)}").mkString(",")}") + assertResult(HoodieTimeline.COMPACTION_ACTION)(compactionRow.get.getString(1)) + assertResult(true)(compactionRow.get.getBoolean(2)) + + // Inflight should be removed by table.rollbackInflightCompaction + val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator + val expectedInflight = instantGenerator.createNewInstant( + HoodieInstant.State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, compactionInstantTime) + assert(!metaClient.reloadActiveTimeline().getInstants.contains(expectedInflight), + s"Compaction inflight $compactionInstantTime should be gone after rollback") + } + } + } + + test("Test cleanup_stale_inflight_commits rolls back stale CLUSTERING_ACTION inflight via table.rollbackInflightClustering") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + // Use the real schedule + execute + delete-commit pattern from + // TestRunRollbackInflightTableServiceProcedure so the clustering inflight is valid. + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | preCombineField = 'ts' + | ) + | partitioned by(ts) + | location '$tablePath' + """.stripMargin) + spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)") + spark.sql(s"insert into $tableName values(2, 'a2', 10, 1001)") + spark.sql(s"insert into $tableName values(3, 'a3', 10, 1002)") + + spark.sql(s"call run_clustering(table => '$tableName', op => 'schedule')") + spark.sql(s"call run_clustering(table => '$tableName', op => 'execute')") + + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val clusteringInstant = metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.get(0) + metaClient.getActiveTimeline.deleteInstantFileIfExists(clusteringInstant) + val clusteringInstantTime = clusteringInstant.requestedTime + + Thread.sleep(2000) + + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 0)").collect() + + val clusteringRow = result.find(r => r.getString(0) == clusteringInstantTime) + assert(clusteringRow.isDefined, + s"Expected clustering inflight $clusteringInstantTime in result; got ${result.map(_.getString(0)).mkString(",")}") + assertResult(HoodieTimeline.CLUSTERING_ACTION)(clusteringRow.get.getString(1)) + assertResult(true)(clusteringRow.get.getBoolean(2)) + + val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator + val expectedInflight = instantGenerator.createNewInstant( + HoodieInstant.State.INFLIGHT, HoodieTimeline.CLUSTERING_ACTION, clusteringInstantTime) + assert(!metaClient.reloadActiveTimeline().getInstants.contains(expectedInflight), + s"Clustering inflight $clusteringInstantTime should be gone after rollback") + } + } + + test("Test cleanup_stale_inflight_commits handles partitioned COW table") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + createEmptyPartitionedTable(tableName, tablePath, "cow") + + val staleTs = "20200101120000" + injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs) + + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1)) + assertResult(true)(result(0).getBoolean(2)) + } + } + + test("Test cleanup_stale_inflight_commits handles MOR table DELTA_COMMIT_ACTION with include_ingestion_commits") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + createEmptyPartitionedTable(tableName, tablePath, "mor") + + val staleTs = "20200101120000" + injectInflightInstant(tablePath, HoodieTimeline.DELTA_COMMIT_ACTION, staleTs) + + // Default (include_ingestion_commits=false): DELTA_COMMIT_ACTION is filtered out + val defaultResult = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1)").collect() + assertResult(0)(defaultResult.length) + + // With include_ingestion_commits=true: DELTA_COMMIT_ACTION is processed + val result = spark.sql( + s"call cleanup_stale_inflight_commits(table => '$tableName', " + + s"allowed_inflight_interval_minutes => 1, " + + s"include_ingestion_commits => true)").collect() + + assertResult(1)(result.length) + assertResult(staleTs)(result(0).getString(0)) + assertResult(HoodieTimeline.DELTA_COMMIT_ACTION)(result(0).getString(1)) + assertResult(true)(result(0).getBoolean(2)) + } + } + + /** + * Injects a REQUESTED→INFLIGHT instant into the active timeline without completing it. + * Used to simulate stale inflight operations for testing. + */ + private def injectInflightInstant(tablePath: String, action: String, instantTime: String): Unit = { + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val timeline = metaClient.getActiveTimeline + val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator + val requested = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, action, instantTime) + timeline.createNewInstant(requested) + timeline.transitionRequestedToInflight(requested, HOption.empty[Array[Byte]]()) + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala new file mode 100644 index 0000000000000..ca77172bd607b --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.hadoop.fs.HadoopFSUtils + +import java.text.SimpleDateFormat +import java.util.Date + +class TestShowInflightCommitsProcedure extends HoodieSparkProcedureTestBase { + + test("Test show_inflight_commits returns empty for a fully committed table") { + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | ts long + | ) using hudi + | location '${tmp.getCanonicalPath}' + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | preCombineField = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + spark.sql(s"insert into $tableName values(1, 'a1', 1000)") + spark.sql(s"insert into $tableName values(2, 'a2', 2000)") + + val result = spark.sql(s"call show_inflight_commits(table => '$tableName')").collect() + assertResult(0)(result.length) + } + } + + test("Test show_inflight_commits returns injected inflight instant") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | ts long + | ) using hudi + | location '$tablePath' + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | preCombineField = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + + val injectedTs = "20200101120000" + injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, injectedTs) + + val result = spark.sql(s"call show_inflight_commits(table => '$tableName')").collect() + assert(result.length >= 1) + val row = result.find(r => r.getString(0) == injectedTs) + assert(row.isDefined, s"Expected inflight instant $injectedTs not found in results") + assertResult(HoodieTimeline.COMMIT_ACTION)(row.get.getString(1)) + assertResult("INFLIGHT")(row.get.getString(2)) + } + } + + test("Test show_inflight_commits min_age_minutes filter includes old and excludes recent") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = tmp.getCanonicalPath + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | ts long + | ) using hudi + | location '$tablePath' + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | preCombineField = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + + val oldTs = "20200101120000" + val freshTs = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + + injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, oldTs) + injectInflightInstant(tablePath, HoodieTimeline.DELTA_COMMIT_ACTION, freshTs) + + // No filter: both should appear + val allResults = spark.sql(s"call show_inflight_commits(table => '$tableName', min_age_minutes => 0)").collect() + assert(allResults.length >= 2) + assert(allResults.exists(r => r.getString(0) == oldTs)) + assert(allResults.exists(r => r.getString(0) == freshTs)) + + // 60-minute filter: only the old instant should appear + val filteredResults = spark.sql( + s"call show_inflight_commits(table => '$tableName', min_age_minutes => 60)").collect() + assert(filteredResults.exists(r => r.getString(0) == oldTs), + s"Expected old instant $oldTs to appear with min_age_minutes=60") + assert(!filteredResults.exists(r => r.getString(0) == freshTs), + s"Fresh instant $freshTs should not appear with min_age_minutes=60") + } + } + + test("Test show_inflight_commits requires table parameter") { + checkExceptionContain( + "call show_inflight_commits()")( + "Argument: table is required") + } + + /** + * Injects a REQUESTED→INFLIGHT instant into the active timeline without completing it. + * Used to simulate stale inflight operations for testing. + */ + private def injectInflightInstant(tablePath: String, action: String, instantTime: String): Unit = { + val metaClient = HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + val timeline = metaClient.getActiveTimeline + val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator + val requested = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, action, instantTime) + timeline.createNewInstant(requested) + timeline.transitionRequestedToInflight(requested, HOption.empty[Array[Byte]]()) + } +} From 88b4e85809b115dc714c4bab0f654d0b252702be Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 20 May 2026 20:14:13 -0700 Subject: [PATCH 018/255] chore: add spark4.1 and flink2.1 profile entries to RC bundle validation (#18796) (cherry picked from commit a687786833a616cb0f646d96726d4ab231f64ecc) --- .github/workflows/release_candidate_validation.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml index cde2891ee3a2b..ff4605c024751 100644 --- a/.github/workflows/release_candidate_validation.yml +++ b/.github/workflows/release_candidate_validation.yml @@ -77,6 +77,10 @@ jobs: flinkProfile: 'flink1.20' sparkProfile: 'spark4.0' sparkRuntime: 'spark4.0.0' + - scalaProfile: 'scala-2.13' + flinkProfile: 'flink1.20' + sparkProfile: 'spark4.1' + sparkRuntime: 'spark4.1.1' steps: - uses: actions/checkout@v5 - name: Set up JDK 17 @@ -108,6 +112,10 @@ jobs: flinkProfile: 'flink2.0' sparkProfile: 'spark3.5' sparkRuntime: 'spark3.5.1' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink2.1' + sparkProfile: 'spark3.5' + sparkRuntime: 'spark3.5.1' steps: - uses: actions/checkout@v5 - name: Set up JDK 11 From 18cddb5b6d866145bd5ee150e49187cb4b7e2938 Mon Sep 17 00:00:00 2001 From: fhan Date: Thu, 21 May 2026 16:32:51 +0800 Subject: [PATCH 019/255] fix(flink): fix disable table service not effective in hudi-flink (#13875) make the HoodieWriteConfig.TABLE_SERVICES_ENABLED effective for Flink. --------- Co-authored-by: fhan Co-authored-by: danny0405 (cherry picked from commit 50eb95c7d6f5da986244e53627e54a703f6f8678) --- .../hudi/configuration/FlinkOptions.java | 6 ++ .../hudi/configuration/OptionsResolver.java | 36 ++++++++-- .../hudi/sink/v2/utils/PipelinesV2.java | 8 ++- .../hudi/streamer/HoodieFlinkStreamer.java | 6 +- .../apache/hudi/table/HoodieTableSink.java | 6 +- .../configuration/TestOptionsResolver.java | 68 +++++++++++++++++++ .../hudi/sink/TestWriteCopyOnWrite.java | 29 ++++++++ .../sink/TestWriteMergeOnReadWithCompact.java | 31 +++++++++ 8 files changed, 178 insertions(+), 12 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java index 4910c721e09cc..fbe29f93ef5b4 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java @@ -188,6 +188,12 @@ public class FlinkOptions extends HoodieConfig { .defaultValue(false) // keep sync with hoodie style .withDescription("If enabled, the checkpoint Id will also be written to hudi metadata."); + public static final ConfigOption TABLE_SERVICES_ENABLED = ConfigOptions + .key(HoodieWriteConfig.TABLE_SERVICES_ENABLED.key()) + .booleanType() + .defaultValue(HoodieWriteConfig.TABLE_SERVICES_ENABLED.defaultValue()) + .withDescription("Master control to disable all table services including archive, clean, compact, cluster, etc."); + // ------------------------------------------------------------------------ // Changelog Capture Options // ------------------------------------------------------------------------ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java index 628474bde7d62..74c5257cf44fb 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java @@ -292,7 +292,7 @@ public static boolean emitDeletes(Configuration conf) { * @param conf The flink configuration. */ public static boolean needsAsyncCompaction(Configuration conf) { - return OptionsResolver.isMorTable(conf) && conf.get(FlinkOptions.COMPACTION_ASYNC_ENABLED); + return OptionsResolver.isMorTable(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.COMPACTION_ASYNC_ENABLED); } /** @@ -301,7 +301,7 @@ public static boolean needsAsyncCompaction(Configuration conf) { * @param conf The flink configuration. */ public static boolean needsAsyncMetadataCompaction(Configuration conf) { - return isStreamingIndexWriteEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_ASYNC_ENABLED); + return isStreamingIndexWriteEnabled(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_ASYNC_ENABLED); } /** @@ -310,7 +310,7 @@ public static boolean needsAsyncMetadataCompaction(Configuration conf) { * @param conf The flink configuration. */ public static boolean needsScheduleMdtCompaction(Configuration conf) { - return isStreamingIndexWriteEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_SCHEDULE_ENABLED); + return isStreamingIndexWriteEnabled(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_SCHEDULE_ENABLED); } /** @@ -320,7 +320,9 @@ public static boolean needsScheduleMdtCompaction(Configuration conf) { */ public static boolean needsScheduleCompaction(Configuration conf) { return OptionsResolver.isMorTable(conf) - && conf.get(FlinkOptions.COMPACTION_SCHEDULE_ENABLED) && !isAppendMode(conf); + && areTableServicesEnabled(conf) + && conf.get(FlinkOptions.COMPACTION_SCHEDULE_ENABLED) + && !isAppendMode(conf); } /** @@ -329,7 +331,7 @@ public static boolean needsScheduleCompaction(Configuration conf) { * @param conf The flink configuration. */ public static boolean needsAsyncClustering(Configuration conf) { - return isInsertOperation(conf) && conf.get(FlinkOptions.CLUSTERING_ASYNC_ENABLED); + return isInsertOperation(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.CLUSTERING_ASYNC_ENABLED); } /** @@ -338,6 +340,9 @@ public static boolean needsAsyncClustering(Configuration conf) { * @param conf The flink configuration. */ public static boolean needsScheduleClustering(Configuration conf) { + if (!areTableServicesEnabled(conf)) { + return false; + } if (!conf.get(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED)) { return false; } @@ -544,6 +549,20 @@ public static boolean isNonBlockingConcurrencyControl(Configuration config) { return WriteConcurrencyMode.isNonBlockingConcurrencyControl(config.getString(HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key(), HoodieWriteConfig.WRITE_CONCURRENCY_MODE.defaultValue())); } + /** + * Returns whether the cleaning for failed writes is enabled as lazy. + */ + public static boolean isLazyFailedWritesCleaning(Configuration conf) { + return needsAsyncCleaning(conf) && isLazyFailedWritesCleanPolicy(conf); + } + + /** + * Returns whether there is need for async cleaning (planning & execution). + */ + public static boolean needsAsyncCleaning(Configuration conf) { + return areTableServicesEnabled(conf); + } + /** * Returns whether Cleaner's failed writes policy is set to lazy */ @@ -574,6 +593,13 @@ public static boolean isBlockingInstantGeneration(Configuration conf) { return (isCowTable(conf) || conf.get(FlinkOptions.CDC_ENABLED)) && isUpsertOperation(conf); } + /** + * Returns whether table services are enabled. + */ + public static boolean areTableServicesEnabled(Configuration conf) { + return conf.get(FlinkOptions.TABLE_SERVICES_ENABLED); + } + /** * Returns the customized insert partitioner instance. */ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java index 6e93c02bc137e..107e8673c42b7 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java @@ -119,7 +119,7 @@ public static DataStream composePipeline( DataStream pipeline = Pipelines.append(conf, rowType, dataStream); if (OptionsResolver.needsAsyncClustering(conf)) { return clusterV2(conf, rowType, pipeline); - } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) { + } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) { // add clean function to rollback failed writes for lazy failed writes cleaning policy return cleanV2(conf, pipeline); } else { @@ -138,8 +138,10 @@ public static DataStream composePipeline( conf.set(FlinkOptions.COMPACTION_OPERATION_EXECUTE_ASYNC_ENABLED, false); } return compactV2(conf, pipeline); - } else { + } else if (OptionsResolver.needsAsyncCleaning(conf)) { return cleanV2(conf, pipeline); + } else { + return pipeline; } } @@ -156,7 +158,7 @@ private static int getParallelismForSinkV2(Configuration conf) { if (OptionsResolver.isBulkInsertOperation(conf)) { return conf.get(FlinkOptions.WRITE_TASKS); } else if (OptionsResolver.isAppendMode(conf)) { - return OptionsResolver.needsAsyncClustering(conf) || OptionsResolver.isLazyFailedWritesCleanPolicy(conf) + return OptionsResolver.needsAsyncClustering(conf) || OptionsResolver.isLazyFailedWritesCleaning(conf) ? 1 : conf.get(FlinkOptions.WRITE_TASKS); } else { return 1; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java index 721b3f94e470b..58be515ca9015 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java @@ -97,7 +97,7 @@ public static void main(String[] args) throws Exception { pipeline = Pipelines.append(conf, rowType, dataStream); if (OptionsResolver.needsAsyncClustering(conf)) { Pipelines.cluster(conf, rowType, pipeline); - } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) { + } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) { // add clean function to rollback failed writes for lazy failed writes cleaning policy Pipelines.clean(conf, pipeline); } else { @@ -108,8 +108,10 @@ public static void main(String[] args) throws Exception { pipeline = Pipelines.hoodieStreamWrite(conf, rowType, hoodieRecordDataStream); if (OptionsResolver.needsAsyncCompaction(conf)) { Pipelines.compact(conf, pipeline); - } else { + } else if (OptionsResolver.needsAsyncCleaning(conf)) { Pipelines.clean(conf, pipeline); + } else { + Pipelines.dummySink(pipeline); } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java index f46e2e672222c..45bfde998af64 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java @@ -112,7 +112,7 @@ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { DataStream pipeline = Pipelines.append(conf, rowType, dataStream); if (OptionsResolver.needsAsyncClustering(conf)) { return Pipelines.cluster(conf, rowType, pipeline); - } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) { + } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) { // add clean function to rollback failed writes for lazy failed writes cleaning policy return Pipelines.clean(conf, pipeline); } else { @@ -131,8 +131,10 @@ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { conf.set(FlinkOptions.COMPACTION_OPERATION_EXECUTE_ASYNC_ENABLED, false); } return Pipelines.compact(conf, pipeline); - } else { + } else if (OptionsResolver.needsAsyncCleaning(conf)) { return Pipelines.clean(conf, pipeline); + } else { + return Pipelines.dummySink(pipeline); } }; } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java index 3d9a6bba96605..aef93b7d10fba 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java @@ -19,6 +19,7 @@ package org.apache.hudi.configuration; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieCleanConfig; @@ -123,4 +124,71 @@ private Configuration getConf() { conf.set(FlinkOptions.PATH, tempFile.getAbsolutePath()); return conf; } + + @Test + void testAreTableServicesEnabled() { + Configuration conf = new Configuration(); + // default value should be true + assertTrue(OptionsResolver.areTableServicesEnabled(conf)); + + // explicitly set to true + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, true); + assertTrue(OptionsResolver.areTableServicesEnabled(conf)); + + // explicitly set to false + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + assertFalse(OptionsResolver.areTableServicesEnabled(conf)); + } + + @Test + void testTableServicesGateCompactionAndCleaning() { + Configuration conf = getConf(); + conf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); + conf.setString(HoodieCleanConfig.FAILED_WRITES_CLEANER_POLICY.key(), HoodieFailedWritesCleaningPolicy.LAZY.name()); + + assertTrue(OptionsResolver.needsAsyncCompaction(conf)); + assertTrue(OptionsResolver.needsScheduleCompaction(conf)); + assertTrue(OptionsResolver.needsAsyncCleaning(conf)); + assertTrue(OptionsResolver.isLazyFailedWritesCleanPolicy(conf)); + assertTrue(OptionsResolver.isLazyFailedWritesCleaning(conf)); + + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + + assertFalse(OptionsResolver.needsAsyncCompaction(conf)); + assertFalse(OptionsResolver.needsScheduleCompaction(conf)); + assertFalse(OptionsResolver.needsAsyncCleaning(conf)); + assertTrue(OptionsResolver.isLazyFailedWritesCleanPolicy(conf)); + assertFalse(OptionsResolver.isLazyFailedWritesCleaning(conf)); + } + + @Test + void testTableServicesGateMetadataCompaction() { + Configuration conf = getConf(); + conf.set(FlinkOptions.METADATA_ENABLED, true); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.RECORD_LEVEL_INDEX.name()); + + assertTrue(OptionsResolver.needsAsyncMetadataCompaction(conf)); + assertTrue(OptionsResolver.needsScheduleMdtCompaction(conf)); + + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + + assertFalse(OptionsResolver.needsAsyncMetadataCompaction(conf)); + assertFalse(OptionsResolver.needsScheduleMdtCompaction(conf)); + } + + @Test + void testTableServicesGateClustering() { + Configuration conf = getConf(); + conf.set(FlinkOptions.OPERATION, WriteOperationType.INSERT.value()); + conf.set(FlinkOptions.CLUSTERING_ASYNC_ENABLED, true); + conf.set(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED, true); + + assertTrue(OptionsResolver.needsAsyncClustering(conf)); + assertTrue(OptionsResolver.needsScheduleClustering(conf)); + + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + + assertFalse(OptionsResolver.needsAsyncClustering(conf)); + assertFalse(OptionsResolver.needsScheduleClustering(conf)); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java index a4e869428c243..69087122a2e81 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java @@ -359,6 +359,35 @@ public void testInsertDuplicates() throws Exception { .end(); } + @Test + public void testInsertWithTableServiceDisabled() throws Exception { + // reset the config option + conf.set(FlinkOptions.OPERATION, "insert"); + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + conf.set(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED, true); + conf.set(FlinkOptions.CLUSTERING_ASYNC_ENABLED, true); + conf.set(FlinkOptions.CLUSTERING_DELTA_COMMITS, 1); + + preparePipeline(conf) + .consume(TestData.DATA_SET_INSERT_SAME_KEY) + .checkpoint(1) + .handleEvents(1) + .checkpointComplete(1) + .checkWrittenData(EXPECTED4, 1) + // insert duplicates again + .consume(TestData.DATA_SET_INSERT_SAME_KEY) + .checkpoint(2) + .handleEvents(1) + .checkpointComplete(2) + .checkWrittenDataCOW(EXPECTED5) + .end(); + HoodieFlinkWriteClient writeClient = FlinkWriteClients.createWriteClient(conf); + long completedReplaceCommit = writeClient.getHoodieTable().getActiveTimeline().getCompletedReplaceTimeline().getInstants().stream().count(); + long pendingReplaceCommit = writeClient.getHoodieTable().getActiveTimeline().filterPendingReplaceTimeline().getInstants().stream().count(); + assertEquals(0, completedReplaceCommit); + assertEquals(0, pendingReplaceCommit); + } + @Test public void testUpsert() throws Exception { // open the function and ingest data diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java index 7515f89043272..3f24c2763faf1 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.PartialUpdateAvroPayload; import org.apache.hudi.common.model.WriteConcurrencyMode; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.FlinkOptions; @@ -42,7 +43,9 @@ import java.util.List; import java.util.Map; +import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMPACTION_ACTION; import static org.apache.hudi.utils.TestData.insertRow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; /** @@ -56,6 +59,34 @@ protected void setUp(Configuration conf) { conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1); } + @Test + public void testUpsertWithTableServiceDisabled() throws Exception { + // reset the config option + conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false); + conf.set(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, true); + conf.set(FlinkOptions.COMPACTION_ASYNC_ENABLED, true); + conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1); + + preparePipeline(conf) + .consume(TestData.DATA_SET_INSERT) + .assertEmptyDataFiles() + .checkpoint(1) + .handleEvents(1) + .checkpointComplete(1) + .consume(TestData.DATA_SET_INSERT) + .checkpoint(2) + .handleEvents(1) + .checkpointComplete(2) + .end(); + HoodieFlinkWriteClient writeClient = FlinkWriteClients.createWriteClient(conf); + long completedCompaction = writeClient.getHoodieTable().getActiveTimeline().getInstants().stream() + .filter(s -> s.getAction().equals(COMPACTION_ACTION)) + .filter(HoodieInstant::isCompleted).count(); + long pendingCompaction = writeClient.getHoodieTable().getActiveTimeline().filterPendingCompactionTimeline().getInstants().stream().count(); + assertEquals(0, completedCompaction); + assertEquals(0, pendingCompaction); + } + @Test public void testPartialFailover() { // partial failover is only valid for append mode. From a15bc7bd882775d8316e16ccdb9de18f0d7bfe47 Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Mon, 25 May 2026 19:23:15 -0700 Subject: [PATCH 020/255] feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (#18809) * feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (cherry picked from commit 06dc09b1fedd6ff5640c95f28c8136e8a78bce98) --- .../format/cow/utils/NestedPositionUtil.java | 4 +- .../ParquetDataColumnReaderFactory.java | 8 +- .../format/cow/ParquetSplitReaderUtil.java | 688 +++++++++++------- .../format/cow/utils/BooleanArrayList.java | 69 ++ .../table/format/cow/utils/IntArrayList.java | 93 +++ .../table/format/cow/utils/LongArrayList.java | 83 +++ .../format/cow/utils/NestedPositionUtil.java | 209 ++++++ .../cow/vector/ColumnarGroupArrayData.java | 179 ----- .../cow/vector/ColumnarGroupMapData.java | 63 -- .../cow/vector/ColumnarGroupRowData.java | 138 ---- .../vector/HeapArrayGroupColumnVector.java | 53 -- .../format/cow/vector/HeapArrayVector.java | 31 + .../cow/vector/HeapMapColumnVector.java | 75 +- .../cow/vector/HeapRowColumnVector.java | 15 + .../cow/vector/ParquetDecimalVector.java | 197 ++++- .../vector/position/CollectionPosition.java | 57 ++ .../cow/vector/position/LevelDelegation.java | 43 ++ .../cow/vector/position/RowPosition.java | 45 ++ .../cow/vector/reader/ArrayColumnReader.java | 473 ------------ .../cow/vector/reader/ArrayGroupReader.java | 45 -- .../cow/vector/reader/MapColumnReader.java | 56 -- .../cow/vector/reader/NestedColumnReader.java | 278 +++++++ .../reader/NestedPrimitiveColumnReader.java | 639 ++++++++++++++++ .../reader/ParquetColumnarRowSplitReader.java | 34 +- .../ParquetDataColumnReaderFactory.java | 112 ++- .../cow/vector/reader/RowColumnReader.java | 63 -- .../format/cow/vector/type/ParquetField.java | 72 ++ .../cow/vector/type/ParquetGroupField.java | 59 ++ .../vector/type/ParquetPrimitiveField.java | 55 ++ .../vector/TestHeapColumnVectorAccessors.java | 139 ++++ .../cow/vector/TestParquetDecimalVector.java | 188 +++++ .../TestParquetDataColumnReaderFactory.java | 272 +++++++ .../vector/type/TestParquetGroupField.java | 134 ++++ 33 files changed, 3285 insertions(+), 1384 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java index a1fa3c8fa8357..3f2f8976b69bf 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java @@ -87,8 +87,8 @@ public static RowPosition calculateRowOffsets( * * @param field field that contains array/map column message include max repetition level and * definition level. - * @param definitionLevels int array with each value's repetition level. - * @param repetitionLevels int array with each value's definition level. + * @param definitionLevels int array with each value's definition level. + * @param repetitionLevels int array with each value's repetition level. * @return {@link CollectionPosition} contains collections offset array, length array and isNull * array. */ diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index 3748e53fc2843..1abc6ed56c0db 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -359,9 +359,9 @@ private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { } private static TimestampData int64ToTimestamp( - boolean utcTimestamp, long value, ChronoUnit unit) { + boolean isUtcTimestamp, long value, ChronoUnit unit) { Instant instant = Instant.EPOCH.plus(value, unit); - if (utcTimestamp) { + if (isUtcTimestamp) { return TimestampData.fromInstant(instant); } return TimestampData.fromTimestamp(Timestamp.from(instant)); @@ -379,10 +379,10 @@ public static ParquetDataColumnReader getDataColumnReaderByType( } private static TimestampData int96ToTimestamp( - boolean utcTimestamp, long nanosOfDay, int julianDay) { + boolean isUtcTimestamp, long nanosOfDay, int julianDay) { long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND); - if (utcTimestamp) { + if (isUtcTimestamp) { int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND); return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond); } else { diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index ab05beb87459d..5468dc86a25a6 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,19 +19,18 @@ package org.apache.hudi.table.format.cow; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapDecimalVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; -import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader; import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader; import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader; import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader; +import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader; import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader; -import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader; @@ -64,12 +63,14 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; @@ -77,12 +78,18 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.io.ColumnIO; +import org.apache.parquet.io.GroupColumnIO; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.sql.Date; @@ -90,25 +97,38 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.flink.table.utils.DateTimeUtils.toInternal; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** * Util for generating {@link ParquetColumnarRowSplitReader}. * - *

NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64 - * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that. + *

Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive + * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader}, + * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For + * nested types (ARRAY / MAP / MULTISET / ROW) we build a {@link ParquetField} tree once per + * split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to + * {@link NestedColumnReader}. + * + *

Schema evolution: missing top-level fields are still handled by the caller + * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields + * inside a Row are handled here — {@link #constructField} returns {@code null} for a + * child that isn't physically present, and the corresponding child in the pre-allocated vector + * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can + * passthrough the slot (see {@link NestedColumnReader#readToVector}). */ public class ParquetSplitReaderUtil { - /** - * Util for generating partitioned {@link ParquetColumnarRowSplitReader}. - */ + /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( boolean utcTimestamp, boolean caseSensitive, @@ -125,7 +145,7 @@ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( UnboundRecordFilter recordFilter) throws IOException { ValidationUtils.checkState(Arrays.stream(selectedFields).noneMatch(x -> x == -1), - "One or more specified columns does not exist in the hudi table."); + "One or more specified columns does not exist in the hudi table."); List selNonPartNames = Arrays.stream(selectedFields) .mapToObj(i -> fullFieldNames[i]) @@ -182,10 +202,13 @@ private static ColumnVector createVector( return readVector; } - private static ColumnVector createVectorFromConstant( - LogicalType type, - Object value, - int batchSize) { + /** + * Builds a constant-filled column vector for either a partition column (non-null value) or a + * missing-column slot (null value). Used both at the batch-generator level for partition + * injection and at the row-reader level for fields absent from the Parquet file. + */ + public static ColumnVector createVectorFromConstant( + LogicalType type, Object value, int batchSize) { switch (type.getTypeRoot()) { case CHAR: case VARCHAR: @@ -278,6 +301,7 @@ private static ColumnVector createVectorFromConstant( value == null ? null : toInternal((Date) value), batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: HeapTimestampVector tv = new HeapTimestampVector(batchSize); if (value == null) { tv.fillWithNulls(); @@ -286,46 +310,41 @@ private static ColumnVector createVectorFromConstant( } return tv; case ARRAY: - ArrayType arrayType = (ArrayType) type; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize); - if (value == null) { - arrayGroup.fillWithNulls(); - return arrayGroup; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } - } else { - HeapArrayVector arrayVector = new HeapArrayVector(batchSize); - if (value == null) { - arrayVector.fillWithNulls(); - return arrayVector; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } + if (value != null) { + throw new UnsupportedOperationException("Unsupported create array with default value."); } + HeapArrayVector arrayVector = new HeapArrayVector(batchSize); + arrayVector.fillWithNulls(); + return arrayVector; case MAP: - HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); - if (value == null) { - mapVector.fillWithNulls(); - return mapVector; - } else { - throw new UnsupportedOperationException("Unsupported create map with default value."); + case MULTISET: + if (value != null) { + throw new UnsupportedOperationException( + "Unsupported create " + type.getTypeRoot() + " with default value."); } + HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); + mapVector.fillWithNulls(); + return mapVector; case ROW: - HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize); - if (value == null) { - rowVector.fillWithNulls(); - return rowVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create row with default value."); } + RowType rowType = (RowType) type; + WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()]; + for (int i = 0; i < childVectors.length; i++) { + childVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); + } + HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors); + rowVector.fillWithNulls(); + return rowVector; default: throw new UnsupportedOperationException("Unsupported type: " + type); } } - private static List filterDescriptors(int depth, Type type, List columns) throws ParquetRuntimeException { + private static List filterDescriptors( + int depth, Type type, List columns) throws ParquetRuntimeException { List filtered = new ArrayList<>(); for (ColumnDescriptor descriptor : columns) { if (depth >= descriptor.getPath().length) { @@ -339,24 +358,61 @@ private static List filterDescriptors(int depth, Type type, Li return filtered; } + /** + * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the + * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP}, + * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by + * the supplied pre-built {@link ParquetField} tree. + * + * @param field the {@link ParquetField} tree for this column, built by + * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested + * types; ignored for primitives. + */ + public static ColumnReader createColumnReader( + boolean utcTimestamp, + LogicalType fieldType, + Type physicalType, + List descriptors, + PageReadStore pages, + @Nullable ParquetField field) throws IOException { + switch (fieldType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + Preconditions.checkNotNull( + field, "ParquetField must be provided for nested type: %s", fieldType); + return new NestedColumnReader(utcTimestamp, pages, field); + default: + return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages); + } + } + + /** + * Backward-compat entry point kept for callers that don't project nested types and therefore + * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload + * with a null field; nested types now go through that overload directly. + * + * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore, + * ParquetField)} so nested types take the Dremel path. + */ + @Deprecated public static ColumnReader createColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List descriptors, PageReadStore pages) throws IOException { - return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, - pages, 0); + return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null); } - private static ColumnReader createColumnReader( + private static ColumnReader createPrimitiveColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List columns, - PageReadStore pages, - int depth) throws IOException { - List descriptors = filterDescriptors(depth, physicalType, columns); + PageReadStore pages) throws IOException { + List descriptors = filterDescriptors(0, physicalType, columns); ColumnDescriptor descriptor = descriptors.get(0); PageReader pageReader = pages.getPageReader(descriptor); switch (fieldType.getTypeRoot()) { @@ -392,7 +448,9 @@ private static ColumnReader createColumnReader( case INT96: return new TimestampColumnReader(utcTimestamp, descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for TIMESTAMP: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } case DECIMAL: switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { @@ -403,106 +461,23 @@ private static ColumnReader createColumnReader( case BINARY: return new BytesColumnReader(descriptor, pageReader); case FIXED_LEN_BYTE_ARRAY: - return new FixedLenBytesColumnReader( - descriptor, pageReader); + return new FixedLenBytesColumnReader(descriptor, pageReader); default: - throw new AssertionError(); - } - case ARRAY: - ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new ArrayGroupReader(createColumnReader( - utcTimestamp, - arrayType.getElementType(), - elementType, - descriptors, - pages, - elementDepth)); - } else { - return new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - fieldType); - } - case MAP: - MapType mapType = (MapType) fieldType; - ArrayColumnReader keyReader = - new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - new ArrayType(mapType.getKeyType())); - ColumnReader valueReader; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueReader = new ArrayGroupReader(createColumnReader( - utcTimestamp, - mapType.getValueType(), - physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type - descriptors.subList(1, descriptors.size()), // remove the key descriptor - pages, - depth + 2)); // increase the depth by 2, because there's a key_value entry in the path - } else { - valueReader = new ArrayColumnReader( - descriptors.get(1), - pages.getPageReader(descriptors.get(1)), - utcTimestamp, - descriptors.get(1).getPrimitiveType(), - new ArrayType(mapType.getValueType())); + throw new AssertionError( + "Unexpected physical type for DECIMAL: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } - return new MapColumnReader(keyReader, valueReader); - case ROW: - RowType rowType = (RowType) fieldType; - GroupType groupType = physicalType.asGroupType(); - List fieldReaders = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - // schema evolution: read the parquet file with a new extended field name. - int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); - if (fieldIndex < 0) { - fieldReaders.add(new EmptyColumnReader()); - } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - fieldReaders.add( - createColumnReader( - utcTimestamp, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } else { - fieldReaders.add( - createColumnReader( - utcTimestamp, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } - } - } - return new RowColumnReader(fieldReaders); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + /** + * Creates the writable column vector that the reader will write into. The returned vector shape + * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled + * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in + * {@link NestedColumnReader} can pass them through unchanged. + */ public static WritableColumnVector createWritableColumnVector( int batchSize, LogicalType fieldType, @@ -523,40 +498,48 @@ private static WritableColumnVector createWritableColumnVector( switch (fieldType.getTypeRoot()) { case BOOLEAN: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBooleanVector(batchSize); case TINYINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapByteVector(batchSize); case DOUBLE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDoubleVector(batchSize); case FLOAT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.FLOAT, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapFloatVector(batchSize); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapIntVector(batchSize); case BIGINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT64, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapLongVector(batchSize); case SMALLINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapShortVector(batchSize); case CHAR: case VARCHAR: case BINARY: case VARBINARY: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BINARY, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BINARY, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBytesVector(batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: @@ -566,112 +549,64 @@ private static WritableColumnVector createWritableColumnVector( case DECIMAL: checkArgument( (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY - || typeName == PrimitiveType.PrimitiveTypeName.BINARY) + || typeName == PrimitiveType.PrimitiveTypeName.BINARY) && primitiveType.getOriginalType() == OriginalType.DECIMAL, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDecimalVector(batchSize); case ARRAY: ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - elementType, - descriptors, - elementDepth)); - } else { - return new HeapArrayVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - physicalType, - descriptors, - depth)); - } - case MAP: + return new HeapArrayVector( + batchSize, + createWritableColumnVector( + batchSize, arrayType.getElementType(), physicalType, descriptors, depth)); + case MAP: { MapType mapType = (MapType) fieldType; - GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType(); - // the map column has three level paths. - WritableColumnVector keyColumnVector = createWritableColumnVector( + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( batchSize, - new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()), - repeatedType.getType(0), - descriptors, - depth + 2); - WritableColumnVector valueColumnVector; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueColumnVector = new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - mapType.getValueType(), - repeatedType.getType(1).asGroupType(), - descriptors, - depth + 2)); - } else { - valueColumnVector = createWritableColumnVector( - batchSize, - new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()), - repeatedType.getType(1), - descriptors, - depth + 2); - } - return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector); + createWritableColumnVector( + batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2), + createWritableColumnVector( + batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2)); + } + case MULTISET: { + MultisetType multisetType = (MultisetType) fieldType; + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( + batchSize, + createWritableColumnVector( + batchSize, + multisetType.getElementType(), + repeatedType.getType(0), + descriptors, + depth + 2), + createWritableColumnVector( + batchSize, + new IntType(false), + repeatedType.getType(1), + descriptors, + depth + 2)); + } case ROW: RowType rowType = (RowType) fieldType; GroupType groupType = physicalType.asGroupType(); WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()]; for (int i = 0; i < columnVectors.length; i++) { - // schema evolution: read the file with a new extended field name. int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); if (fieldIndex < 0) { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant( - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize); - } else { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); - } + // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled + // vector of the correct shape; NestedColumnReader.readRow will pass it through when the + // matching ParquetField child is null. + columnVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = - createWritableColumnVector( - batchSize, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } else { - columnVectors[i] = - createWritableColumnVector( - batchSize, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } + columnVectors[i] = + createWritableColumnVector( + batchSize, + rowType.getTypeAt(i), + groupType.getType(fieldIndex), + descriptors, + depth + 1); } } return new HeapRowColumnVector(batchSize, columnVectors); @@ -681,56 +616,245 @@ private static WritableColumnVector createWritableColumnVector( } /** - * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. - * - * @return The physical field index or -1 if the field does not exist + * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching + * Parquet's canonical 3-level map encoding. */ - private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { - // get index from fileSchema type, else, return -1 - return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + private static GroupType unwrapMapRepeatedType(Type physicalType) { + return physicalType.asGroupType().getType(0).asGroupType(); } + // ------------------------------------------------------------------------------------------ + // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil) + // + // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children + // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This + // matches the Hudi schema-evolution contract and is the companion to the null-child branch in + // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`. + // ------------------------------------------------------------------------------------------ + /** - * Check whether the given list type is a three-level list type. - *

- * group (LIST) { - * repeated group list { - * element; - * } - * } - * - * @param type list type - * @return true if the list type is a three-level list type + * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed + * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally; + * primitive top-level fields produce {@code null} entries (callers don't need a tree for those). + */ + public static List buildFieldsList( + List children, List fieldNames, MessageColumnIO columnIO) { + List list = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + RowType.RowField child = children.get(i); + if (isNestedType(child.getType())) { + list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i)))); + } else { + list.add(null); + } + } + return list; + } + + private static boolean isNestedType(LogicalType type) { + return type instanceof RowType + || type instanceof ArrayType + || type instanceof MapType + || type instanceof MultisetType; + } + + @Nullable + private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) { + boolean required = columnIO.getType().getRepetition() == REQUIRED; + int repetitionLevel = columnIO.getRepetitionLevel(); + int definitionLevel = columnIO.getDefinitionLevel(); + LogicalType type = rowField.getType(); + String fieldName = rowField.getName(); + if (type instanceof RowType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + RowType rowType = (RowType) type; + List childFields = rowType.getFields(); + List fieldsList = new ArrayList<>(childFields.size()); + for (RowType.RowField childField : childFields) { + // Hudi schema evolution: a logical child may be absent from the Parquet file. In that + // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the + // pre-filled null vector instead of recursing. + ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName()); + if (childIo == null) { + fieldsList.add(null); + } else { + fieldsList.add(constructField(childField, childIo)); + } + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(fieldsList)); + } + + if (type instanceof MapType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MapType mapType = (MapType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof MultisetType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MultisetType multisetType = (MultisetType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", multisetType.getElementType()), + keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof ArrayType) { + ArrayType arrayType = (ArrayType) type; + ColumnIO elementTypeColumnIO; + if (columnIO instanceof GroupColumnIO) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { + while (!Objects.equals(groupColumnIO.getName(), fieldName)) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + elementTypeColumnIO = groupColumnIO; + } else { + if (arrayType.getElementType() instanceof RowType) { + elementTypeColumnIO = groupColumnIO; + } else { + elementTypeColumnIO = groupColumnIO.getChild(0); + } + } + } else if (columnIO instanceof PrimitiveColumnIO) { + elementTypeColumnIO = columnIO; + } else { + throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); + } + + ParquetField elementField = + constructField( + new RowType.RowField("", arrayType.getElementType()), + getArrayElementColumn(elementTypeColumnIO)); + if (repetitionLevel == elementField.getRepetitionLevel()) { + repetitionLevel = columnIO.getParent().getRepetitionLevel(); + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.singletonList(elementField)); + } + + PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO; + return new ParquetPrimitiveField( + type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId()); + } + + /** + * Parquet column names are case-insensitive in Flink's lookup. Matches upstream + * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent. */ - private static boolean isThreeLevelList(Type type) { - if (type.isPrimitive()) { - return false; + public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName); + if (columnIO != null) { + return columnIO; } - GroupType groupType = type.asGroupType(); - OriginalType originalType = groupType.getOriginalType(); - return originalType == OriginalType.LIST - && groupType.getType(0).getName().equals("list"); + throw new FlinkRuntimeException( + "Can not find column io for parquet reader. Column name: " + columnName); } /** - * Construct the error message when primitive type mismatches. - * - * @param primitiveType Primitive type - * @param fieldType Logical field type - * @return The error message + * Case-insensitive column lookup that returns {@code null} when no match is found — the + * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to + * emit null {@link ParquetField} children for fields absent from the Parquet file. */ - private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { - return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name()); + @Nullable + private static ColumnIO lookupColumnByNameOrNull( + GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = groupColumnIO.getChild(columnName); + if (columnIO != null) { + return columnIO; + } + for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) { + if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) { + return groupColumnIO.getChild(i); + } + } + return null; + } + + public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { + while (groupColumnIO.getChildrenCount() == 1) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + return groupColumnIO; + } + + public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { + while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { + columnIO = ((GroupColumnIO) columnIO).getChild(0); + } + + // Three-level list: skip the synthetic `element` / `list` wrapper when present. + if (columnIO instanceof GroupColumnIO + && columnIO.getType().getLogicalTypeAnnotation() == null + && ((GroupColumnIO) columnIO).getChildrenCount() == 1 + && !columnIO.getName().equals("array") + && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { + return ((GroupColumnIO) columnIO).getChild(0); + } + return columnIO; } /** - * Construct the error message when original type mismatches. + * Returns the field index with given physical row type {@code groupType} and field name + * {@code fieldName}. * - * @param originalType Original type - * @param fieldType Logical field type - * @return The error message + * @return the physical field index or -1 if the field does not exist + */ + private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { + return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + } + + private static String getPrimitiveTypeCheckFailureMessage( + PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Primitive type: %s. Field type: %s.", + primitiveType, fieldType.getTypeRoot().name()); + } + + private static String getOriginalTypeCheckFailureMessage( + OriginalType originalType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Original type: %s. Field type: %s.", + originalType, fieldType.getTypeRoot().name()); + } + + /** + * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience + * for callers that need to mirror Hudi's original behaviour where a missing column produces an + * explicit null-valued reader rather than being omitted from the batch. */ - private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) { - return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name()); + public static ColumnReader emptyColumnReader() { + return new EmptyColumnReader(); } } diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java new file mode 100644 index 0000000000000..d51d7ee754b8a --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of booleans. + * + *

Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper. + */ +public class BooleanArrayList { + private int size; + private boolean[] array; + + public BooleanArrayList(int capacity) { + this.size = 0; + this.array = new boolean[capacity]; + } + + public int size() { + return size; + } + + public boolean add(boolean element) { + grow(size + 1); + array[size++] = element; + return true; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public boolean[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final boolean[] t = new boolean[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java new file mode 100644 index 0000000000000..4787dbb5b9ddb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; +import java.util.NoSuchElementException; + +/** + * Minimal implementation of an array-backed list of ints. + * + *

Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class IntArrayList { + + private int size; + private int[] array; + + public IntArrayList(final int capacity) { + this.size = 0; + this.array = new int[capacity]; + } + + public int size() { + return size; + } + + public boolean add(final int number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public int removeLast() { + if (size == 0) { + throw new NoSuchElementException(); + } + --size; + return array[size]; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return size == 0; + } + + private void grow(final int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final int[] t = new int[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } + + public int[] toArray() { + return Arrays.copyOf(array, size); + } + + public static final IntArrayList EMPTY = + new IntArrayList(0) { + + @Override + public boolean add(int number) { + throw new UnsupportedOperationException(); + } + + @Override + public int removeLast() { + throw new UnsupportedOperationException(); + } + }; +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java new file mode 100644 index 0000000000000..a51291f9d8441 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of longs. + * + *

Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class LongArrayList { + + private int size; + private long[] array; + + public LongArrayList(int capacity) { + this.size = 0; + this.array = new long[capacity]; + } + + public int size() { + return size; + } + + public boolean add(long number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public long removeLong(int index) { + if (index >= size) { + throw new IndexOutOfBoundsException( + "Index (" + index + ") is greater than or equal to list size (" + size + ")"); + } + final long old = array[index]; + size--; + if (index != size) { + System.arraycopy(array, index + 1, array, index, size - index); + } + return old; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public long[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final long[] t = new long[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java new file mode 100644 index 0000000000000..3f2f8976b69bf --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; + +import static java.lang.String.format; + +/** + * Utils to calculate nested type position. + * + *

Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.utils.NestedPositionUtil}). + */ +public class NestedPositionUtil { + + /** + * Calculate row offsets according to column's max repetition level, definition level, value's + * repetition level and definition level. Each row has three situation: + *

  • Row is not defined,because it's optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Row is null + *
  • Row is defined and not empty. + * + * @param field field that contains the row column message include max repetition level and + * definition level. + * @param fieldRepetitionLevels int array with each value's repetition level. + * @param fieldDefinitionLevels int array with each value's definition level. + * @return {@link RowPosition} contains collections row count and isNull array. + */ + public static RowPosition calculateRowOffsets( + ParquetField field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) { + int rowDefinitionLevel = field.getDefinitionLevel(); + int rowRepetitionLevel = field.getRepetitionLevel(); + int nullValuesCount = 0; + BooleanArrayList nullRowFlags = new BooleanArrayList(0); + for (int i = 0; i < fieldDefinitionLevels.length; i++) { + // If a row's last field is an array, the repetition levels for the array's items will + // be larger than the parent row's repetition level, so we need to skip those values. + if (fieldRepetitionLevels[i] > rowRepetitionLevel) { + continue; + } + + if (fieldDefinitionLevels[i] >= rowDefinitionLevel) { + // current row is defined and not empty + nullRowFlags.add(false); + } else { + // current row is null + nullRowFlags.add(true); + nullValuesCount++; + } + } + if (nullValuesCount == 0) { + return new RowPosition(null, fieldDefinitionLevels.length); + } + return new RowPosition(nullRowFlags.toArray(), nullRowFlags.size()); + } + + /** + * Calculate the collection's offsets according to column's max repetition level, definition + * level, value's repetition level and definition level. Each collection (Array or Map) has four + * situation: + *
  • Collection is not defined, because optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Collection is null + *
  • Collection is defined but empty + *
  • Collection is defined and not empty. In this case offset value is increased by the number + * of elements in that collection + * + * @param field field that contains array/map column message include max repetition level and + * definition level. + * @param definitionLevels int array with each value's definition level. + * @param repetitionLevels int array with each value's repetition level. + * @return {@link CollectionPosition} contains collections offset array, length array and isNull + * array. + */ + public static CollectionPosition calculateCollectionOffsets( + ParquetField field, int[] definitionLevels, int[] repetitionLevels) { + int collectionDefinitionLevel = field.getDefinitionLevel(); + int collectionRepetitionLevel = field.getRepetitionLevel() + 1; + int offset = 0; + int valueCount = 0; + LongArrayList offsets = new LongArrayList(0); + offsets.add(offset); + BooleanArrayList emptyCollectionFlags = new BooleanArrayList(0); + BooleanArrayList nullCollectionFlags = new BooleanArrayList(0); + int nullValuesCount = 0; + for (int i = 0; + i < definitionLevels.length; + i = getNextCollectionStartIndex(repetitionLevels, collectionRepetitionLevel, i)) { + valueCount++; + if (definitionLevels[i] >= collectionDefinitionLevel - 1) { + boolean isNull = + isOptionalFieldValueNull(definitionLevels[i], collectionDefinitionLevel); + nullCollectionFlags.add(isNull); + nullValuesCount += isNull ? 1 : 0; + // definitionLevels[i] > collectionDefinitionLevel => Collection is defined and not + // empty + // definitionLevels[i] == collectionDefinitionLevel => Collection is defined but + // empty + if (definitionLevels[i] > collectionDefinitionLevel) { + emptyCollectionFlags.add(false); + offset += getCollectionSize(repetitionLevels, collectionRepetitionLevel, i + 1); + } else if (definitionLevels[i] == collectionDefinitionLevel) { + offset++; + emptyCollectionFlags.add(true); + } else { + offset++; + emptyCollectionFlags.add(false); + } + offsets.add(offset); + } else { + // when definitionLevels[i] < collectionDefinitionLevel - 1, it means the collection + // is + // not defined, but we need to regard it as null to avoid getting value wrong. + nullCollectionFlags.add(true); + nullValuesCount++; + offsets.add(++offset); + emptyCollectionFlags.add(false); + } + } + long[] offsetsArray = offsets.toArray(); + long[] length = calculateLengthByOffsets(emptyCollectionFlags.toArray(), offsetsArray); + if (nullValuesCount == 0) { + return new CollectionPosition(null, offsetsArray, length, valueCount); + } + return new CollectionPosition( + nullCollectionFlags.toArray(), offsetsArray, length, valueCount); + } + + public static boolean isOptionalFieldValueNull(int definitionLevel, int maxDefinitionLevel) { + return definitionLevel == maxDefinitionLevel - 1; + } + + public static long[] calculateLengthByOffsets( + boolean[] collectionIsEmpty, long[] arrayOffsets) { + LongArrayList lengthList = new LongArrayList(arrayOffsets.length); + for (int i = 0; i < arrayOffsets.length - 1; i++) { + long offset = arrayOffsets[i]; + long length = arrayOffsets[i + 1] - offset; + if (length < 0) { + throw new IllegalArgumentException( + format( + "Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", + i, arrayOffsets[i], i + 1, arrayOffsets[i + 1])); + } + if (collectionIsEmpty[i]) { + length = 0; + } + lengthList.add(length); + } + return lengthList.toArray(); + } + + private static int getNextCollectionStartIndex( + int[] repetitionLevels, int maxRepetitionLevel, int elementIndex) { + do { + elementIndex++; + } while (hasMoreElements(repetitionLevels, elementIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, elementIndex)); + return elementIndex; + } + + /** This method is only called for non-empty collections. */ + private static int getCollectionSize( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + int size = 1; + while (hasMoreElements(repetitionLevels, nextIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, nextIndex)) { + // Collection elements cannot only be primitive, but also can have nested structure + // Counting only elements which belong to current collection, skipping inner elements of + // nested collections/structs + if (repetitionLevels[nextIndex] <= maxRepetitionLevel) { + size++; + } + nextIndex++; + } + return size; + } + + private static boolean isNotCollectionBeginningMarker( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + return repetitionLevels[nextIndex] >= maxRepetitionLevel; + } + + private static boolean hasMoreElements(int[] repetitionLevels, int nextIndex) { + return nextIndex < repetitionLevels.length; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java deleted file mode 100644 index 4c9275f3b0932..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupArrayData implements ArrayData { - - WritableColumnVector vector; - int rowId; - - public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) { - this.vector = vector; - this.rowId = rowId; - } - - @Override - public int size() { - if (vector == null) { - return 0; - } - - if (vector instanceof HeapRowColumnVector) { - // assume all fields have the same size - if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) { - return 0; - } - return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size(); - } - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean isNullAt(int index) { - if (vector == null) { - return true; - } - - if (vector instanceof HeapRowColumnVector) { - return ((HeapRowColumnVector) vector).vectors == null; - } - - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean getBoolean(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float getFloat(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public StringData getString(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public DecimalData getDecimal(int index, int precision, int scale) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public TimestampData getTimestamp(int index, int precision) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RawValueData getRawValue(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] getBinary(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public ArrayData getArray(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public MapData getMap(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RowData getRow(int index, int numFields) { - return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java deleted file mode 100644 index 69cb6feca13e4..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupMapData implements MapData { - - WritableColumnVector keyVector; - WritableColumnVector valueVector; - int rowId; - - public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) { - this.keyVector = keyVector; - this.valueVector = valueVector; - this.rowId = rowId; - } - - @Override - public int size() { - if (keyVector == null) { - return 0; - } - - if (keyVector instanceof HeapArrayVector) { - return ((HeapArrayVector) keyVector).getArray(rowId).size(); - } - throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector"); - } - - @Override - public ArrayData keyArray() { - return ((HeapArrayVector) keyVector).getArray(rowId); - } - - @Override - public ArrayData valueArray() { - if (valueVector instanceof HeapArrayVector) { - return ((HeapArrayVector) valueVector).getArray(rowId); - } else if (valueVector instanceof HeapArrayGroupColumnVector) { - return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId); - } - throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector"); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java deleted file mode 100644 index 439c1880823f1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.types.RowKind; - -public class ColumnarGroupRowData implements RowData { - - HeapRowColumnVector vector; - int rowId; - int index; - - public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) { - this.vector = vector; - this.rowId = rowId; - this.index = index; - } - - @Override - public int getArity() { - return vector.vectors.length; - } - - @Override - public RowKind getRowKind() { - return RowKind.INSERT; - } - - @Override - public void setRowKind(RowKind rowKind) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public boolean isNullAt(int pos) { - return - vector.vectors[pos].isNullAt(rowId) - || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index); - } - - @Override - public boolean getBoolean(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index); - } - - @Override - public byte getByte(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index); - } - - @Override - public short getShort(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index); - } - - @Override - public int getInt(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index); - } - - @Override - public long getLong(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index); - } - - @Override - public float getFloat(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index); - } - - @Override - public double getDouble(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index); - } - - @Override - public StringData getString(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index); - } - - @Override - public DecimalData getDecimal(int pos, int i1, int i2) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2); - } - - @Override - public TimestampData getTimestamp(int pos, int i1) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1); - } - - @Override - public RawValueData getRawValue(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index); - } - - @Override - public byte[] getBinary(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index); - } - - @Override - public ArrayData getArray(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index); - } - - @Override - public MapData getMap(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index); - } - - @Override - public RowData getRow(int pos, int numFields) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java deleted file mode 100644 index 3d7d8b1f0de0f..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -/** - * This class represents a nullable heap row column vector. - */ -public class HeapArrayGroupColumnVector extends AbstractHeapVector - implements WritableColumnVector, ArrayColumnVector { - - public WritableColumnVector vector; - - public HeapArrayGroupColumnVector(int len) { - super(len); - } - - public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) { - super(len); - this.vector = vector; - } - - @Override - public ArrayData getArray(int rowId) { - return new ColumnarGroupArrayData(vector, rowId); - } - - @Override - public void reset() { - super.reset(); - vector.reset(); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java index a0dced01e5e8d..2f21a323302f1 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java @@ -57,6 +57,37 @@ public int getLen() { return this.isNull.length; } + // --------------------------------------------------------------------------------------------- + // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths} + // and {@code child} fields so legacy callers continue to work; the new {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + // future Flink-2.1-style caller use these accessors. + // --------------------------------------------------------------------------------------------- + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public ColumnVector getChild() { + return child; + } + + public void setChild(ColumnVector child) { + this.child = child; + } + @Override public ArrayData getArray(int i) { long offset = offsets[i]; diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index 0f088df55c1ac..14aad22039e0a 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -20,30 +20,97 @@ import lombok.Getter; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.ColumnarMapData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; /** * This class represents a nullable heap map column vector. + * + *

    Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from + * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward + * compatibility: the {@code keys} / {@code values} fields are typed + * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing + * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original + * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use + * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}. */ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { @Getter - private final WritableColumnVector keys; + private WritableColumnVector keys; @Getter - private final WritableColumnVector values; + private WritableColumnVector values; + + // --------------------------------------------------------------------------------------------- + // Flink 2.1 Dremel-style state. Populated by {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and + // consumed by {@link #getMap(int)}. + // --------------------------------------------------------------------------------------------- + private long[] offsets; + private long[] lengths; + private int size; public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) { super(len); + this.offsets = new long[len]; + this.lengths = new long[len]; this.keys = keys; this.values = values; } + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public void setKeys(WritableColumnVector keys) { + this.keys = keys; + } + + public void setValues(WritableColumnVector values) { + this.values = values; + } + + /** + * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract + * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}. + */ + public ColumnVector getKeyColumnVector() { + return keys; + } + + /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */ + public ColumnVector getValueColumnVector() { + return values; + } + @Override public MapData getMap(int rowId) { - return new ColumnarGroupMapData(keys, values, rowId); + long offset = offsets[rowId]; + long length = lengths[rowId]; + return new ColumnarMapData(keys, values, (int) offset, (int) length); } } - diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java index ae194e4e6ab05..0c640ce92ee40 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java @@ -37,6 +37,21 @@ public HeapRowColumnVector(int len, WritableColumnVector... vectors) { this.vectors = vectors; } + /** + * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code + * vectors} field so legacy callers continue to work; the new {@link + * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + * future Flink-2.1-style caller use this accessor. + */ + public WritableColumnVector[] getFields() { + return vectors; + } + + /** Counterpart of {@link #getFields()}. */ + public void setFields(WritableColumnVector[] fields) { + this.vectors = fields; + } + @Override public ColumnarRowData getRow(int i) { ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors)); diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java index 98b5e61050898..a37b88352cf52 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java @@ -18,21 +18,29 @@ package org.apache.hudi.table.format.cow.vector; +import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.columnar.vector.BytesColumnVector; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.DecimalColumnVector; +import org.apache.flink.table.data.columnar.vector.Dictionary; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.table.data.columnar.vector.LongColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector; + +import static org.apache.flink.util.Preconditions.checkArgument; /** - * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to - * provide {@link DecimalColumnVector} interface. - * - *

    Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector} - * because it is not public. + * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide + * {@link DecimalColumnVector} interface. */ -public class ParquetDecimalVector implements DecimalColumnVector { +public class ParquetDecimalVector + implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector { - public final ColumnVector vector; + private final ColumnVector vector; public ParquetDecimalVector(ColumnVector vector) { this.vector = vector; @@ -40,15 +48,180 @@ public ParquetDecimalVector(ColumnVector vector) { @Override public DecimalData getDecimal(int i, int precision, int scale) { - return DecimalData.fromUnscaledBytes( - ((BytesColumnVector) vector).getBytes(i).getBytes(), - precision, - scale); + if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) { + return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale); + } else if (ParquetSchemaConverter.is64BitDecimal(precision) + && vector instanceof LongColumnVector) { + return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale); + } else { + checkArgument( + vector instanceof BytesColumnVector, + "Reading decimal type occur unsupported vector type: %s", + vector.getClass()); + return DecimalData.fromUnscaledBytes( + ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale); + } + } + + public ColumnVector getVector() { + return vector; } @Override public boolean isNullAt(int i) { return vector.isNullAt(i); } -} + @Override + public void reset() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).reset(); + } + } + + @Override + public void setNullAt(int rowId) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNullAt(rowId); + } + } + + @Override + public void setNulls(int rowId, int count) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNulls(rowId, count); + } + } + + @Override + public void fillWithNulls() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).fillWithNulls(); + } + } + + @Override + public void setDictionary(Dictionary dictionary) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setDictionary(dictionary); + } + } + + @Override + public boolean hasDictionary() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).hasDictionary(); + } + return false; + } + + @Override + public WritableIntVector reserveDictionaryIds(int capacity) { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).reserveDictionaryIds(capacity); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public WritableIntVector getDictionaryIds() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).getDictionaryIds(); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public Bytes getBytes(int i) { + if (vector instanceof WritableBytesVector) { + return ((WritableBytesVector) vector).getBytes(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void appendBytes(int rowId, byte[] value, int offset, int length) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length); + } + } + + @Override + public void fill(byte[] value) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).fill(value); + } + } + + @Override + public int getInt(int i) { + if (vector instanceof WritableIntVector) { + return ((WritableIntVector) vector).getInt(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setInt(int rowId, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInt(rowId, value); + } + } + + @Override + public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void setInts(int rowId, int count, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, value); + } + } + + @Override + public void setInts(int rowId, int count, int[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).fill(value); + } + } + + @Override + public long getLong(int i) { + if (vector instanceof WritableLongVector) { + return ((WritableLongVector) vector).getLong(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setLong(int rowId, long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLong(rowId, value); + } + } + + @Override + public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).fill(value); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java new file mode 100644 index 0000000000000..fcdedfbc9d71d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent collection's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.CollectionPosition}). + */ +public class CollectionPosition { + @Nullable private final boolean[] isNull; + private final long[] offsets; + private final long[] length; + private final int valueCount; + + public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { + this.isNull = isNull; + this.offsets = offsets; + this.length = length; + this.valueCount = valueCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public long[] getOffsets() { + return offsets; + } + + public long[] getLength() { + return length; + } + + public int getValueCount() { + return valueCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java new file mode 100644 index 0000000000000..fe95419ac3218 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +/** + * To delegate repetition level and definition level. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.LevelDelegation}). + */ +public class LevelDelegation { + private final int[] repetitionLevel; + private final int[] definitionLevel; + + public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) { + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + } + + public int[] getRepetitionLevel() { + return repetitionLevel; + } + + public int[] getDefinitionLevel() { + return definitionLevel; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java new file mode 100644 index 0000000000000..5438b67973238 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent struct's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.RowPosition}). + */ +public class RowPosition { + @Nullable private final boolean[] isNull; + private final int positionsCount; + + public RowPosition(boolean[] isNull, int positionsCount) { + this.isNull = isNull; + this.positionsCount = positionsCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public int getPositionsCount() { + return positionsCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java deleted file mode 100644 index 6a8a01b74946a..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayVector; -import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; -import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.table.types.logical.ArrayType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Array {@link ColumnReader}. - */ -public class ArrayColumnReader extends BaseVectorizedColumnReader { - - // The value read in last time - private Object lastValue; - - // flag to indicate if there is no data in parquet data page - private boolean eof = false; - - // flag to indicate if it's the first time to read parquet data page with this instance - boolean isFirstRow = true; - - public ArrayColumnReader( - ColumnDescriptor descriptor, - PageReader pageReader, - boolean isUtcTimestamp, - Type type, - LogicalType logicalType) - throws IOException { - super(descriptor, pageReader, isUtcTimestamp, type, logicalType); - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayVector lcv = (HeapArrayVector) vector; - // before readBatch, initial the size of offsets & lengths as the default value, - // the actual size will be assigned in setChildrenInfo() after reading complete. - lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - // Because the length of ListColumnVector.child can't be known now, - // the valueList will save all data for ListColumnVector temporary. - List valueList = new ArrayList<>(); - - LogicalType category = ((ArrayType) logicalType).getElementType(); - - // read the first row in parquet data page, this will be only happened once for this - // instance - if (isFirstRow) { - if (!fetchNextValue(category)) { - return; - } - isFirstRow = false; - } - - int index = collectDataFromParquetPage(readNumber, lcv, valueList, category); - - // Convert valueList to array for the ListColumnVector.child - fillColumnVector(category, lcv, valueList, index); - } - - /** - * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating - * if there is more values to read (true). - * - * @param category - * @return boolean - * @throws IOException - */ - private boolean fetchNextValue(LogicalType category) throws IOException { - int left = readPageIfNeed(); - if (left > 0) { - // get the values of repetition and definitionLevel - readRepetitionAndDefinitionLevels(); - // read the data if it isn't null - if (definitionLevel == maxDefLevel) { - if (isCurrentPageDictionaryEncoded) { - lastValue = dataColumn.readValueDictionaryId(); - } else { - lastValue = readPrimitiveTypedRow(category); - } - } else { - lastValue = null; - } - return true; - } else { - eof = true; - return false; - } - } - - private int readPageIfNeed() throws IOException { - // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); - } - return leftInPage; - } - - // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper - // TODO Reduce the duplicated code - private Object readPrimitiveTypedRow(LogicalType category) { - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dataColumn.readString(); - case BOOLEAN: - return dataColumn.readBoolean(); - case TIME_WITHOUT_TIME_ZONE: - case DATE: - case INTEGER: - return dataColumn.readInteger(); - case TINYINT: - return dataColumn.readTinyInt(); - case SMALLINT: - return dataColumn.readSmallInt(); - case BIGINT: - return dataColumn.readLong(); - case FLOAT: - return dataColumn.readFloat(); - case DOUBLE: - return dataColumn.readDouble(); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dataColumn.readInteger(); - case INT64: - return dataColumn.readLong(); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return dataColumn.readString(); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dataColumn.readTimestamp(); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { - if (dictionaryValue == null) { - return null; - } - - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dictionary.readString(dictionaryValue); - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case INTEGER: - return dictionary.readInteger(dictionaryValue); - case BOOLEAN: - return dictionary.readBoolean(dictionaryValue) ? 1 : 0; - case DOUBLE: - return dictionary.readDouble(dictionaryValue); - case FLOAT: - return dictionary.readFloat(dictionaryValue); - case TINYINT: - return dictionary.readTinyInt(dictionaryValue); - case SMALLINT: - return dictionary.readSmallInt(dictionaryValue); - case BIGINT: - return dictionary.readLong(dictionaryValue); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dictionary.readInteger(dictionaryValue); - case INT64: - return dictionary.readLong(dictionaryValue); - case FIXED_LEN_BYTE_ARRAY: - case BINARY: - return dictionary.readString(dictionaryValue); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dictionary.readTimestamp(dictionaryValue); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - /** - * Collects data from a parquet page and returns the final row index where it stopped. The - * returned index can be equal to or less than total. - * - * @param total maximum number of rows to collect - * @param lcv column vector to do initial setup in data collection time - * @param valueList collection of values that will be fed into the vector later - * @param category - * @return int - * @throws IOException - */ - private int collectDataFromParquetPage( - int total, HeapArrayVector lcv, List valueList, LogicalType category) - throws IOException { - int index = 0; - /* - * Here is a nested loop for collecting all values from a parquet page. - * A column of array type can be considered as a list of lists, so the two loops are as below: - * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.: - * [0, 2, 3] <- index: 0 - * [NULL, 3, 4] <- index: 1 - * - * 2. The inner loop iterates on values within a row (sets all data from parquet data page - * for an element in ListColumnVector), so fetchNextValue returns values one-by-one: - * 0, 2, 3, NULL, 3, 4 - * - * As described below, the repetition level (repetitionLevel != 0) - * can be used to decide when we'll start to read values for the next list. - */ - while (!eof && index < total) { - // add element to ListColumnVector one by one - lcv.offsets[index] = valueList.size(); - /* - * Let's collect all values for a single list. - * Repetition level = 0 means that a new list started there in the parquet page, - * in that case, let's exit from the loop, and start to collect value for a new list. - */ - do { - /* - * Definition level = 0 when a NULL value was returned instead of a list - * (this is not the same as a NULL value in of a list). - */ - if (definitionLevel == 0) { - lcv.setNullAt(index); - } - valueList.add( - isCurrentPageDictionaryEncoded - ? dictionaryDecodeValue(category, (Integer) lastValue) - : lastValue); - } while (fetchNextValue(category) && (repetitionLevel != 0)); - - lcv.lengths[index] = valueList.size() - lcv.offsets[index]; - index++; - } - return index; - } - - /** - * The lengths & offsets will be initialized as default size (1024), it should be set to the - * actual size according to the element number. - */ - private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) { - lcv.setSize(itemNum); - long[] lcvLength = new long[elementNum]; - long[] lcvOffset = new long[elementNum]; - System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum); - System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum); - lcv.lengths = lcvLength; - lcv.offsets = lcvOffset; - } - - private void fillColumnVector( - LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) { - int total = valueList.size(); - setChildrenInfo(lcv, total, elementNum); - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - lcv.child = new HeapBytesVector(total); - ((HeapBytesVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (src == null) { - ((HeapBytesVector) lcv.child).setNullAt(i); - } else { - ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length); - } - } - break; - case BOOLEAN: - lcv.child = new HeapBooleanVector(total); - ((HeapBooleanVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapBooleanVector) lcv.child).setNullAt(i); - } else { - ((HeapBooleanVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TINYINT: - lcv.child = new HeapByteVector(total); - ((HeapByteVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapByteVector) lcv.child).setNullAt(i); - } else { - ((HeapByteVector) lcv.child).vector[i] = - (byte) ((List) valueList).get(i).intValue(); - } - } - break; - case SMALLINT: - lcv.child = new HeapShortVector(total); - ((HeapShortVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapShortVector) lcv.child).setNullAt(i); - } else { - ((HeapShortVector) lcv.child).vector[i] = - (short) ((List) valueList).get(i).intValue(); - } - } - break; - case INTEGER: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - lcv.child = new HeapIntVector(total); - ((HeapIntVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) lcv.child).setNullAt(i); - } else { - ((HeapIntVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case FLOAT: - lcv.child = new HeapFloatVector(total); - ((HeapFloatVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapFloatVector) lcv.child).setNullAt(i); - } else { - ((HeapFloatVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case BIGINT: - lcv.child = new HeapLongVector(total); - ((HeapLongVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) lcv.child).setNullAt(i); - } else { - ((HeapLongVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case DOUBLE: - lcv.child = new HeapDoubleVector(total); - ((HeapDoubleVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapDoubleVector) lcv.child).setNullAt(i); - } else { - ((HeapDoubleVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - lcv.child = new HeapTimestampVector(total); - ((HeapTimestampVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapTimestampVector) lcv.child).setNullAt(i); - } else { - ((HeapTimestampVector) lcv.child) - .setTimestamp(i, ((List) valueList).get(i)); - } - } - break; - case DECIMAL: - PrimitiveType.PrimitiveTypeName primitiveTypeName = - descriptor.getPrimitiveType().getPrimitiveTypeName(); - switch (primitiveTypeName) { - case INT32: - lcv.child = new ParquetDecimalVector(new HeapIntVector(total)); - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - case INT64: - lcv.child = new ParquetDecimalVector(new HeapLongVector(total)); - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - default: - lcv.child = new ParquetDecimalVector(new HeapBytesVector(total)); - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (valueList.get(i) == null) { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .appendBytes(i, src, 0, src.length); - } - } - break; - } - break; - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java deleted file mode 100644 index df7c5d85bc4ab..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}. - */ -public class ArrayGroupReader implements ColumnReader { - - private final ColumnReader fieldReader; - - public ArrayGroupReader(ColumnReader fieldReader) { - this.fieldReader = fieldReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector; - - fieldReader.readToVector(readNumber, rowColumnVector.vector); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java deleted file mode 100644 index ee65dd22c4369..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Map {@link ColumnReader}. - */ -public class MapColumnReader implements ColumnReader { - - private final ArrayColumnReader keyReader; - private final ColumnReader valueReader; - - public MapColumnReader( - ArrayColumnReader keyReader, ColumnReader valueReader) { - this.keyReader = keyReader; - this.valueReader = valueReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector; - AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys()); - keyReader.readToVector(readNumber, mapColumnVector.getKeys()); - valueReader.readToVector(readNumber, mapColumnVector.getValues()); - for (int i = 0; i < keyArrayColumnVector.getLen(); i++) { - if (keyArrayColumnVector.isNullAt(i)) { - mapColumnVector.setNullAt(i); - } - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java new file mode 100644 index 0000000000000..b4f04b20c4772 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.NestedPositionUtil; +import org.apache.hudi.table.format.cow.vector.HeapArrayVector; +import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; +import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.Preconditions; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code + * Row}). Resolves nested structures using Dremel striping/assembly; see the + * striping and assembly algorithms from the Dremel paper. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream: + * + *

      + *
    • Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code + * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code + * HeapMapVector}/{@code HeapArrayVector}. + *
    • Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a + * {@link RowType} may contain {@code null} children — meaning the corresponding logical + * field is absent from the Parquet file. Those slots are passed through unchanged and do + * not contribute to the row's repetition/definition-level stream. + *
    + */ +public class NestedColumnReader implements ColumnReader { + + private final Map columnReaders; + private final boolean isUtcTimestamp; + + private final PageReadStore pages; + + private final ParquetField field; + + public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) { + this.isUtcTimestamp = isUtcTimestamp; + this.pages = pages; + this.field = field; + this.columnReaders = new HashMap<>(); + } + + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + readData(field, readNumber, vector, false); + } + + private Tuple2 readData( + ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException { + if (field.getType() instanceof RowType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) { + return readMap((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof ArrayType) { + return readArray((ParquetGroupField) field, readNumber, vector, inside); + } else { + return readPrimitive((ParquetPrimitiveField) field, readNumber, vector); + } + } + + private Tuple2 readRow( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector; + LevelDelegation levelDelegation = null; + List children = field.getChildren(); + WritableColumnVector[] childrenVectors = heapRowVector.getFields(); + WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length]; + for (int i = 0; i < children.size(); i++) { + ParquetField child = children.get(i); + if (child == null) { + // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector + // (ROW branch); keep it as is and skip contributing to the level stream. + finalChildrenVectors[i] = childrenVectors[i]; + continue; + } + Tuple2 tuple = + readData(child, readNumber, childrenVectors[i], true); + levelDelegation = tuple.f0; + finalChildrenVectors[i] = tuple.f1; + } + if (levelDelegation == null) { + throw new FlinkRuntimeException( + String.format("Row field does not have any non-null children: %s.", field)); + } + + RowPosition rowPosition = + NestedPositionUtil.calculateRowOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If row was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors); + } else { + heapRowVector.setFields(finalChildrenVectors); + } + + if (rowPosition.getIsNull() != null) { + setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); + } + + // Hudi-specific: collapse a present row whose every child is null into a null row, so that a + // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the + // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely + // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the + // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + int rowCount = rowPosition.getPositionsCount(); + for (int j = 0; j < rowCount; j++) { + if (heapRowVector.isNullAt(j)) { + continue; + } + boolean allChildrenNull = true; + for (WritableColumnVector child : finalChildrenVectors) { + if (!child.isNullAt(j)) { + allChildrenNull = false; + break; + } + } + if (allChildrenNull) { + heapRowVector.setNullAt(j); + } + } + return Tuple2.of(levelDelegation, heapRowVector); + } + + private Tuple2 readMap( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapMapColumnVector mapVector = (HeapMapColumnVector) vector; + mapVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 2, + "Maps must have two type parameters, found %s", + children.size()); + Tuple2 keyTuple = + readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true); + Tuple2 valueTuple = + readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true); + + LevelDelegation levelDelegation = keyTuple.f0; + + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If map was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1); + } else { + mapVector.setKeys(keyTuple.f1); + mapVector.setValues(valueTuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), mapVector); + } + + mapVector.setLengths(collectionPosition.getLength()); + mapVector.setOffsets(collectionPosition.getOffsets()); + + return Tuple2.of(levelDelegation, mapVector); + } + + private Tuple2 readArray( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapArrayVector arrayVector = (HeapArrayVector) vector; + arrayVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 1, + "Arrays must have a single type parameter, found %s", + children.size()); + Tuple2 tuple = + readData(children.get(0), readNumber, arrayVector.getChild(), true); + + LevelDelegation levelDelegation = tuple.f0; + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If array was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1); + } else { + arrayVector.setChild(tuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), arrayVector); + } + arrayVector.setLengths(collectionPosition.getLength()); + arrayVector.setOffsets(collectionPosition.getOffsets()); + return Tuple2.of(levelDelegation, arrayVector); + } + + private Tuple2 readPrimitive( + ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException { + ColumnDescriptor descriptor = field.getDescriptor(); + NestedPrimitiveColumnReader reader = columnReaders.get(descriptor); + if (reader == null) { + reader = + new NestedPrimitiveColumnReader( + descriptor, + pages.getPageReader(descriptor), + isUtcTimestamp, + descriptor.getPrimitiveType(), + field.getType()); + columnReaders.put(descriptor, reader); + } + WritableColumnVector writableColumnVector = + reader.readAndNewVector(readNumber, (WritableColumnVector) vector); + return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); + } + + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { + for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { + if (nullFlags[index]) { + vector.setNullAt(index); + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java new file mode 100644 index 0000000000000..72809db1b2ceb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -0,0 +1,639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.IntArrayList; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; + +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.LogicalType; + +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; +import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; +import static org.apache.parquet.column.ValuesType.VALUES; + +/** + * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package + * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} + * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code + * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- + * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it + * a faithful copy of upstream. + */ +public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); + + private final IntArrayList repetitionLevelList = new IntArrayList(0); + private final IntArrayList definitionLevelList = new IntArrayList(0); + + private final PageReader pageReader; + private final ColumnDescriptor descriptor; + private final Type type; + private final LogicalType logicalType; + + /** The dictionary, if this column has dictionary encoding. */ + private final ParquetDataColumnReader dictionary; + + /** Maximum definition level for this column. */ + private final int maxDefLevel; + + private boolean isUtcTimestamp; + + /** Total number of values read. */ + private long valuesRead; + + /** + * value that indicates the end of the current page. That is, if valuesRead == + * endOfPageValueCount, we are at the end of the page. + */ + private long endOfPageValueCount; + + /** If true, the current page is dictionary encoded. */ + private boolean isCurrentPageDictionaryEncoded; + + private int definitionLevel; + private int repetitionLevel; + + /** Repetition/Definition/Value readers. */ + private IntIterator repetitionLevelColumn; + + private IntIterator definitionLevelColumn; + private ParquetDataColumnReader dataColumn; + + /** Total values in the current page. */ + private int pageValueCount; + + // flag to indicate if there is no data in parquet data page + private boolean eof = false; + + private boolean isFirstRow = true; + + private Object lastValue; + + public NestedPrimitiveColumnReader( + ColumnDescriptor descriptor, + PageReader pageReader, + boolean isUtcTimestamp, + Type parquetType, + LogicalType logicalType) + throws IOException { + this.descriptor = descriptor; + this.type = parquetType; + this.pageReader = pageReader; + this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.isUtcTimestamp = isUtcTimestamp; + this.logicalType = logicalType; + + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); + if (dictionaryPage != null) { + try { + this.dictionary = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + parquetType.asPrimitiveType(), + dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } catch (IOException e) { + throw new IOException( + String.format("Could not decode the dictionary for %s", descriptor), e); + } + } else { + this.dictionary = null; + this.isCurrentPageDictionaryEncoded = false; + } + } + + // Not invoked directly; callers use readAndNewVector instead. + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + throw new UnsupportedOperationException("This function should not be called."); + } + + public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector) + throws IOException { + if (isFirstRow) { + if (!readValue()) { + return vector; + } + isFirstRow = false; + } + + // index to set value. + int index = 0; + int valueIndex = 0; + List valueList = new ArrayList<>(); + + // repeated type need two loops to read data. + while (!eof && index < readNumber) { + do { + valueList.add(lastValue); + valueIndex++; + } while (readValue() && (repetitionLevel != 0)); + index++; + } + + return fillColumnVector(valueIndex, valueList); + } + + public LevelDelegation getLevelDelegation() { + int[] repetition = repetitionLevelList.toArray(); + int[] definition = definitionLevelList.toArray(); + repetitionLevelList.clear(); + definitionLevelList.clear(); + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + return new LevelDelegation(repetition, definition); + } + + private boolean readValue() throws IOException { + int left = readPageIfNeed(); + if (left > 0) { + // get the values of repetition and definitionLevel + readAndSaveRepetitionAndDefinitionLevels(); + // read the data if it isn't null + if (definitionLevel == maxDefLevel) { + if (isCurrentPageDictionaryEncoded) { + int dictionaryId = dataColumn.readValueDictionaryId(); + lastValue = dictionaryDecodeValue(logicalType, dictionaryId); + } else { + lastValue = readPrimitiveTypedRow(logicalType); + } + } else { + lastValue = null; + } + return true; + } else { + eof = true; + return false; + } + } + + private void readAndSaveRepetitionAndDefinitionLevels() { + // get the values of repetition and definitionLevel + repetitionLevel = repetitionLevelColumn.nextInt(); + definitionLevel = definitionLevelColumn.nextInt(); + valuesRead++; + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + } + + private int readPageIfNeed() throws IOException { + // Compute the number of values we want to read in this page. + int leftInPage = (int) (endOfPageValueCount - valuesRead); + if (leftInPage == 0) { + // no data left in current page, load data from new page + readPage(); + leftInPage = (int) (endOfPageValueCount - valuesRead); + } + return leftInPage; + } + + private Object readPrimitiveTypedRow(LogicalType category) { + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dataColumn.readBytes(); + case BOOLEAN: + return dataColumn.readBoolean(); + case TIME_WITHOUT_TIME_ZONE: + case DATE: + case INTEGER: + return dataColumn.readInteger(); + case TINYINT: + return dataColumn.readTinyInt(); + case SMALLINT: + return dataColumn.readSmallInt(); + case BIGINT: + return dataColumn.readLong(); + case FLOAT: + return dataColumn.readFloat(); + case DOUBLE: + return dataColumn.readDouble(); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dataColumn.readInteger(); + case INT64: + return dataColumn.readLong(); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return dataColumn.readBytes(); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dataColumn.readTimestamp(); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { + if (dictionaryValue == null) { + return null; + } + + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dictionary.readBytes(dictionaryValue); + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTEGER: + return dictionary.readInteger(dictionaryValue); + case BOOLEAN: + return dictionary.readBoolean(dictionaryValue) ? 1 : 0; + case DOUBLE: + return dictionary.readDouble(dictionaryValue); + case FLOAT: + return dictionary.readFloat(dictionaryValue); + case TINYINT: + return dictionary.readTinyInt(dictionaryValue); + case SMALLINT: + return dictionary.readSmallInt(dictionaryValue); + case BIGINT: + return dictionary.readLong(dictionaryValue); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dictionary.readInteger(dictionaryValue); + case INT64: + return dictionary.readLong(dictionaryValue); + case FIXED_LEN_BYTE_ARRAY: + case BINARY: + return dictionary.readBytes(dictionaryValue); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dictionary.readTimestamp(dictionaryValue); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private WritableColumnVector fillColumnVector(int total, List valueList) { + switch (logicalType.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + HeapBytesVector heapBytesVector = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (src == null) { + heapBytesVector.setNullAt(i); + } else { + heapBytesVector.appendBytes(i, src, 0, src.length); + } + } + return heapBytesVector; + case BOOLEAN: + HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapBooleanVector.setNullAt(i); + } else { + heapBooleanVector.vector[i] = ((List) valueList).get(i); + } + } + return heapBooleanVector; + case TINYINT: + HeapByteVector heapByteVector = new HeapByteVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapByteVector.setNullAt(i); + } else { + heapByteVector.vector[i] = (byte) ((List) valueList).get(i).intValue(); + } + } + return heapByteVector; + case SMALLINT: + HeapShortVector heapShortVector = new HeapShortVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapShortVector.setNullAt(i); + } else { + heapShortVector.vector[i] = (short) ((List) valueList).get(i).intValue(); + } + } + return heapShortVector; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + HeapIntVector heapIntVector = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapIntVector.setNullAt(i); + } else { + heapIntVector.vector[i] = ((List) valueList).get(i); + } + } + return heapIntVector; + case FLOAT: + HeapFloatVector heapFloatVector = new HeapFloatVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapFloatVector.setNullAt(i); + } else { + heapFloatVector.vector[i] = ((List) valueList).get(i); + } + } + return heapFloatVector; + case BIGINT: + HeapLongVector heapLongVector = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapLongVector.setNullAt(i); + } else { + heapLongVector.vector[i] = ((List) valueList).get(i); + } + } + return heapLongVector; + case DOUBLE: + HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapDoubleVector.setNullAt(i); + } else { + heapDoubleVector.vector[i] = ((List) valueList).get(i); + } + } + return heapDoubleVector; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapTimestampVector.setNullAt(i); + } else { + heapTimestampVector.setTimestamp(i, ((List) valueList).get(i)); + } + } + return heapTimestampVector; + case DECIMAL: + PrimitiveType.PrimitiveTypeName primitiveTypeName = + descriptor.getPrimitiveType().getPrimitiveTypeName(); + switch (primitiveTypeName) { + case INT32: + HeapIntVector phiv = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phiv.setNullAt(i); + } else { + phiv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phiv); + case INT64: + HeapLongVector phlv = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phlv.setNullAt(i); + } else { + phlv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phlv); + default: + HeapBytesVector phbv = getHeapBytesVector(total, valueList); + return new ParquetDecimalVector(phbv); + } + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static HeapBytesVector getHeapBytesVector(int total, List valueList) { + HeapBytesVector phbv = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (valueList.get(i) == null) { + phbv.setNullAt(i); + } else { + phbv.appendBytes(i, src, 0, src.length); + } + } + return phbv; + } + + protected void readPage() { + DataPage page = pageReader.readPage(); + + if (page == null) { + return; + } + + page.accept( + new DataPage.Visitor() { + @Override + public Void visit(DataPageV1 dataPageV1) { + readPageV1(dataPageV1); + return null; + } + + @Override + public Void visit(DataPageV2 dataPageV2) { + readPageV2(dataPageV2); + return null; + } + }); + } + + private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) + throws IOException { + this.pageValueCount = valueCount; + this.endOfPageValueCount = valuesRead + pageValueCount; + if (dataEncoding.usesDictionary()) { + this.dataColumn = null; + if (dictionary == null) { + throw new IOException( + String.format( + "Could not read page in col %s because the dictionary was missing for encoding %s.", + descriptor, dataEncoding)); + } + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getDictionaryBasedValuesReader( + descriptor, VALUES, dictionary.getDictionary()), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } else { + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getValuesReader(descriptor, VALUES), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = false; + } + + try { + dataColumn.initFromPage(pageValueCount, in); + } catch (IOException e) { + throw new IOException(String.format("Could not read page in col %s.", descriptor), e); + } + } + + private void readPageV1(DataPageV1 page) { + ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); + ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); + this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); + this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); + try { + BytesInput bytes = page.getBytes(); + LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount); + ByteBufferInputStream in = bytes.toInputStream(); + LOG.debug("Reading repetition levels at {}.", in.position()); + rlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading definition levels at {}.", in.position()); + dlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading data at {}.", in.position()); + initDataReader(page.getValueEncoding(), in, page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private void readPageV2(DataPageV2 page) { + this.pageValueCount = page.getValueCount(); + this.repetitionLevelColumn = + newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); + this.definitionLevelColumn = + newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); + try { + LOG.debug( + "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); + initDataReader( + page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) { + try { + if (maxLevel == 0) { + return new NullIntIterator(); + } + return new RLEIntIterator( + new RunLengthBitPackingHybridDecoder( + BytesUtils.getWidthFromMaxInt(maxLevel), + new ByteArrayInputStream(bytes.toByteArray()))); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read levels in page for col %s.", descriptor), e); + } + } + + /** Utility interface to abstract over different way to read ints with different encodings. */ + interface IntIterator { + int nextInt(); + } + + /** Reading int from {@link ValuesReader}. */ + protected static final class ValuesReaderIntIterator implements IntIterator { + ValuesReader delegate; + + public ValuesReaderIntIterator(ValuesReader delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + return delegate.readInteger(); + } + } + + /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */ + protected static final class RLEIntIterator implements IntIterator { + RunLengthBitPackingHybridDecoder delegate; + + public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + try { + return delegate.readInt(); + } catch (IOException e) { + throw new ParquetDecodingException(e); + } + } + } + + /** Reading zero always. */ + protected static final class NullIntIterator implements IntIterator { + @Override + public int nextInt() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index 3572b117a6313..1826419db5d44 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -18,7 +18,9 @@ package org.apache.hudi.table.format.cow.vector.reader; +import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.table.data.RowData; @@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,6 +42,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -46,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -53,7 +59,6 @@ import java.util.Map; import java.util.stream.IntStream; -import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; @@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable { private final MessageType requestedSchema; + /** + * {@link ParquetField} tree per top-level requested column, used by + * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List, + * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for + * nested types. Entries are {@code null} for primitive top-level fields. Built once per split. + */ + private final List requestedFields; + /** * The total number of rows this RecordReader will eventually read. The sum of the rows of all * the row groups. @@ -158,6 +171,20 @@ public ParquetColumnarRowSplitReader( checkSchema(); + // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across + // row groups). Only columns with nested logical type get a non-null entry — primitive columns + // still use Hudi's specialized ColumnReaders. + MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema); + List requestedRowFields = new ArrayList<>(requestedTypes.length); + List requestedFieldNames = new ArrayList<>(requestedTypes.length); + for (int i = 0; i < requestedTypes.length; i++) { + String name = requestedSchema.getFieldName(i); + requestedRowFields.add(new RowType.RowField(name, requestedTypes[i])); + requestedFieldNames.add(name); + } + this.requestedFields = ParquetSplitReaderUtil.buildFieldsList( + requestedRowFields, requestedFieldNames, messageColumnIO); + this.writableVectors = createWritableVectors(); ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices); this.columnarBatch = generator.generate(columnVectors); @@ -340,12 +367,13 @@ private void readNextRowGroup() throws IOException { List columns = requestedSchema.getColumns(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = createColumnReader( + columnReaders[i] = ParquetSplitReaderUtil.createColumnReader( utcTimestamp, requestedTypes[i], types.get(i), columns, - pages); + pages, + requestedFields.get(i)); } totalCountLoadedSoFar += pages.getRowCount(); } diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index fdfe5d6fa3a33..1abc6ed56c0db 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -26,12 +26,16 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY; @@ -252,21 +256,115 @@ public TimestampData readTimestamp() { } } + /** + * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard + * timestamp encoding defined by Parquet's + * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy + * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations. + * (The older INT96 encoding is marked deprecated by the Parquet format spec — see + * + * LogicalTypes.md — but is still supported here via {@link TypesFromInt96PageReader} for + * backwards compatibility with files written by older Hive / Spark / Impala versions.) + * + *

    Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a + * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use + * {@link Int64TimestampColumnReader} for batched-vector efficiency. + */ + public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader { + private final boolean isUtcTimestamp; + private final ChronoUnit chronoUnit; + + public TypesFromInt64PageReader( + ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(realReader); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + public TypesFromInt64PageReader( + Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(dict); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + @Override + public TimestampData readTimestamp() { + return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit); + } + + @Override + public TimestampData readTimestamp(int id) { + return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper( boolean isDictionary, PrimitiveType parquetType, Dictionary dictionary, ValuesReader valuesReader, boolean isUtcTimestamp) { - if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { + PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName(); + if (typeName == PrimitiveType.PrimitiveTypeName.INT96) { return isDictionary ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp) : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp); - } else { - return isDictionary - ? new DefaultParquetDataColumnReader(dictionary) - : new DefaultParquetDataColumnReader(valuesReader); } + if (typeName == PrimitiveType.PrimitiveTypeName.INT64) { + ChronoUnit unit = resolveInt64TimestampUnit(parquetType); + if (unit != null) { + return isDictionary + ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit) + : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit); + } + } + return isDictionary + ? new DefaultParquetDataColumnReader(dictionary) + : new DefaultParquetDataColumnReader(valuesReader); + } + + /** + * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the + * column is a plain INT64 (not a timestamp). + * + *

    Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and + * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} + * encodings. + */ + private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { + LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); + if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + LogicalTypeAnnotation.TimeUnit unit = + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit(); + switch (unit) { + case MILLIS: + return ChronoUnit.MILLIS; + case MICROS: + return ChronoUnit.MICROS; + case NANOS: + return ChronoUnit.NANOS; + default: + return null; + } + } + OriginalType originalType = parquetType.getOriginalType(); + if (originalType == OriginalType.TIMESTAMP_MILLIS) { + return ChronoUnit.MILLIS; + } + if (originalType == OriginalType.TIMESTAMP_MICROS) { + return ChronoUnit.MICROS; + } + return null; + } + + private static TimestampData int64ToTimestamp( + boolean isUtcTimestamp, long value, ChronoUnit unit) { + Instant instant = Instant.EPOCH.plus(value, unit); + if (isUtcTimestamp) { + return TimestampData.fromInstant(instant); + } + return TimestampData.fromTimestamp(Timestamp.from(instant)); } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( @@ -281,10 +379,10 @@ public static ParquetDataColumnReader getDataColumnReaderByType( } private static TimestampData int96ToTimestamp( - boolean utcTimestamp, long nanosOfDay, int julianDay) { + boolean isUtcTimestamp, long nanosOfDay, int julianDay) { long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND); - if (utcTimestamp) { + if (isUtcTimestamp) { int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND); return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond); } else { diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java deleted file mode 100644 index 79b50487f13c1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** - * Row {@link ColumnReader}. - */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector; - WritableColumnVector[] vectors = rowColumnVector.vectors; - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowColumnVector.setNullAt(j); - } - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java new file mode 100644 index 0000000000000..0f5e00779a2f5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +/** + * Field that represent parquet's field type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetField}). + */ +public abstract class ParquetField { + private final LogicalType type; + private final int repetitionLevel; + private final int definitionLevel; + private final boolean required; + + public ParquetField( + LogicalType type, int repetitionLevel, int definitionLevel, boolean required) { + this.type = type; + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + this.required = required; + } + + public LogicalType getType() { + return type; + } + + public int getRepetitionLevel() { + return repetitionLevel; + } + + public int getDefinitionLevel() { + return definitionLevel; + } + + public boolean isRequired() { + return required; + } + + @Override + public String toString() { + return "Field{" + + "type=" + + type + + ", repetitionLevel=" + + repetitionLevel + + ", definitionLevel=" + + definitionLevel + + ", required=" + + required + + '}'; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java new file mode 100644 index 0000000000000..f91dcca965d64 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's Group Field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension: + * entries in the {@code children} list may be {@code null} to denote a Row child that is absent + * from the parquet file but present in the requested logical schema (schema evolution). This + * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees. + */ +public class ParquetGroupField extends ParquetField { + + private final List children; + + public ParquetGroupField( + LogicalType type, + int repetitionLevel, + int definitionLevel, + boolean required, + List children) { + super(type, repetitionLevel, definitionLevel, required); + // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for + // schema-evolution missing children in ROW types. + this.children = + Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null"))); + } + + /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */ + public List getChildren() { + return children; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java new file mode 100644 index 0000000000000..f6af6f9ff479e --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.column.ColumnDescriptor; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's primitive field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}). + */ +public class ParquetPrimitiveField extends ParquetField { + + private final ColumnDescriptor descriptor; + private final int id; + + public ParquetPrimitiveField( + LogicalType type, boolean required, ColumnDescriptor descriptor, int id) { + super( + type, + descriptor.getMaxRepetitionLevel(), + descriptor.getMaxDefinitionLevel(), + required); + this.descriptor = requireNonNull(descriptor, "descriptor is required"); + this.id = id; + } + + public ColumnDescriptor getDescriptor() { + return descriptor; + } + + public int getId() { + return id; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java new file mode 100644 index 0000000000000..7cb62824e8543 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector}, + * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's + * nested-Parquet reader (FLINK-35702). + * + *

    The accessors are wrappers over the existing public fields so legacy callers continue to + * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the + * Dremel-style read path is exercised end-to-end by integration tests in + * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes / + * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes). + */ +class TestHeapColumnVectorAccessors { + + // ----------------------------------------------------------------------------------------------- + // HeapArrayVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapArrayVectorAccessorsReflectPublicFields() { + HeapIntVector child = new HeapIntVector(4); + HeapArrayVector vector = new HeapArrayVector(2, child); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector replacementChild = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setChild(replacementChild); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(replacementChild, vector.getChild()); + assertEquals(2, vector.getSize()); + + // Backing public fields are kept in sync — preserves backward compatibility. + assertSame(offsets, vector.offsets); + assertSame(lengths, vector.lengths); + assertSame(replacementChild, vector.child); + } + + // ----------------------------------------------------------------------------------------------- + // HeapMapColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + + HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values); + + assertEquals(3, vector.getOffsets().length); + assertEquals(3, vector.getLengths().length); + } + + @Test + void heapMapColumnVectorAccessorsReflectInternalState() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector newKeys = new HeapLongVector(4); + HeapLongVector newValues = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setKeys(newKeys); + vector.setValues(newValues); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(newKeys, vector.getKeys()); + assertSame(newValues, vector.getValues()); + // The Flink-2.1-style ColumnVector accessors return the same underlying child. + assertSame(newKeys, vector.getKeyColumnVector()); + assertSame(newValues, vector.getValueColumnVector()); + assertEquals(2, vector.getSize()); + } + + // ----------------------------------------------------------------------------------------------- + // HeapRowColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() { + HeapIntVector intField = new HeapIntVector(2); + HeapLongVector longField = new HeapLongVector(2); + HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField); + + WritableColumnVector[] originalFields = vector.getFields(); + assertEquals(2, originalFields.length); + assertSame(intField, originalFields[0]); + assertSame(longField, originalFields[1]); + // Backing public field is kept in sync — preserves backward compatibility. + assertSame(originalFields, vector.vectors); + + HeapIntVector replacement = new HeapIntVector(2); + WritableColumnVector[] replacementFields = {replacement, longField}; + vector.setFields(replacementFields); + + assertSame(replacementFields, vector.getFields()); + assertSame(replacementFields, vector.vectors); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java new file mode 100644 index 0000000000000..02fe1e61ccb05 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.columnar.vector.BytesColumnVector; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ParquetDecimalVector}. + */ +public class TestParquetDecimalVector { + + @Test + void testGetDecimalFromInt32Vector() { + // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true + HeapIntVector intVector = new HeapIntVector(1); + intVector.vector[0] = 12345; + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromInt64Vector() { + // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true + HeapLongVector longVector = new HeapLongVector(1); + longVector.vector[0] = 1234567890123456L; + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + DecimalData decoded = wrapped.getDecimal(0, 18, 4); + + assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtLargePrecision() { + // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path + BigDecimal original = new BigDecimal("12345678901234567890.1234567890"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 30, 10); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtSmallPrecision() { + // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the + // dispatch must fall through to the bytes branch rather than require an IntColumnVector. + BigDecimal original = new BigDecimal("123.45"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalThrowsOnUnsupportedVectorType() { + // A large-precision request must have a bytes-backed child; any other writable child is an + // illegal combination and must be surfaced via Preconditions.checkArgument. + ColumnVector unsupported = new HeapShortVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported); + + assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10)); + } + + @Test + void testIsNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + intVector.vector[0] = 1; + intVector.setNullAt(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + assertFalse(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testWritableIntRoundTrip() { + HeapIntVector intVector = new HeapIntVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setInt(0, 42); + + assertEquals(42, wrapped.getInt(0)); + assertEquals(42, intVector.vector[0]); + } + + @Test + void testWritableLongRoundTrip() { + HeapLongVector longVector = new HeapLongVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + wrapped.setLong(0, 9876543210L); + + assertEquals(9876543210L, wrapped.getLong(0)); + assertEquals(9876543210L, longVector.vector[0]); + } + + @Test + void testWritableBytesRoundTrip() { + HeapBytesVector bytesVector = new HeapBytesVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + byte[] payload = new byte[] {0x01, 0x02, 0x03}; + + wrapped.appendBytes(0, payload, 0, payload.length); + + BytesColumnVector.Bytes out = wrapped.getBytes(0); + assertEquals(payload.length, out.len); + assertEquals(0x01, out.data[out.offset]); + assertEquals(0x02, out.data[out.offset + 1]); + assertEquals(0x03, out.data[out.offset + 2]); + } + + @Test + void testResetDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(1); + intVector.setNullAt(0); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + assertTrue(wrapped.isNullAt(0)); + + wrapped.reset(); + + assertFalse(wrapped.isNullAt(0)); + } + + @Test + void testFillWithNullsDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.fillWithNulls(); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testSetNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setNullAt(0); + wrapped.setNulls(1, 1); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java new file mode 100644 index 0000000000000..9d6607d03febb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.flink.table.data.TimestampData; + +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when + * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702). + * + *

    The factory is exercised end-to-end by integration tests through + * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece + * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each + * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} + * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values + * using both the values-reader and dictionary code paths. + */ +class TestParquetDataColumnReaderFactory { + + // ----------------------------------------------------------------------------------------------- + // Type dispatch + // ----------------------------------------------------------------------------------------------- + + @Test + void valuesReaderDispatchInt96TimestampUsesInt96Reader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MILLIS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MICROS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt32DoesNotUseTimestampReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + // ----------------------------------------------------------------------------------------------- + // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches) + // ----------------------------------------------------------------------------------------------- + + @Test + void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_123L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMillis), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + assertEquals(0, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + long epochMicros = 1_700_000_000_123_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMicros), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMicros / 1_000L, ts.getMillisecond()); + // 456 microseconds remain → 456_000 nanoseconds within the millisecond + assertEquals(456_000, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + long epochNanos = 1_700_000_000_123_456_789L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochNanos), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochNanos / 1_000_000L, ts.getMillisecond()); + assertEquals(456_789, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMillisFromDictionaryInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(epochMillis), true); + + TimestampData ts = reader.readTimestamp(0); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + } + + // ----------------------------------------------------------------------------------------------- + // Stubs (only the methods exercised by the dispatch + decoding tests above) + // ----------------------------------------------------------------------------------------------- + + /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */ + private static final class StubValuesReader extends ValuesReader { + private final long fixedLong; + + StubValuesReader() { + this(0L); + } + + StubValuesReader(long fixedLong) { + this.fixedLong = fixedLong; + } + + @Override + public long readLong() { + return fixedLong; + } + + @Override + public void skip() { + // unused + } + } + + /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */ + private static final class StubDictionary extends Dictionary { + private final long fixedLong; + + StubDictionary() { + this(0L); + } + + StubDictionary(long fixedLong) { + super(null); + this.fixedLong = fixedLong; + } + + @Override + public Binary decodeToBinary(int id) { + throw new UnsupportedOperationException(); + } + + @Override + public long decodeToLong(int id) { + return fixedLong; + } + + @Override + public int getMaxId() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java new file mode 100644 index 0000000000000..a071ef937611c --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link ParquetGroupField}. + */ +public class TestParquetGroupField { + + @Test + void testChildrenWithAllNonNullEntriesAreRetained() { + ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1); + List children = Arrays.asList(c0, c1); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertSame(c0, group.getChildren().get(0)); + assertSame(c1, group.getChildren().get(1)); + } + + @Test + void testChildrenMayContainNullForSchemaEvolution() { + // A ROW field present in the requested Flink schema but absent from the Parquet file is + // represented by a null slot in `children`. The group must allow this (Hudi-specific + // extension over Flink's ImmutableList-backed equivalent). + ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List children = Arrays.asList(present, null); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertNotNull(group.getChildren().get(0)); + assertNull(group.getChildren().get(1)); + } + + @Test + void testChildrenListIsUnmodifiable() { + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child)); + + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null)); + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0)); + } + + @Test + void testChildrenListIsDefensivelyCopied() { + // Mutations to the caller-supplied list must not be visible through the group. + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List mutable = new ArrayList<>(); + mutable.add(child); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable); + mutable.add(null); + + assertEquals(1, group.getChildren().size()); + } + + @Test + void testNullChildrenListThrows() { + assertThrows( + NullPointerException.class, + () -> new ParquetGroupField(rowType(), 0, 1, true, null)); + } + + @Test + void testEmptyChildrenListIsAllowed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList()); + + assertEquals(0, group.getChildren().size()); + } + + @Test + void testFieldMetadataIsExposed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList()); + + assertEquals(2, group.getRepetitionLevel()); + assertEquals(5, group.getDefinitionLevel()); + assertFalse(group.isRequired()); + } + + private static LogicalType rowType() { + return RowType.of(new IntType()); + } + + private static ColumnDescriptor descriptor() { + PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f"); + return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0); + } +} From 532edf360a4bf4d2e39b747e6e0748c50857bbd0 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Tue, 26 May 2026 10:28:30 +0800 Subject: [PATCH 021/255] chore: Fix stale zip file for variant backward compatibility test (#18815) (cherry picked from commit 7e5b5351bbfa4fc09a64b19319c047600360f611) --- .../variant_mor_avro.zip | Bin 87511 -> 104069 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip b/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip index 59244af7dd22612e0873e208a652b22c2e1668e0..610188de288bf6bec9576f65eda7445394c52a17 100644 GIT binary patch literal 104069 zcmce;1$33k(ls0-?(R<9J;dEziMtzdcNZeWl>l+#?ry~0jkvqp7iR8dCigPDBlrK$ zS`VC%Mb|#nUDdm)cOO{^U=T=vZ?6NTDve*>{Ob<{0Bit9U0YLKD|>BAYg=tyM_X%J zB}E7T;Ffkcvyv`2GY4lV06>s)KmY)cZ~sU3*-*${2)EEr|N8Br`_WJ|G$z*82BwDE zHoCUKeY5r-mY#n7G%66lTqLXfbF03<>mtZczx(!R z{@5y`5j#7RF1s$Zz99<(H4~ezKD90#9UZkH9X-R-PYk+@jE2;7+6;6IEOg8aPk%A7 z($TZCveH;s8`IEh(@`@rQZunXgLwbP5ScS}m;u4<_}s(S?Vm$^LiqiK@k6M;`sX*F z5B^_1|32Q+o0mTulE&1^z|dLSz|hF~#z>Y2~({@7`qHHuI9AymE$YuIIalDc1B zRKGg1m)WV&q6CC~QGh#a9LmzH<&}M!RBc(OP=6PS-aL!MpUSB1XxpYAZSD^I+#9>5 znh`2M006fy001a{=8az$5RHwkwT)=1uUh3TtQjV*v5FJWws#BlSvm35sC^s^3)1u3;r5#yF?4Z{@;-uRW{ zgWtApotQD9N9nLN;9hv8r9KSLr<9FC0TnB`!mV6>*&q8lyxu?2uVT2=HS@WEHp>nhB?8*8To(XSh-tnAWHSiAOHN7SM z=a|4Qapca1`>;ak4b4>G=WOyXwD8`!|4%COk=wbrp zSI2Hm1}^8(ujw2In2bkXVZx$o3gV=`F0BEbB7mL4XIsZd^y|r)k^(qrU|Pv=e+}-# zkTB_Wa1HV4GdwEu!DC@{W&WZEhdy&c-;mVQtz*Vo^FuLqq`P&WLoSeIP<6wdP>83j zwBlYz_iW}AVWa9goha2g+Q<@`3Qc(Q3=gZX99IJRq+j}aw|T^Jp-**&H<^ysj@E9dJaFKS+4C~cwl zXb-%ZL@HFCdr5c>i;>Kr`+m3b0M&{@#fG+-qF8wc zQ_tPz<2*yjhKZI{{V{SZ&<80;6R|>?v;HoxVjA8Z1DJTUH02iA3P{%zd9lv2t6&v9w=tgY?s z>}_>_2mLRZVE!v6Xuj{9zsur($x{q}$()g?g`ugz-?c^XQ(JcWCWe-}zfH_v`WS>i z8`|E~($K=x%J8?bf3RIYUrcc5#|&go+s(H}`A>~a^EC31KihJ{B_ZPjuFLgLS{f27_VzilyeTAhU&bR#3NHmH!SPtDOVd zJC^N&{c^gAn^oi7Djj`Dmr7`oRKR}4_+U{H32vZ_JQ{pE(e#}kR@D;JiYzBuk%STs z3fp^~$z34q;TqB*1)g%=5Tbk0F5(+b>qL3v=1V&NysRkY6h=RmdjxH%%1Y{N?e|M_+cvq6D_LPMZb<&db8P)pcSdNSO6Fu0GK3 z^f=O`StT(wAYM=bsgQ^uDrO-w0JalOVdeDuh;C+~Gku(wc_k-xB#r0)>BzY_H3>8= zW){JoW29-SCek*7e2*&{63HMI*^A~AG$(V8SkEYO__=p@j5vf;p3q@Q7m^Vzk{=wI zN1A(q>X!-FEv4qe)%={j@L6EIcBWUa%gUup><-kgjy+sriEiU@0_@M=irVd5?xK zD08&T5>9r|Yt0m#n0p2#ZB{P8E1+veTW}~Fnr2xayNgiow!Evrj|^t@5~6bN55Rrj zq9xB0fGl`~^4hb#Em`vF1WA?ExCh6Az28MA-*P3TP7PBDZ`_iX()SYNyT=9+!=oD5dNVMHp3%cJmSSrcicd1z!g8S%MJ@YtDC7 z7aHFMW0RU;IP~IPe7*)!KB9LA;qgwZNTS;wOR9LyRytJMeQ{pfdwsRj;RYEJQ%IIb z5Ts3O<=zNBVtoq`hpWqPozC0&3Y>8LJ4jbmqqc-B|h zVpHtbH!-F3AX7#1pUGjijQ1eA_Y7@s0F?}4io>(`4}H>QND$(=46VoM1x?0ss|<|i zM(wRl*@%x8YB>Bsh}rV(4JE26ZxC7w_nmgM%%p`_`7(oI6a4lDKwwWHO;DgLz1@_Z z?J5r+BdO8u&b(oP;H_pE5@b#8ihx>$Or#FbeMbm{@VT&2_AaXu0ZP9PQcRbe7gNYb z1y!OB-l426v%ig(n9^yz2tb$Uh3_b;Y^hb2bq3@)u)L<*tw%E!7iaE9O)(;=2zJu~ z=!Hl5v;=}`*F49i#jCWfo>opSghc$bpM86z{?y*^-+S5*SNlKR%3_Ax^PWR}Liqh9 z_NSme+s0VvnAn~nwKaU>U}$G=Xz;Ur?0IVVM6f%j4GaL_0S^Em_xB(?Cjcz0>wilRjxz+;J@QR%KD?}Y7I@!lWTydoxu{BkpoZDW@;D`ix71nL0mXTi>UZeui7lCrlXk69B(asu@RvB_ubNCvee_e3 zMg4XA`1M@Pj=wrCkG$xjwvx*cUZM}vj|0B+HxymwcW@{47T{;8`D%z39N9K-L{A6G z`Zd5tCz3m(-wWM@@MFBt(QvAC>uSI4>{pNBa)~Y9NiRjkf)%Oaba^QYADJ4p*dr~o zPwfrm+?&w#D|?@*{a5E&uY^d&UMlJs3&2I`Xzb-tj>4>hZpH#bwA`{%z9OR+ehm%R zu0dPO2nLicQlSZ9%VQ-H;K!;*&=C-1XL!LL9E%)^KU=zSKYSb|1s@Q=Q6`i71_kXg znPH>eeVHUX0rKT1h%O}!3i;UlNm5RGJtNx8eMWGlBL-;b9QynEUqR)8KR;z1C-rx!))kP;0 zg-KBmzY6UO_s$K2_GR?x;5@LOatj&SeAH@sw0*E+yf$8SyFLgyxlH;%jJ!@ZMS*-G z>s{f)F=Ouu7tXX*moCip4yBN4%{Fl|&RWcpzE4M<|gwxqwI>}C4yy@1q zC0C=m?0B%x8bOmbYiDE);+OlE>Ex{gs|cb!GFxXf?BVEw9w$tk&=0eT6va5D^3G_9 zFJKc~$ObTzx`4}D^Gc9JOIa`&}nDilNbxdpv#4lxaXWcbEKu zhsl9YfhK7IMOC1V>hg%WL}s5}eUYMBbfE$IDCe$8p*+XnJVME^c4n*y z4OlRl`7_nV_hj=!{@!do#)~XX9YxXFui+y7p~DoYcUoTXQ~U>F zY5^1>+6xKW^*zQ-yK#9#9bFwgLtR~ceO(V3d7_C8l5ILj;Ln$gazFjt{X69%ho(*;2F$_xf0gn8pCsSX6l5?)@W zomw3Vq^|ncQHNap7w%yOz}+n6QR3~ed}c1L>w9KfBQI!z@%v3n&{DxaL&2w4KtrT% zzm<7uBiI9~0(;?eL$(lJ=VMnB3q0z>wTL(5g-jMN7bN>xt(@H17b%Cb+b=0i0Iy6= zMW{H4*3U?-x&)XT#Z{;kY`YX9Y;hyiZZD|5ycGVeHP|j|qu47$L=L1_A=V^dz#)Oj!WXp}_8&%DkzC+o+S(h-2w@psW)b!{ ze()%Mg?oZ7r|L@R7Mn!-?&~GSW)o$g6{2e%z4)9?lccy%h8oS`c_*(gG$OJ2`GZbISnWeci z`MD1Et&c>Q9qF)TbVwF8s=xytBBIN_C~u1|DpRJneetk|>mZ~VtTM%IVqL2P{g6~b z<|$y4`;dK>Z9Wq;KZZf5hR%;*GJHNW$DuH($D&UCNZB=NX8{W;=9g5C_F({B$Y!JP zg%&uhKHAZQRbQXjQX!dwSN=w>-aFWkx2q@m_#1>l%PG9NR!;RPe&gpUv>6X~_p0~j z7b`NWmyw6F=Uh?}rn9=43N(lO7z`xw;r$0hbc|7b>kNr~SY?L1nchjM7oO(q?aDOo zXbr>N`+6|4G)AH#7vxK2sS?UhNUpw!WLi>6&*ETkWqI7oLZ_3TqdC4v%iGFfYpTaO zr~KF=TaV{i;jx%b?#Goh@&PW@n@F*VtX#Rtq21!@d=iS75eqIvNe1y zd_L>{8A0E-<{!-XQ)|r%i5^)>gsKhzqEx5zLo!O30h7ycOz;bFEWs@0h?oj|B{_S> zRbnfy)z(!A4>Fz>-wYj}WhB*On7u0?V(t$*ZZS*)YwV^mo#9-U^KKT}8RmG*Lo5|J z!-nHWg|N2EA{T0#3Am$BuLMvHtJ23cZ491j0T$u<@i%llg4h|I$}s)W{J^dNkU&avf~}t|sd5Om1-mvgg|_H~%ABn)P{z8Cx3$nJwBbEr!bJ%jI|gqsWAtF$L*apleB zz5rZtFmUHBV3T3Urx2fhtz1$ z7~u<{|DxxNzTxVU#{S+Jci2&A-E-at63n_oI2a0U>dS#0JUKZsUxj|*ZPP9h3m-Bn zu4uIEE+>s=0t2e^_flZe<(E?63Ua@mzC2<00z$9psT4r1cZWU8oj8&EwG@anz$~Pm z>8)T&Q^!Pw;KX$TMd+)Fr_fv(9|&y{v?0Ikbdag^TaZVqZ3s*=iK(*^?SqPj#IuDk1_vT$5!4g| zQG4VBTtGQEdRZr+klLm7qDw^9rKUTN^Yi<|lmhqT>x}Nl3hhWH>P%Cx8D;JaKh957 z6=`DYTE1~WV@wJ?lE#9Rz%x|hNwb?U`cJh$SHDFHEyLI7Ir_2dG#r6(N}>fig&7Nq zgsHP&pX@}FjP&53E7FR&=B=^uzWb+IKzr-V*cM$*$m58v0orpZ^;<2V;EaY2n$X7? zYZPI>`gUend0I~1LNZWN(U4QuxSu^YR|I5yDCD(RowL}s9Uj&~2O!((oPza5q!ml0 z;4ZO!C3DGKOboB49MQR)mF?-EU1qrOu|uA!r3GMKKn>tw60<(Ug4@DYac&}zqtz8& z@Ufg`KlB^aV@?BWTU7jL^`W-zrTKZuKN8*!Nkm?{1;34lWotZmIkuJ zXYs;1YcwV0MJ1rMBY9mh8x-PEeEfaF4-=hDEySxB=A6Qr2t30MJlQ)km3=fERX%kq zmO%J3t4VeTNl6@=$psJlArF{czMJlD{xo+~oanY+Gv$~;W;NJ^yL6TV_n`)Hd^hJE zLK*}iU_}$pZ*Zqq;AsNcWiN^PBY=X|2=x*;L?>DR`F5^^&o^`jw|t@m5eMt;-r*y1 zAbu(q?gFxcMwNY`0dCu>kf#40kRrKLA0CJ#UjU(|ceCJ}7N}5>2_G42&=sNy*&jy9 zi>X{4UBK!h*UbKFiYcu=8=JSc_(#mPoTLu_^)2T$pzh-XY-$jwd6WBi*53CTx~u}b zqV&`kvkKdRrhXDMKElhhYrH@b;571dRJM%{{n)|wEY~GC>z=bA6uy@`_@4s0-PZe= za$^hb@J{-_Z&-4s4FAr>^3X_)G?g32@=<>YPm7N_btsf&bX|6^9_orJ%O~W+#-=EY2 zgBLXpuU!yq?aPCFXn*5<|3FEy?EjSpu4HAn88+It$Z5Q-*12=q(oJ^vYd z4!Fgb+Hxaulr!Mcp`{D!t{b8;ChbIO$+Ur9rr7*zpbjo%_f`sB<);Ut|5jQ3mbL!U5%o_q|uf)uYX7Fscb`hL#E-8%`ki9602Qh4bmL8+BtE4w=q7?4J*+|9_@j(eh0@x zN4qYQef9n2f4z^+6B+zjgtR$y5UoEM;dTHcenEL6?d4;(x=k)hzy;PcF*dUW8O%+ z-8q|QQyW+@WVqamr&25vSC65d|DhfH{Zjt1cJN=)xc^!T|3o(OQ(@VETl)Jtl2?Ce zb<=z^^pAxl!vDzTKUbOv{4@LiOnUNDnah9A>_5_?F#e?v`khIBr*9Act&4nKl9n%B zA}2ur0Km}$0OldkPN0huR0Z4~R zGgkHcRkuWqlmp>Lqo&P=mpdMDGGI#$HlCv0^7gSDdVu$R1>NzU!K&uPFOw)gO_^2m zV=fJ(${n3<;xVEW>!4yB75-w5ahlW&mzPB(OqfwltqHZUgu5Z}YC*)^kZ z93o8Z(9NPdLTo%|L%^nRx%F0}V@&ad8rntsm{k3GDM5tC!mPkqj-#|L*DUs%-mXaw zEPaG7pHl4^F=}l9E3O@c)0zAl6$F1Q^|)HpmGBO0hSq`MYX1G zHUsTV--{*eVW@<_f?lI?e}kyQNqid=-5B-2mH|Sc7VNjoqlXoVvTy;nW)V!`*XB5K zs_>Q-Hb&H4ZLbzNxeCAZr&$O4z;l1pO{1P}jAxqirk$dujlW5M$rvHQ3>GFe2btWx zyB;OjLCnku_?5G`-a5U?^CFB#((&;khsW!#9?$C#%xmA8 z!5;js&XU{OX=TW9>e>=lCx`L&?%vAjQtI-Y=I(0GvikCRt{R6-I_>Fn;-VnKy7^(= zG!7s&GL`!3GV-y>JLAQ};AA}GeHg&o>ow^3jrssDD#y9e_Q%kjbIkGGiIQ!C6xIcb z^QeN=yVE6@9Rl7;SE!>xMeALny@c3RcRkN+ThcEXn9-5b$O~{-~mDz@PG)>qp{h{!0J4RoIzD*J#(TH>w zJqwPGWvKlGJZ2m#v5n)_Gq@|i$&|5q8pJJzrpzP>CI}rt$44Ql7m%d&nu{zKjncTs zbxcogKS)5%6zGML6@};M`s?dbfb5fxQ7)OaWgm|#a19*r6|}p;Z3H*JEWblgGH6j1 zZ2&${?LIMdMh*`NuK;p)MjoNZf}-59b7^u1$M0XmjQMd=b_B(4qi6(bzWkJruNH|A zUY#76Y_Qg2(80gEPW6R!*>vhGaZ%dBc(;Mf&v#<gri`(Z zcl2EE9Ej*d@(F<~F^E(@)Qrv-!t#LJzbxEZe>yXPc9UcS5yN7X5}wDIxgrx2FhRba zuk*0l3}G80OJklc?*+5lVBT2mym!+1c$xw?W3u=bsf#!Ebc zVa_(@q&?oXeZQ|fa%&V73+6MOtG?kg5UtRhD#M<+DFJJVQ0dydRG%u_(s=XC;$fj? zNjew<%9NXE-TBbymf%}i2r>yYXun1W`@Aj>PxqPgkBYZ#j5YGBY0>2?iNHotNH7zs zV9$OnoD$1|zh*mNZYi0_5k?{e4>>qV?UeWUB&R0YH~~0h#e7-mjfH&irpiZ5Rg;9k z>V@|qQB*|AdD~)vRecr!D1dA6E{@)$>U^A@52JUH#){OBW`WQF$%JRvxZ+NW;oQAY+Q7#o7qsNpog^y9<4|K3eP{Z= zkQ3voI7y8c^EbpsN%}ycZX$1%#2ePPWD{V}NI8#NV4g71Uhj+IdApeRG-1&PzcN(` z;hgtrPdoyM9JaUWWQ^$0u>mmyF`ARzYTy>wp=80*Jg-Kb=nL9KesiFOFd~!HEiiVY zXdx0Qz^0Im2?+&y4t@_L%*e%HVkVQ?IshTXHMDn|n?dLGHmk+dKrDv*JXCWvT$Z`_ z=9M2sRPTgZ;7v~c>W2Jyn5;Uc_+0)8iT*%DTm-Oy*F?m`b!^9cNpDr{P|Xp$v4F-5 z-&Y|^Mp#E+gP2-Ne^6*7P@u`_)+VdVm)PaNTDLJn(7OmaQLGu!fI2gUHJbUCT%1*Y z6>l{(&m)7+U4rgj%~xwrcPrTRs~^2kT0;q8vn!+>lDS#S5xSm*7!65WE}jXTXIoKN z)1d*Aj~Bf~VCS~vG18G)FEW_KhPdu5TnXz9GK2P~z#{Z8CV!o|N>2TWqx|#whFg6k z$c7OafWBB2j0^Br4o8Px5D}cRNDlF-3Y=bwhxXD9#)j!+e=R+WxSeGCuiO}eX>>e^8XVJGPgMH#gfjgmz!>u}=DC;dadj+LyU-2{yF!A<2=+@r0i+=XY`J`RD zAfv~VfJ?|O`Pfw*f9R&>#?uKYf6s{oba0Ftp$uTgp~C=h2wbSH;8#K~2*{kVBZ$EJ zkqLqfhcAh!B4eUrsp*jjpjHu-R=5~u}?e&CVXgwO{Kr8SX+?mypbb>J{eC_{!QAM z(J18&0W5yzmtxGTPizS#nER0$U)dJY8UvOg%6F@4#JEu~>v3Iuku^&fL=&*;Kvnnw zE0<>n-hw!g2iWT*PPm$9FzYyY!naoydQ1s2w{qMQ)U=H?LgF!R#f-mwpON;|3Dl33 zkR-4pWTxhezqwg89@t=NX6Kn z8gxb#0=meFx6hvvX1=7YuioRUrW8F2dt4_bFhm6Ll4c0#4VUVA*53O&+sC7$&8E}+?&8R1sDB4D;|II4$YYk7fGGnqPE7aPLo3 zchH14jF+1~7!I7pIRKwZWiga$HO%tzUaqNehVQqMUd*ED+vNa#60o{A_)cAS-<5; zrBM1T)aziS1Jy=F=*7%%hhu7e2o?4B=bk)WhB7mlm0(Y|+HLI{o=eU~qxMD00l5;8fS4x8tklf})FMWX1v@>? zxfbV`2znAG+2JYIJG}l-;ao63(Jtnm{qucC}&rFzc1)U53*-n=ZYHI!W z#Tu=@w+;PXGV_xd`ltQ%ECOn0=(Hok=f?rRZ%nHFJ^aslFaLEs@LjI(v!2UyOwRB6 z&McmQeS3`m9;V;>^oO#a|7ZyPQ|C@uykkDWQvEzun1nBV+`RD_sq``qSuVH?a8Hdm zeG{1u&l-(zuB1bV3(FlvV9s`QusUve{V4mCrbjSeSohJ&n*NleGw@n`qVy-Sp})Jp zo-Ttwdp+xgJzZ!1^-k2E^}+MW`uzg>n>_3pwx=I{4DZjcpWjPvwEyP%`8lTVi~67Y z?4JkS4|2wLeER$o)Tg4;|z~Y(u zKWwxANC3#p^m~u~Z1pW*_FAp>Oh}#OA*`YI69oCf8}J#t-l1j`rdb3fYC*y}8RaF3 z5DuYw%}nxIP)V+A^(x-ZwwJxzU6AJ54}Wuw(wCSwN@1~pkX-(gDD@=t@WNIVRnvW= ztNQViPL^@g&7M$5q*fnwn5Ws*WcxO_@5iP# zyBvxSG``?2W0^-T{jXN1S8)W`;Bij@Z90+;^Cllu;K)0J#Rr(tRq(73yuI6O^Ptlv3&T zc(IXq0atCg;~hkbgt{Yv8=J%=%+Gts(h~nB7qGOoZDPVxwu2hY-Hw-51G9)UWdJdT zuW8P^Maf^x3I*sefgEVV#v;noxYOU2Y(4?&lWxCuN^qO6NkIUkIeq-(@%`#*UH^P% zAUfm59VlE)80+VuF(Lj7a!{V|BOptAlu8 zUe~)$UJp5oGLeJYcb86&N2y+q4@swYNGexv;5}s%Nz*w%JQtWVoa&E^t9S26;7X>F zydG8zFW2V|QWB0<&xh))>x+5zdEe}uaoamBHO$oxz(rNyLy@_mVS00m|Y5P^tDG79pYs&0{r@+IU>=PX9qVuh;W?TV;kGB00T zN=Syii?Sz`citVDftE?po(X3G*lV1g@dSDz?2%fAD+5Us3wY@X^+s*c#V6&IWF`bCFd<2i(z2ez&XYXWKP#7W z4wgA4;snj&d*Q_=lK}TY$<4Fn848WMS`vhL>@&ji$2I|VRj}720H^z`LHW{ z*a^AwEHhVjHQLDsUCc&~L11Ap{vt z$OGr7mdn;g;J6%va_|o&f{h6v&OMMc#rj_!qf*-Q#@M2sV~AweCE|LGf%iw}2lJ|T zfya{-F*y~q!|{3XQMz8orJ%LrBEj;Fc1e!WJ9V>gi;u!%3UOCkX-~Tl4kC#g!(14g zw5Ga5Tsylw;>>QjllnN*SQ^Axgu?L}#rT(U4XW`T5TmQ~ihwgk+DK))!MlLHzlubz zg|xg>b3FcpukXlD2TY4xfwyBsz`o&|g9Z;}T4<9ao1dK3?ppvU ze~G223HVf)GnJmsB-KabZ6b7~U+w6s$JX6;${36;1TA%!?AO zgBo;MHkf%zQB`_Hfy zekDUhB!x2<7W33vrY#9m$y?3S?h{)&*Ums}tjyxP_ zSj@MWRMKp2pMML`nw+qAtmpHt-L|Gf#%(EEnI@x*7VMBEUz&-DWp zyQmFS0Kx#Fax7vp>tA<)m$ZZE+L#oo6BMr(B##`tMVIqr)~5NS6ha)o%ER#qgByaW>_vE(71tF+Tw7CwZ>Uiby#^xlIDa> zdgyfVbs})~b{(}_=sDIZX|MZBae+ZyZj+$_`>ZV~;8moQ3N-esf6x z>_+AE@^nMAht?WKTargvA760>ew#z07ZOLtlwcvrkeYRA&V!haRR9oOqD->~3x27z zK!SwrAPRV1t}c6DRQVV$k~w3!bCUQBQNKWO>J!psCJfU|%f()n;xNIXd<4ZuP*MA> zK9b{zwie22N&-KLcHaq3?aXRc8JVTPvZIN=2^X3pY7^`boe~r&78o=-$+sn|rdGwx zHW##>hEdD~3xJuk9%gfVy;B-pfyQ3GETzvLv9Cvm<~}{p+)aW!6<16) zxY;O~f&5}ZG)?c2;vG~97Wvm6WOHE@u=uG=~)ZdfU!b&m#w$Wa`!?qG!~qE^8cXo<}oH0(&bYL@3I z4*RYH2>9L-Is3OheJzt8!4l5_+u&vKAgwfL0^T4P>ELK472c0|_s{6! zb4Zmr&M5!G4#BTn`}ws7IyQ#qcdnmq(fkbHzc1PRk-YJPMsn=|!y3s0NV4-CuE@t* zm`wRMOepuKOkc32-_^Do0W~mNxl>0a+h2QlCx_4-7bg|Jfjqs}u7`2AM5WuYGE9KP z-4DEPZ|+xo?u_48;D584{LgCl`TYEiY7&xKxcz_F7W;ur{I7N`{%BsJK061|8*Vv2 z_v!D8&Hq6$_XGUjo%;8RIp+nkLX4+k4w&|DYo`8d(tjTSEX+)749}Lq&ldjzX1&$w zmx%%jaCtCtFp3iZgPE{}l(sZ&G|G<4*(mMKh*FGbBJ-kN3_T;V8T$E`q6uC-)@yj9 z+2Tof;87GpQFcaw$S>tTrPsEY`BVy~Po-Y6udOfNIu&M&md0q}gsVzE-mFwwY9UT? zn^B^QqEvBM^6B5*i+zbISHS3=cW;M=$;*5wk~Zv z4!aAJ=oFUtKtuGQ(?SGdaKAY51O-i;OO!9M=0OK$P9yjWU0|s6!Tfn-rZG^?;Uk2B8+vZqRKfHA$1;Hji>X z%&xo25Hc!hM>V!nRQ+v6?uN#~J}TAE!9KAFA3sQ=?ah#>nY5_v#wSR4sP{^)$`7>a zjlJW%->=FXsukkthfEYs z4(%=73ta8tRe2hF;UE`#aL+%QJ+5xHU&}c8%NF7hq$n3uf;~R;uJFh>!_n6CIqMI3 z9ev4Iy6yvCUH1GcT>5yoqAB3mm-=42p?;hS^$P&++mMeEOgQCHk~J7_`1*sR!x?Hb zfz_42B%Ovdu5G;TrRpr9nVohELV_ujSsR8#GH!FG?ynTS5fX4!<-p%j3S+Mz4J?y? zff+Na=&n7Uc9(d_a4pyKVOXPH%mnknWqPw)h5TC?Rdr<`$c#ZpBD(daH3h42Y4lZG zn=B2}iVZBv+%lm(zJkC>rXUgpf1fDY0symjqc)$jiJl>}8oeZQL4#KH3;FBTq2a-c z7T|_3?G+Ym%(w8=WGh0XYL%Vkk}{#6;WI$sNjnwvkuqT--VLC)GU?~i5tE9L5Rs(D z+0kZrX93U2`-A0SWrco#HmZofj>Lne7-U@b%}4Wv?>KKSsQn5}Z}Qa}wF4dwu7M~< zS8!w@8#aM7G97$|QJ8NeetAe90aIwQzG2A4OM)4*B_~zLa9vT@l*&S)Iu8t0RLNOK z=Ma%Dc4hYH{p>ybnY!9YU7am057Un0d zs&9gx%Bb5kw!;P$-i9ca6LFSSEPTdetG%`v(VjGr`!8NEM4%v84Qv6@ouI)U14ydd zm?;gBB8?6Di9i^S`1#!0n4o&;(JX!yENT8rutd{Q{!Oq{OeB^d`y^O$B*=XEBv?w4 z{3ck+444audix|;a@4I5EWw|a=I@+*s-iT%RZ%aqaDJ(xc99%~7VP2ueWzX503$r3 zws`abBj2PgkmUi-XO*Yv*H2pysE|f>P!Fk)T4b{B8FH*FgzuXq3ZcDK4SNvg3@#x60S5k<1HYYaX_CzTYb`vq< z5yUzl*&@=!%1j01j(h4r5;h6wY{r6c#P`7Y=P|Pde$^{g;W8R(R6`YkHN;6G4{?6e zD-AvAl@j!N1}#!0&!sb}vz@??Cn~Ix(!VHTlj`ki-m@5q?wn>@AV>p?kIn^G0zR;z9BxVs>PI9?L^lU%8oG%wDq8Mxn6E9b zLW_66&=KG!B-pG-fL3*$H~lspDwtdyCJWfl)!z{?YX%oXdWUfEE2@9CuF0hb(@B4I zDRPY@wirW}R;hU^Qa;J+rFB_G1IWtwmFx14Pim#f-_%O&5pk!$5?%`tN?zi6nR9D% z^VBr%Hx|K^9HI@*v#EY_05b*$n3;g>{yyH!t~S;x9=>!OE0u@#PbCx}==He>0{~Su zij%~-RK23C-7eVu^4`j(-@GDy4Ai+y&!%`JTKyrIm&QF$pu zWEgbTV%haVo?xTLsk{)$udjC{GLGSIpgGC)bXduTDZT&2(X#uQId|~G``BXto8fB-@)W)dUoe52StDt%^ z&);5Z+Oo%~Bxu&IwuI8X3_!&bz2LVm7r(`J^oE4V{3cc+l?IM~ZIL_gmzsBK)RcOC z@tJXG`l#e;HfDTYu_>A*MpsatJ4@_-k8KV=5rK?B^-gj^;-W(KE*u&v+aG!zJUqEPDQZRNFhdDIWF&2Q%=X&U$fx4bU1NKk#^qf zx$Z6~@=gD50C?|sn3sh|G&yT6wgI_aIa?S7dUdwNm7OQTER~)gTnA3L zrWw?>Nw5p8AL#_W!t2}*)&gkUl%sa5&6R~rM8Q0z;ji*FZ7o^TEFXne4Y0{9pcF1R zi5DTg<9L}3J>aj`fAzUs?5cr@xomCwq>m}Uba_z{T`Ku3UYX$d{ zS@%cob5<@Ey!zVJ7YO&YX$q=vQz9B=(dUD+s{U_u_Ea3@`_R1ABVGI5tEdhfKvufq z4T}P@oA|ihHKJQWV)<$K2*S=!9lE@uPZj4i6Q{M6B233X2;ALBYj(z>jX_ zJr4sydqY3bRTBm^*&HTx@*v80MN9WmiLLWrd1lmI_J#V2ZO1$Yp0~aHIECiKDoFwU z#A@LjfOlqe-m9|bFxl=abkc=SqV7F>rn97iIi*YAP$vrBt%ZuCF|T`d07chb;ZoEH2x79;KeGGQSSRZKphfGwB zp)TIP9AI7!bLczGYC`Lk{P+v4t@7PCu45-gpcfmE^k$>oJ($~!(7xM-;@i?FGmyGwEa%N zPe&8haMXNv#ktzme9cxc8AU7Q>RC(XI%l)j8%r=ZPZt0@?T!YvFBb%xBc`V5$-A$o z?<76g=$b8^PP7LNO3cysl*p?fELfiI03e}Au%R$L-2o78RttVDox-zOK_BaB(S3U8 zb(x>}H7jI=@X@~LdLD_!8yj9Cf$WtYE`}Y~8o`Kwu>6;`&{vA$aNuCVjqpyyD<~L zMDlM0YJ zxG9*q7gR~eWD7%>0uR&#Fcr+UtzCx*s3g>=Y-a7eY9RUxGLJA1R>18QGEbYnET-a- zafQP*zFNgTt0dg%XL!7rY!w>}miVa8z9!7VbSq|cEU_gRU%rj^FqNZ0>X8ozlz&czq*4z zlTaHOq@8a@zeuzJk6|&wgsbJKIF^eE940%db-3-{KzgGk`^vlo?H!!(09JqZ>FwdL z@FoXDhn*==fq}j|0K+@t@csYC+B=44wxw&^v0YJB#kOtRHY>Jm+qP}nR>iiRicv|v z)aqWldv))$hj4Vk?PfE0tOrU!ri~WM;tdcVz1c5uxiPxlN$a0im%$IHD3vj+iBj z#KX0@t`@4tPph$;-&uI!X4sakGZ~(3-*MvifH`6R0}GEH2T zn$N<7#G-68EgXeei3two)qq(E?GqAxAJOVT=T`5bFRZlzcnDg<1j`a^wCF)2;t9P3 zI)sWo_{-q>q4hYN%efnue0h*J?}jJev$sLhj$_Hj&DGNVESs&zIy=$AA?EezRkx=u zwsiN$&KgZ2h7-x=KuudK{eE~V^t$0wzC7)qb{%YsP485UUkiqaD&9Hq-edb+ybotm zTdav4!QNq0AoU=2DJ?>&dGKvR!MA|OqWb8I@!G_+%X2QI#t)MiD&lDd~B^dL`|Hd22saEwj#ga#E5r$B+v55(YTZtw;m*@K|>ohO~=FK zZz&a175bDgzli3VIh<3e(>F*4r9&664CbCA>f-5r8xjz1;-4h?V{u~YS8-xW>T9tK zJk&=5b+*k}?VLWa3AHUzPuigEm(>9ssz+<$cd|4Y4du=PmW&5F4>@E7e237EG6^d4 zi~L}G`SQ<-7<)xj>Adr}oL7X8oU5of!IJ2m2oE+2-0Y}dK{!G9Tk9?rK8;)~=P`n+ zB#n*8rnIVNV2URZ36;2Olvt6-r7W{ixDr~l-ITH)8%z|+@j@p8!$4j{Z6SggZ8X9L zwtH+?r)OzoX=iF^bZ~dEuyS*=W_sctdHoU8{_~dl{}F!wlXUZEoc7N)jlah957CBp zW*hsj)2jcQ6t&;k_n!^vSJcw4e3A;1(%Q4!kFyC2t3W`4rDTZ&3K4T85V8M!Isq!; zG2)GfJEx88^x}o|)$iZ6YL?42jvMV>o=`cY`?#z_3eg%Hc+KAo5PC0RoG-$GJ?qfbIKeHp}jPv&2fX~9Z@@qD#| zsbfjZNo7JZQy`n6oW&Te@q&5F!NGat=NRG^aR_1Vyoq#XFVRl3K-7wh#Yr-S#s~?- zkTzy_nd-ix^Bn(BWHdHl&U3^Ku7)O6GXgev~N zH2?im{!wU0xI;^hnKdD@40I)I6G5>!X+CgY{=(O`a=@~ixFbf3^ERSVNUkIJ9+b>; z#oJ=uq1vph!}UU?yQa>v$kiN(f14j-f4N(3aKZJ`Qs^;X4c zW(Z63uEx^>L;pqa;dxNq8{P#aAip-s?{xcOz>DQqJC>V2vDr7T&cRopeA-OLH28Aw z+bAKZ^!07i5bAFj0AFm(b-UhEAq9limQk6^f0}(I9p1@cnjAsIcZxip9%fW&cE@Go zlL6^8rM!|7O=k19a#Wi%@{rz<@N3D`PnE>|#1>H|;Uf@^bY=zQ=dyBc8P$H-fo4VC z<@L(q`hIm7@o+Wy-gtS58R%OWu1!2hbnh{Bla=<=$^16-4Q(;Puw%OT zqs_+TZ$ak&$S|dwGXM4$7Wn>l5B+m}l!mR7ipwQp6$vjR^R^D!_&l z*nEF{#+66zgZ`z(w11Pde}AC-U*}+PmIz)X7H{s4<8KSD-!lGqQ2)CAUw!U>B@nd# zyqHXkjCAx&e_#^-$7=13QbXqT3@Vac#bDDiv09JukmGl%ZX)u6zl@m+$Uh}N^(ZzF=gSSL>NzG9^HOTU*g}-HR1@mss+Vh>C$(B2w(u{7RreH@-BJDy(RDI z6+* zSD^Qw-sr}n@H5djQ4ze#UHc-)v8i5W&pJCVs?E<*_!{St#vB%(lnX!V@l?A8zYQXslXr!Q{ZgL)p};!SjqO0kx~twJ7h-3Fsxo zehW4rO)X*{M;r+zH8ldtU#@7o%*=dscv&rfIo&?o#7g$!db>I6@Z+=!N!n|-*ER!;SHKkkXq-+4rmBpxT0w;a?!pf@Dgm#94MCc3fhY&EJICo%hs zk1gRcM#N=PnlllCp=QvlkkStY4I7ea0x=q)t74@#uE9wgpFs3ITI}G2xM1!$N|k6H zJ7JuBq9Kj@fHw(L-O&SuzBeY_Ok8?UAr4UA_Ysm(rx?*Laj^tB&^8A$$gTK0U+QDS-cMiI*Ol1+b$2Pm%%bDEyI?*(=b_3G7cX3)S@kz={R zQERHH<&(b7jm~sNAbwKUKVAq7oOblp38Ya#kFCLr;V*jn@J!3h`5w9IIA(fTCOdgKw*@h_&gyh7fn zZk&U~_k<+ZdpOBnnwPo>aw|Utvx9CuK-lj(2C}=pmaflOgTY~Vs$C+qk#A+jlJ^5- zdLXX!y$}3S7M-=}7jpnj7eAE6$vcXkx65WeU+yFv*9PkvVa@M9v3qR8DR1N`IhoaS zGUGvhQx+A$GuA$oMFo6TyVJsH#l^v8+rqyqi`pcGVF6}20u+URQx?g36ZPu$?$-XL z*aq?UVq3&4TO{7EVw=!Mv8};w-gGA7g9^~VW&EN7vb1`bzT{@uAkD{paDND+(#dp{tDr|d=7jhI^4}KjT5fY%r##d#mZ=6u)-G;!jU0mzw5Lv#j!MD> z*U9mlEPK{zAbj(JYel~#j5*cwRzvD@%#N>jKn`CfnzoqXqhR_hpN)zlWSzbcT`&VN zcZGiBfS<(fV9dAbDK9Y)^c4gMPZZ)#-K0hh9)OIi_uWq<1D?dI6Wabj@}-Rd#L78d z3#E-1|A+>#1Tj>{m3-sk|6<%2T=_h2{oyMz5o)w53tU6*+v%!jg`OX!cqiA+%e3mt zM17~$VOYiQlho}073w$laNZdO7ILLY0ZN586g%FceD(z`zEpIfX}JU_O{2oV`Th38 zSB$?S`!8S7R)61G=gF$I=448hB0#cnVtu=}XezL|)~`RTX@Z~`Hd^wa67dU-1iEwe zAlyY-AvNx}l=FEKGIZ_`4=>k?KaBUN#lX5;$D#fb+zgwo-RvpS;SJoA7DgNP+U}Fh zM=}8L{2pTn%S}4<%OU5ru56bhpFLI4L_)3(=kZ9+gtk~g zJAyOiCLx?6t1{ZFU%p}x?uW0K_kje?Y7wTEMHvcUdK=QX_04wFpR1BiU%LDVlnhWV! zLTSC@R>$5CB=9KLgEA(v6Zu6dH$EEtquN&Z%U8TT6ArteNM6tRyRZ0BZL_+s+pC;! zXfvOfW5Tp0Z8!-2d$o-`EV2)-CHb^$OvbWJ|B0T^??7AR-rfJBYNplT1AX^fwT(HF zGtu><+7_11>MT!Dv_$3Gvel>j6cvzvP-VAuOUX;S=r-qnZgZ zg1?r4(-yPmgLi^-8La|w>>vr@?0fu8SnLV!iQc$&Dh)AwFh7v^5Ef0b$e^v1lJt*9 z_2NsHl|G^Y#%30D*Og}~*{fWMJ4%-AOO1PMd2+g65jps*4=h%7hTDpaz`-xG*3!N1 z_&-u#>)!6*lV0n_#_BG1oY3BZ{3&_B~Z94z3#r&_b*Pm^&{y3(1{pG2@dMEsc z#;*UaZT(-j^s6IE#LDooR?mecwS~`!hy%tN&^B0hWO-OUC-F#2-fu2&jZ^vUL`!P4 z+28A`c>H&w-M+6+ej(bgEBcI~H@sVt0byvLMES>w_~*s>(++|Eo@Hii@X=lOqiKWv zKav>#n-s4<{~-z`sZrto8gMy>+2aoL@$3G&{xrJ(zh?LkKj<&$Wqi|-@h`)s9F~YMMFIeL;{X8A`F~i-KMecVpWOell7BYvb2UhnPm4|90|jzu z6lBeCKCOI9{-L?a5XPfqAc!aiu@b`~bf4(raPT~2K-Xjxq7`NPVU^cpN@UC{RGyT> z$S;5j)X>%zOD`uSka{Z>H{U&8n>W#5=?^pA5;r*?((UxQNV^k^Duv+>!MCtIu4n^t9zyCTTLj2I2x!3l}_0w7>tY%YK@Dt%>$zy?5!9K5y~S9 z8Ooav;=_=R%S925r{jw*n2+bn=;|6Tw4qyes_EPGMHXzCPUG{maU%B(p;{0)r+A-#fL=Z-x%(AF?*Ieaw=ed^ zx+Q(yasj~vA_9dzGWZ$wwIe#L&LzGDV*cL|TMWwE{6L+P(XEi;K-iRp-ASd)CWkNK zwM_Pk{O3@<5NaTj^5NySwg6!y;n64+b@GMqkT7Br9z8{a5a9s&>O_lhGPAL?3euZ- zumBE;pbtPmfVz@=qX8v?K;05Un(+P9|Z#8JAs-?*lC7Q2@pW8NNYD( zJ*l1{yAXRyQBvXQDCqQE&&6`7+Lo!1Oag@@$bnwd|9Vg`X6+rw@` zfCrYKL7J)c!y|wgbXEH$ImMvjLY z3gn9d#c~atqZ0MmH^v)niNAPQ4OT#eB7n4c9#HD*giA6+3jaBfrZy1AE)+zfn2CJ} zZP8#_Y&J$V7jP;x$0zF&xxlA58c}x&AO{sjA0FISRG>&aSzmzw$-XGOJ4>=;PXxaD z!8{;1fttbzWL(|=BrxoVd>@?_jZivEt3ieFbZHVy{c35VESP~pnIDg@uR{jkKNkax zJM;!JH3VmC0i@tEhNqlM?BF=^Ay>|61AZ4k5aUB!0C2&`enbF*>gK^QpBX51r89Mc zK=DaKi4TgYfDVzX*_wqG!OW+GI)scYT`0(dtDnNf@{nmH$!bFm+U#@J3q7+F5uvwK z6;RfIL^abCLX>=9_5a4wL_wIuhPI(!C-W`KpzQmhTo@i5ZK1>GlslQINyw_3_+)V@UaQVLl?8%Fb&lOcMiD%Pbo0It(VP@Kng-JYc^}&1d=-PIaqVaTT>HF+aJ?t{L4hylhnF)susJ z6u@POU%{qT+xPtO!Y3X7YmQk_r-5T@k2jEV9)CL2x?S0-`oi+BpXKWYhZP z+aS>;N>!3#Ysd_?4%sgX)a%7w?aV9^U$Hb`BZC9Cn>*va1Vk1(s?;r+?>S`T`uNe# zLnhK{Ew44m=Xd%ga?@=&hol5ty0R`vF7P;f~iCPaID z;|fYL90KNeihzi>44@yLc@JFtzH_dqPMVEutAS%KllnW;E%*b3^b0@qk#EZ*0W z^XOSAbfF3!Z8+n^d}m=?@IbI3K6ipf?O4t$(m}^ zYts{I@t%{n_&q!JvLEkkdCGa#03MZ~Mudp04fruig-P>g;T*`{HuWW^iNgR*^*?>l zoKI;+&iAor(6PN2%JGH;H8l_Qp5qm0;(#1hs^~0>oRB$Li~7#sGM8vcnxSJU_b90a zytv6~X`6l`3oK@e@|g_O;Y&1fa{F}7hz1yaefJqmDg!8~H93ps5lEpyU0aER#q*h^ zxnzXJQZ%&`0!e@%kuV4fBGB^pqiO?wE{AKRh#d2g=;Ij%y5frf0%?Jsa?$rC6(Fi= z2n=(DXIKU$D8z`Ney9VyBDnT16lT?$*yDHZ#(DZ=tWw!0c`XUeMf1Y+0doAi$On~j z9=e}OyAy1mtA%ucK(G7ws=*Ezk#t0KZ=6a^ zM|z`029dj4i*qf@7@Acru*BaOgmeY>g!f_ZS{TyU+>a6%;xd>Wh+m~Agp5Hdj96*- z0oSO(t4OR*RvJoGqL)5u)F-EC)n=*HizVqAHXS($G>-e3iLk< zS7_3U}B(h@NG81YcBA<>vXY?#KYUhi#x9C zxnMofKjYSexp-Y64qG2dUmq>!oXOwFd~JA)*Oo?5>p3(dYz@S%I(=zLg@W3#LsrhX zRv;NWUsaBz!xzrIE6)m&^uzprp_;&>P-W^dHezp?Dw2DCIO8l;Bas8oAVLcB{`Ft1(&tcnYbeLcm=Qx{(}Rz!Tf4Bu{h?`W4MdjEs^^* ztV)3M8{@9hJL3954RY?R(2jcCR(5o9J7w4?TF$**zW zjH)GP+>-S7JX5wE2Wk$Bq+2!_R#lJb8+L`H)||_T*Qhn*TYwVsts{TD;+ltLupZ7< z?@7|Me1tzZnq+h!lTB;7J&G&@x@cD_LavKT36xW;Ux^32?>$yE;Zn~KroGX*o0e`V zc=R0iR*2mgdfAj0y>7+8cQTi>Hx4@I3@suu43yTH9dCcGV6hyPX4>{mR zPN-;XwRU<7Q;JMWIWxgbvNw=Q zZNI@1w%J@%&Qyyi8)5v~(v0cYaAeeZqA(QU#;zrmX0&kVymAdaWu!Uj67H#L7@VrE z{%R{TWts4UI#TroO=J5GqWwHCiT20VRw*ql_bt}5Be$7eTPAahN3Fbo!2#Gk{7;sE zHO&rXsAV*-;FLN&u_WIMpF_h<&u+@I`|5E@y=4H{>u&RRz^0!(r;R%y8>~%yZ*T)mCnr<2{b#Jk% zO?ABbm+!eE44=J@uU+e!s0KFzY}`T*;NDBdP>ZtRVhW|lb}ls_q!~yWJ6%qXr}agX z76jqQNV!|-Y|XAG-Pqkbx-6OqO&PUcKJ1YM3mlEyUUJO_y%>La)a7I29xaLv>(=kvL)JXVo3}zz>2R@`P7&H`wLLwr}NDG)(XJEYe;* z^;o{#pql49MMtoQoX)1L-`?R8-ZWZ9ZTZ!VkMB8l91m#sU;RhroQN+aODCH1SaNUr z;&_!`u(5t5yvB$BFqeR8xc5-`s4H1?fuQWfW(|=IMg75UEfJaRvW-WBYSTa-O7$e4g8Q895Warb$ zf{kExw^5zq2VYIfub9Vf<~n-TTkWrP*al1PSvxiy9a$FAjG$kad#xM1uuf$@VHUnm zMNF7F&@N|7<(Rad4Fr5{gkS>Xw9&_CVmWY{>ZhisqO6ai+k4Kc_2Q_svIL?+C%Z(f zSGzT{AGx=FpbQh7R(&Hyi@1~y^Fm2gRy=1m2(ilNPg!hC#VzzvmG^s2Yp=k};AS(M znxjg&p@S0FlTNv7Q(dVjx;q;`i^kmrFVa2`6SL5Pi2Q1;hNxjyk3(`ZIoAHY$IF$< z3(U#&iDuN2x7w5?@k-hHAxptTd4gD~olObBV1Lt@lZHzfQy|;k-P)7RD~Wc{nFyR(-Qy zzN3@AJni}Yg*Z-XolbMM`6SYkE>$2iqi26 zx4C6k+_U-(FjIh}`rJs9ejo$E*)r~%2{iVLL|YDU`Z@NGvYPfLdhz~OvrT_?rJHhJ z;CAZ5FF_miN$jLTRgcSaJx#XBst*oa=Prcv0w;w>)bMA}vXsm9>*f4C({8?0CUp-& zkDw1No_9Z<=#D=_XK8f3J;l;2szXbXVOU|dn@^8$U#HvIcFnI6F9&5-rc<}l;7**3 zkSf0Tb*?W*CJv;RX2sm@UOY_~9B+Ev#NoYVYNie~lb_XZlcQO|xwk(C&9Rq#!vJra zK!f)5kdxD|`9g56d(pwxv1V%AMu@*yJ)HeGMSo6l-9;nq%3v$2HD@Du)@q!#*4x#A zA$LLKW`TXbt9#h+qbo7vr?~D@eHr_S`|xF{)kLPpUQb)u{rO|X6r$FTMyNNP5y%0{ zo5(5R3oYxu>sYc$^M&!c(&{s1=%hLKri%N^>)}f*w2S5>k0-63euEp^ENzc&O_>?z^M?n%F2-GR;5Su)+BU6Ur(Z2yb_%**XQR?L$Ce!qCX=Ng&Ge=) zxUVC`2bwMo54qU=k%cq`G6u6cjTXi(N5eX%qF#`@P6@hR2d?l>v{`Rni+@HMY)`Y+ zc|-|66EQyX-&x!W9Y-x@O{K_o5uTf~}utUcWU``UkpO&F#5S{=jxRAJ?C@X!{>n@ZWmV zXj^}PcN2RPD@*O)zwXaqmpqjS1=$G2$oQ1}uq36ZgeZmRo}453sKS+GxdIKH91Tq^ z9c>*QqcQ}njS|fVRofMtoSK-RfE*c@YykZMD-q!xACgj1Q&K>Pw#JY&=Nk*7&QJ9( zbT7&jXv*>9xB6>9?Jp|%KjA%E%3q+C?mq>!&7Ece^M3`Xy;4a#-Uuz=3j*HIi0E6{W##9ptRL1EOk&1{4bNT^eIi{ucO!hGAV!8QU6A-X(<^QDOvwl(EdBU zz7mBcb=6K?>TxjbsYwc~$dJD%zVH}`dD((k9a1l&&a$z4m?=>>B88}V>8Rk-*4@9} zC3iXrxzuZP)N-OMXMFXX-C4mGXExSmy?ZW`@f%5PVk6fqwfHEij7rKeOgJ?Ms>z zL#7VY8$CU9rm;Cwh|35SlzvwK=kjshV zzg0^J)KZehTE9jQ2YDo@;~8=-*X&@HzUEjCi+5t9T+Amx9-7*cqp32Rm$dv6DI_VClesy z#nS`zUpbm1NJCHbPU@)^2}BUXbBc5zt_A?V8Co46@bSze=!7U@G}_Ao*CPEO!3wBf z2r&>4`S5%XS~xJ!aA_9GI~4+dC&36rI9MMfn1_pbjk#T(p84rJ*6$=(WZ)MGKG>#1 zG6g~d>`C_43YOVOiU)=G_?8<|{=0fQC`d7#t9u!%N+F7IJ8L=vRu<>AI z!NHCl-+=_M{RIYc9Y@7wZ*aLFG)C3rw*zr>Yxc4z`+ERBh5M&2@#6A<=(n%=Un6Xp zbRvJC;D~QXfFCGW6KQriU{Lm^lA06kH(3atoZZAUQkY_m50s4m603GiEc@-{>t3r@FO*oW}VMXU_>-`hvh~#Q%7cT`pO}Llfhcsew ze<>!&^>Gt|HF5|N!Kf9sC-U+5z3(wNZvKd~LbngeLVXIKKZ*A6R$c@H0!2))%-}g@ za%jp8Vu3!){q)|aykvHisLf3;M;JEFeD4;yC1Wv4zG5^ULlHQAL-sh9;0%NS@+V~g z20;HjZ2u?}vEhPeX#8FnWH?Yj+hYMjPMvX8j}qG1jv3+nSG10I$r zke7)+vNu}hN`fWAU7ErrWS&9zJXNk&NQt^tUoC%w!9X%x6wg^_p5Xpbp}Nz*!C*e! zJ~d*R>w;$6-(WD(Z!nm0=&RpvFgOk}gWVtcy8JVVc4*fSR<<#0Kf@0%eqEP_?{eC; zs)0^C5oN_9R%v(hA1=U zi0tc|`q=snzgCf>z&4`GXUL$$Ap0Po??Cx|0$;nzrvHw(yGi0D!P(E6Y0K=EV~6_&xblH`aZ1aCnj(w4&qc*_BNi|tXz;fD z2MzW`X{*JboXheGq7}TcydYpS$+Wd9k-q>55#I*TVQSs?=LL#~aQ`?J8XgtalOhIy z*8VO=SD=dBdgf;TWp98GHcE;Oc9f&V*gLu&_~6{XA_xJnIF7DD0GS>rO|}#43ZlGN zy@XSe7B~QZ3(RK)wNKQ*O z-K5kGWUg_abP|!7ITE=4aSh?R$2OQx05-r|>Vw3q$=y8&wFR#6?jPAWO*`+L3&`&E=s3(w zIo}mCL&A8zhzJ7{8CiXTg29CkkVK4TgE&eg222^Acao|7ESB+t8%h+lHj`2CXMFGCE&f>OSELpK2Dae4&ZH%s#Yeq_?kX+Z*iy zxP}fO%Tn>}S7VA@e5j}=@szN1!z;4t@@_|Qnx?Cqn%EKo7N60m1E>BDA`fJF5=LYx z-3zjPG$uef4Wb-)GY}#f9PV*&w`dxFmhPnPOUZShP=82tJPi1FQMM{fbs~s-1oYU= z9~q?@wLApzw4l{gWELA_6_!fugEJgo@~RnbvG-vqTkMnnPK2ABh+my1gh)UvbU%nN z)EYJ52N5P)y~hl^g*Ea#r|a+TNi;J%JfVvmko!ET=b-(h#MM1kGB>49(G@ChWW1@R zVyA03cjieud!}?>k!r%QW-S)p$3y&5le9ilw5CFTYaFX}kYEYM{q>N7wfeFtwYL@$ z5O~N)g`*`KDQASO3<>M;x~dmo@f>w^UqZPqnd# z&D_n7`GwWIU(sYu2y>evPMXakgKX^2CqflqtqYD4qMO!>#|`=j`HX>-Uh>0+DoDzX zdTa*hy(j^Jad4} zSzSDHIg0Q#akv=Xx_<8w_K2)Fn_TkWrSZBbSJjZea$tItFfrgvY+iQ#35rN2rG&58 zmZsh6;wY#Z!oOiId5J%^c&qi=Y_4L%-F1p|TJf-B0>ycG|03e`Vl`ENN%f z@2%U{F((_ULsbcA(otP`t?mo@$N7}2KMHnlz+9x+-!dGYPfuU^9t?z8m^jMH!`KaL zR_HnO-v?<=U$1I2%dk2m(_U=F*PUOk0v@(;nzYBxxTUjNT)OodJk;2joR1%6M<8J{ zPo`O65!*0>p3ch`T;-}cSoz`>Cr0Xx9~cJ=Q&VE)VzdS)M!LHw(kuOe;WXV2> zbhwU%0K+eRx$El(28(bp^<;F6S^h4ESyx1FUn_8ERiHU=elJ|4!S<-B8#6@OH^(Qs zkqOZ?eZx3u@^DwK;od$KlM#a&JF{38IkJIetit+ZrhKpH<*OfuRWEL^S;W+!uIqvq z>l)QEYi%{b_S_V)swLW`=8GtE>9_T93fXr>>(|kOFG;RMXPn7-B|-;NRHK_74@taF z9uM)#$O?mJHkuq;Mw)fxO3%Ip)RqQL*k+gZuAYysyYjqB%DzR$?%b^}DOh(&waia*rU60l>jK%nHd+PPAkW}{2zWKc!x!Dpgi zk5%Raq{P}w?}e5u73jJL!AzAOlGYmm;KG=A=^ODaPvU;-_}8oJgEi z6_#>F5k8()2)=%2!g-Hfn4BCjM&2nT4WG-FlR4h6AnygIY2f(&CVM5u%StVDS z+0B*J7WWv?iz)uZBhShv3;76Fm4;tW*Up@nN;NH~Yi4+|I5{}(2@q5t%H%kx8^tQ& z`^F?X1a`4wdAAyNjFi?+>5*w$*|I+N=;370GJG8-;WJ=GO01oBWF{TTe%7{C{uIdX zusEYo>S(B0;?k&cn`my%QE%#`Z*fNLQZ}ZowAs2H<5B18GQP06xwc~c^LZfL#G}vD zHcXkAD>QOb$)?P9KA7~Sk27j>CHZq#hi-$@Qhh74HgjjOy2sP?eyN!s97824iiWum!nB%t1`Zb< z^ORpR^uaJ(AEUaYISUb0=0ZHuLng>ErT0HuqnP+AnQ8~@nD)dh;-m(H` zB77Uxz5a!;9oq^3|8>;yUrfutAI$&VnWbl;WMuu1QEaBQ!@*yGVpl%5>)H{2suQ71 ztDIz9+?J^%+vc?`fHV$1jXl?IocF;?{>yI*C zMUg!g`$)qdq=Yu2td1sfZb1DsqW7^FYfKq#XDAP+BfuvN&dJC@-rkn$%t^ro3oGf; zv`LS`qenzK=p#5Q`I#8q%@GQc6j-fur5Rl9PscUt+gHFM9)5#}h_koL!J`NN8*~(Y zlmYn%bad&{Km;O!kP+x7hB-eFAQgZ>ouLD{v93WU^b=K(n5Mm+r0xHPXNu_?pQ{qQ ztuYWfHxmqy7$U;Y9!juMj_(PA6u6BYfSadpO@j0!yCkYZ8WKwq>0_0Dl~;|NjYp8| zc`w~eO1=bw@#Id=zw!#bWt~ju_08yLz&%81hj&VjhE{;K7Lt5p3(+6i*ghELD7sHB z1h4=KQ8YSO1u%z+jd6r1kZ{X@g(4Q$KXy1ne;t&fy9dO%Et%lEpx92BwVIufC-s#W zE;nI*?RS!G|6l~9ZRV-{on{E18pFE+IuO2+V-nt}Ab%P_8$=Ml$Lp49yF7MAyEu4E zn{a>GaBrFHAO#qC9bX+PD*PNGYeaQ4huah`Dq$WR?*v_AaPA3R%f3%~iNVTAAIxNu;} zIs|k5aG?lz=)3zw@xlPoH~A1S>E^g`SGp_96S>yvDv~}~B1pI}tMfr&desa_xgAb82X>hxQwxvb#3_|PSq*=OS3MjB8(c!&{HtOt9} zaftE2dqzCMK0}CTBmK!}ka7Log4sxAjtIkfn|$4}1Y*%7RUVJB`4{J)v+#T(aX*T^ zfo020=mjBUGoyQiznb8v;34LOud>si&>!N94WXaqjSvgbYLj7gb-iPt+dqo}+7(n! z$Eyj*PP!4Zua?!VZ8_; zc6+TQ9&WDX3M(xD?}b1;fGdd&B_@Pcb8Hx-KpJN+h_EU{4=8rjm7yu`2-6IXfJ`?+VXpb-@`&ACF*otMr@DJ)E}Jxs(l7l~iWiHA0OZ$>gp95yW5a{f)?ZU-n;I=j z&>UaNfsRi{_|@Vt(WTH<6v&+*G30X+NhvI8cmNK?Fkm47h^b!db9NX~D~Jz7CIVug zp$rHKC=U^&J7%PaDA6kDcVIJdn-FA0oVq#=G_57i%oi)0eAO|J8FMG+ZQ(lhr5$WT z2Ul5P4WXt4wdD*rSxG(DU4md*j?j2M)n+|q$NrS(n@gI)kBA*9xw!UtH}MQxVi66f zAb`4QN@AFuJON|nH4hprDlj85K#i+6@{xE$kE7blX_IKQBnA$)V+N9&TOoe`d|(k>bg6byn^h-fK zw-?T!{qGf2GDz5)rw;`cbp-7lT@nx;vJc7hU_+{^CP5|x#B7tO6xBTbHw884mx9WU zmNT1|BiU4HLA;CYSdNbI{pY*o&}iL1m}3+%UbdPe&XwE>Q$@GE_mR#d0iT9H6gsjl zv12yH&~}@z#esLMMO~&6ZIO!!ZZhEV3XcX&*l{Smqyt>mN~6XA>&|YUjW2puT{g~m zrb~CbB~j51sx!c+Xhs5FM66x$sm~b365>zedvS(ah;vzzn6@jB`XwkwV}Jv($d@Fh z1I9PSM72kas-k9g!^>S=i~`R6UGwP3it79_F;ydQ<5U)i7e|JaV%$d0F*-J_Bmjf4 zcn3I^0PbVX-@blgsljn86&}f(g>(7ML46Yr1GLru1f{v3(hTu;2bC$u8}y0vmxJmf z{mVg>tL%)6{mns@VQ|??yrB5ypti4efLXTKr4zEu=Yhq~k?ARboG{>!(zvAY$F;y1 z8v87NW*~)<+fuS_{SGN{Xy}x9y7IQX{_%8NtA znav2(emJPxzZ_Ig-A{M}21vg-Dr!IAhl6T$f+|BG(T%X}Tr4r;a=t{zixbct z-5z08-~=n3#2`_yTp8z3Wk+A^{HX(3X0gx$}Q-#WtJbel(+P@_BCLqbRJ1%q}Wm4 z_cw#id%n#j6F*Js4C>AmS_-T6^{`o*iW`${DsHGi%&lT8)j_2+>zTKg>~7?r+`647 zWQ=6y3An`>nX8e(#R8tM+oQ#&QKOgz`;B_%iVmc9ml6+T>en@3qe8(wEbkPT=O*Cre%Gw_xK4mw zW;b8#b9;=0sbkqN7B7sOZfidA`?eGohnRKCa0Au0{HFk$w#v$j4PUdvqmUY-tT^|m z`&u^Mhozs>vGne3iP7I(51e221RrMIb1tPv2~6ZKlnYG-Rqm*g*ANL3+vLC_+SJY`M(px4W#S5n*5EPO0IF zRnJdkCmp>#YO09wVZBt2Cl9D}20HC){`7e1moGb=8w}q+NYk#phk2e_WS6-0+ajfF zh&#WScnBtVTiJKcbNK!M*kysRxL5D`e#y#tEtr6}B~`vxlXBL&ytatDd6N7l^cs2- zr%^gFr|kN=(DpqA!b)@Qxa&3LoSRdS-+_ggNBH^}rlCJnN~JMNklgJN@H**kPPfNx zln+tdS^Z)Jw?x-Rd|7LhhVpLK-_hE6OC^~L>i9WC96n`X(H?I@iu^rZbKblfZz^7O z-Q2IC(K^$3bH+1=QLO}Jdr=-E_lBhW=IElh z*z=f0AHmF!_R}&G>+yIod$Is!0_HFv%Wd`~EYW7#JJpeQDQA5t6F26UjS{^L4^xexu0(srL*FO^;7ZVN2t};$tN>m?K8j7*=f|N&n zLkU*r^F$B3iwlPfgJW-NO=nd;UQVKh5T@U7qJDMAgtbW!7`~z7=bFBDJR-CzUH7pS z{-K1&N2PqoA5cTTr99kGLqcD^eB-~v!8#%}r-@_{A%1JS*s-KWyzE+_o!$?czuJS3 zzEiy`$PsJ+v>nk02YayJK3^iV^fds++l_!J{9-rtMYS3V!(=*lIGu04(Q#NN zNbyox!4H9_f6=29Q_6Pn&Q56)3G}ZrcV%F+c$LPc`r|zATEqL^P{i*N(Nl&dGDW!6 zn*+wl6Hv}LzEvhfY2|Y#QYIY*s?RlW{wGREoTfG8Oee1XE4AL)zaGf+XM<{&w{1RX z!F8_}y~89|)}Cix7>us-FnbQ~2!cT<1(L^dxo~MBU`Hn~oLj6LGJn{rd1j~8UBv1oc+4lcR5E3|aq$rN#8@7vy5;Nh-cG5nsrvf+Yc zUPl<_R-9P);K_Yo*nq~iD^IasItdXykXaD% zSbWWxcYm=k>UOkrqiBo&;yK5|uGM{6pVH%@@%sLrtW4&6++I=}m7Y^dRd>iV^XT<7 zOcNn?@e4<$8TdtCp#R&681Z;ElPy0CLQQPE0tbuKsjv*V(`ika`Fqi%N2loHs|&Kq zFRc6NF2YxmSXb(CU#)EK7bVjAN@Sq)AZwy}omH%nYESPe5ZM(llMPQQX*=0=MKy4v zk?_KJs%-T}LXt{&d7T0bH3lYd%GHN<4hoV)X7`&oFYjvmX`B4KUG{fZo!eRnTy!_W z*d2s+*`Nr<=I?!a<_EO}EW}Qu{Lg3M?Bg9d_hclFTc~PjeOWI@7*N!%zV#2k>1&-I zTx9lHnDNdt?xTUb6l|4fboQ(oGJpAe62V$jR85DJrKszd@|-)y4Bjir05?xbx#A|= zLTiKJ($_T&*FxrU<@5PjA}+pHS*7H(&!wtE4I`pxs!(Wt36l^`=MnqR+110tHoFI7 zYH=RY^_G^WLigT}QT>xM!9ixxvq)p^cE=XW2zX8B!w_$16-y`^PhHShAjlG{lRU$+i~paC&~Lpnrm6axA~%THy5u3#G$m2ETn9;c%-jzFdD|K_n?RkbN)sQLm>cZS<>D*L&i#_4l zMK5>j#F8Lh*^H%|2Ipfk7sg{e^*-Lobao;AjLQ;7A^s2zKh+CwRUJj_01j@`br*uzzk!dS!__@l!B(BVF83%%`*y9aMhN%31$pplUzKaMQeDy3b=&snMrG*`0XCS zN|uXl#K{D2qsU)gJo6*u;Q3M2J;VJyp8vOo;{R>r|6d`m|3N7JACrUs-f@EeHgNy% z=;i-=oS=A=R16&L%^R~ zqrHmFUPk(b_|Ao|2@tvwMiM9lfbA+yGk8pS5AID4LZRi~Op!gphYkVO6&7${TTH-@Mj zj7*tr0KK)Y$0+_TIzJ&(e=BvQ_a97gUc=tgTFAus^P8d1q%J%{s2#>ZgkH4mXJws{ z6O7;aKv@5;OtI)UQ;d9NiUrgvQ20Hrh{^UZJSvDb7!kcyEeZ4})I58t;^b!N9}dem z5C}fidkP1r1R+AwDq?>rMAn4}3XDZI$&^nu(^CQ08q|$~Csz1u@%~U+2MkNnlRZQX zTr;2K50UW}yzIw?yg(sr7%E~QG%<=ha9QV7WK=5qo5=W62cmL!$YJ8W0fruFk{)cG_~7M96ZAj@&5WAzq52pC624>MYbT zl^Pq|KU%%3Mt3~2 ziUL`FQ4C^F@(_jPChxp>9_S$Og)$+6E|LNWg$^b(7=R?c(>M{Z9LPOMZ1RD~+b}q; zEK%>g$xu-S>suh@8EslKZ9_X+lU()CFLO4|cOS>XKkdR(>6D_f9ms_sA3#Rp$h9Xd zjFKYiQG8w(KgW0zhLk~y&V))dhDpg6>gmdW44rIO(d-p!$L$;3P6vD79hLqYD$cz^ zMRkl(1boHxSE$(f8!8sMN|WF}!@ojB%U7sqhw>f-6_2KvE{q#3iAlQ}?5K|OZPuCD z;Dm0ZYGK^~HPj?%`h0XB(0;UfMQX=2*9OCUSMpFp35Go?X`n>H1j^XAz3mk2Dc@rz zHGpo*C|?gl!|st=h5d`k*cU+n`I{*i6^T6i0+$K`jNsw3hsmydpw!> zHZegiW~mq(7FRnL*PIcki)xRE2Z;g5kjrBi^TEx=Lx{K)4lv&IDVji;;J|0Mm5Ujl zSVlBONtS9Zm1CLLle801{456{pcAR=4K4^zjR^fUo>QO8;~2*R11Z zhA{;?G0%u@CdN%vgBoJLiUw$`N*}5>UP%e1e0!O`fKNo;6rZ~ppkTrN=-RjcW-^j7 zse??$@twKBh7RRDZNmpwnK$*J7BsAt-0w3Fy03c0ko6tWsr;B4 ztd(p3go=D4soCUC1vhbO+tQKs=xK1jyY|2c)Jy)J6{yX1z3;iG?U3nh(UJzU@qkC1 z)OSKwVqICW(mvEOmbRs@CZoCEF+4H&7YYNlAWAk=GvOZG_7CNi7CAR8J@Dw^o_G)@ z>hKhb*C_2I;Krc<%bC?FBQ`?USTyw`8o4Rn99Q%#`Ey7eiGM0c$ z#uT_U95ZI_A7)9{EWw=D5^x>x{jTun@P>?Z}U ztXN9%2UZ;aChASX(;GjvqCu49e_@3lvwVhpFxH63dF;DB4STiiX)PA)z)_>;E9UrK z_48D5b_FN;5&s%43D4G=34Cy20dao6vJsSFJR7uwZ53utIx|qToI?}+jlLKP8Z!Hv zRs){OAQ=4mNV=J_LJAO8*308ubke5(6rAyzcN71nU%!3Z7dyVw0MovcaApB142l)U7T zOjw-PzZCUUgO2@mEPi##Qs+khf+*xj;zzkXxRRC5nEt}$*HEOOF+Uu;!35)`@`1q% z{9cz}-UkB9`jsn|9sYwW-nk}bUoTeZdVhrVB6gx%E1)J#j__s7eKRYRbFH2;F=8^L za*@dhE7A2VyE+~z5mL|wcw?k#h6I}r6O7(B1goL{%!6MAP3 z@ZbuiZmG^)NA*F4mX#1wf)|*j=NZoiG8wbq&Y3V*6Nc#1yQEZ3&}gVO3q&(s+{8-B zY>f1MHL+92#c|4sZ}iA?(K^V5%L&O?Jv@WGW_N~UOengf9*e#N|0Eh8e_8~nl(seG z4WG7yb7BVR7$W_(F?qA4&=EmgX{#+S=e~977Hxy7E~jNQ9&xP&I9IkQCz%d!r;lA4 ztKu;BundC)twS`4!l&*$!JG-Y0%+a#;@0{yW6YCwO^QtqmME)gf-I6}KRgs{eB`0! zbI%)j9u09BWeSHIYa55LNH^4Pc2RyI?)?gNioy>?TlwoK$xyD6s8^5iHvVca9ey&m zc8R}n@G{`)G1U^9i6>$q3G`OL=zV9t7NiG`V&x{n*WRO1ecBcMRTQ)tR9-jzA}94( zfUUZ<$QZ+5eI<}9eX+20jN-?So3@$_;Nkiu(x|dJy8V3PV>C%q4|(H5j{d-!ww?5{ z;kB%r%nov@tzB;%0FgP*#o0O8t4zMbNbW;XRrj^hE~HQ)MiP!mp}Iy1N8L_5YQU7d zo#AeA@JaQ0g!Y5xEL<~+9dS~X<7Dbj-<+aR-scRGhv{i`ie>V#v=ITNcvTraR@HK( zk=hAw{X;B;z*9VeMOHtrW03*}Z2Tfr8g*OFHX@m5d68TiIGn zp>OYO#_Vv}rk3Brc7Dx9ri{WTbw;=EtWq+o^IMc<#j=OAx}#P(WzDH}DlvVR-YW0SUHpe;d$o%0JKL))GDRP8U?;eT-H{UG z=ykEjWSLig`5+BR07vp6F{1;Lk?=8eZ#=NnyzN$6=tt8%)ye2^VK+1LR z3^L0tAfpIECdqb**U@OeA*hVS2GkNWu|`|_5acrKHd2kSbK^wihoXcX>pb0WGOWoJ0x8P^*;Y9 zrJ!NSz9CF3OcN>q$G+p@(MAg!c(Q+bbXvc5l^gZA{|Hy}6*sGqeAUgh)a<0dZwkQb;fAATTZP5*SvjoQ`N$z=S`wTRzSE@Yqg9) zLvxmosz{dnMtFmgx?o&79n#6fhfKk5&Bm4HUj}k@P7)WmC8i0k_`99X??|RQ7{^Ut zylN;Gk7vfoLqzhWgpopF_T3soh7iL|58%0th6hI~ThcrB>`-K04p<{8XQmt!hC0A~ zM7BnNt<$G273!a^7EgqX2wQsSPhI!t3Phn;@r~sYhL}PQWxu7_eqA^PI5QJCR-#hu zBDQ>}P0X6w+$_dUTQm{7SR9fcE^GuG<0U-{e!PHBz07S9vM~$i#-mS1Zg5ydVPd)8 zsT?m%ePrYF$$)xLeCOkClHq?)B%r^(wPnV2@jeCxTWR z73y}Q99aY;A^HoaOst&_n&N!NRjzv7UeX`faF$)rWesZS_?5rw%vff(HBY~+4-=gI z7`B@6Sk$E3zuEMSIShjs%FV%{NS~H&eJh3VJ#3(XOOLgQz6@83BE@{PD9+*%tsx}= z-XS@*y={NX-tg{8<&d>FXKAS4tDs9ts+u`Sa-{X9w^+(Y`z6+PhriW{)^rHwd@5M@ z(yi=$iIW^>>#RGNhjN>)mXop#d?h)pi<>!Nt_!oSo%ZHNWiplVvqj8qD)V-4CB4R= z_$B?(BV(pRhK-V*R>tF_xU938R)Ok48qwv%qy_()6Xk*GP?OuI(}KEp4m*86eh5Y| zjS?^jrPfKyu9f%p@EML7j|IAx6aqh4hU^Fv(5SB+e_IG9sv9asAbUJxPELg&i#w)W z+&+!qi?er%-AUR=#R||~d^T+h5_9$OmEgr{_9LYC)O7f!(Nf80U8O2*%?7GgInR6< zX2hm6pDyEE!?vyt=ycz5$EVdY9P8rrBnDOVeZxt%c(y0iyPPp&fv44>W~t?yOa~Vk(XPY?zh|x1dY`+#P{lsN zOtDc>hYxUf*vpKs{&sUo_92@aKRWz5i9wa-7m`C8c%D9z*=$>VO+TW9vqdHEH_ut! zt^~EP;|$%JBV8vkC{4&7#(=k$`}F&D93|uAABOqz4>n?cYA)cVW4;ewdywS&*)>>F zP@E4sr!#?BtX_1h)D){z(E`AMbR3YkZ#l7Mt7!gW!Yot0|0&ZlXR((|5MdDYLp^U_ z${4ly@m8T7=V0OPgm0+&a%DdEeGH(aTzswNRv@6RA~r=RX?cL4OOMXMxKRzEp-9O` zpo^W}SuLq${i|yk$`>%VlwhQy5hkyKKFE+O zOaqcIIOp*#_}v~OjVq0q8=Mz3l_FvIbMksEhBSDU^bI=~isb8j?;(rVG`~-NTPZ)p zd5871xoKzgx%=7ictkn`ider+>zn099?eU`hL3%-R`h1r+pLq5mb>)RG)DxD?>*vo zyS44-A1Zn{MK(+@JsfWP@!~g8%BU^SjIh*U%*y!7-T96tv!=RC*7zD;49#atz2g-d zYI^JK_NMCswv9%{&6OU+78(yJTwj1W^>}Z4^v-J!A&pUXz=j%MDw^6}3IG>XfEnYu z;;Ou@VYud-YQtI3v9SoBu?K`mV`uncx|eo#_crz&{+V+bKFKAU2!rIR$t9(je{Jqx()|bhyzS=ywY10cS<3$ZC_(-{-0`Ob`MWdz z?=a$@>Ght(dE#9wONGzjw6{Jzf;xBJvh2!Z@Q2$Dl-h_oC2hXVos%5-f^m6dy;~P` zzt+yd?LOs`X|%N-v*YFyH5JnbU~W&1NE~3@N|he)tG%OnHj8!!I2k>@Dnv6BvfHxq zJH58RVoHGh7YTBL0AJwW_e}l;SpH261qJ#XbZr0Q9R1F2^}hSxAci*bS+pHUuFanC z5=5K3B)XY%&e}fmZ@`Ez{C*~U_u7YK|ME->5}+r}L1L)+zk}ia4_w**n&JKyL(|da zib+s5SYm=ps4%EL-wG?QsKllijXVVVKI7zUufO}i6Kl9ww>Wa{d->q+j8qZEgXmo(7nTn2n;C&v0_{8r zp1>b#8>k5q(K?gRY@ie!x%OkP;KKJG==vcH1SAl?JfU!+_(VzczFo4=NWsD|VATH5 zVBl07f?(h4k-^aJC`q=~C7XV6_(AJ}i~BPG$Wb#sV|xXQa;dHAeAa7&=7ymq`vTt6{R@xyj)e9;9~mYq z4Txqj2+Q_Hgm3MKSOY(5xc9<^h`pr8gG4hbk_jk9N5eovP=R?9K@=*JA(;I!06$I~ zUR3c+CFSF1Sm>JZf=_HHv|#jEIvMaCv7apn4@-5x!V@BDg2nJtOGsOhvsm)BZ=uT2 zFJc6!;legX1Cfc5wKkW(!=sUu*UP#UzSWkk&Qh*SVg-T~3Eo)GGZ+wLb=o*+W!OLHnQ3Cm|x>esN=7 z)Lr<=%4l-zN9X#M`o$kDI`p|x%Q(d6bU#pw9zG}$bx90(ZP8=a-%;p9y|(BmZ^`w* zO?04y;E)-Qx8MB`c`IX+^QwgcV$O617s4`q*(;L3`70p2nbl_~@G-h3RRXg;bqdhK zfs&MLy8eMMzY(qx0E(m=tCMsY7iJdfKcpUMU4SS>&t>20ndRvki;Zk z{bn2Xn`?FCE5aofZ)!o1pRj5DXaOFb2U4Rg7 zbXUUaD8xGu!rc&m1|eLi3>{9v{33=Dgb}td4ET`Fc%rAx7->yngCCGV~pd3zA}d# zy|di`!KbM_-yw(M&$J4He@xrK>9-~;3M|~7_9_3 z!-hyvCB>Bps!{f%!zyj2c;p*M!T3-IK37Yi&-kNbiV&IgqmYAvywJjjpTVew5MJL) z-=K8leqs5C7wRVp!}W(3%J|z0ZQ;QRH|8A!)^SW$@ADZ~bK_t393j@;?W(4#O~b^< zn&%kW=LsnIB!FRq>A}a7lF4G`%*UErk-t1~hl(Z1*iiI~JrB0z*f(rKko$e`T*!}$ z!QRNY8(Dk2w)?(=UR`R*31b!Hq!1%fBa-efbEXr6h z(n`X)A~Go0qi`@K?l3I`e}is^tH*-F5h2?4CQFu+Y5d2k)wBj#hAu(G!EJ}Y{f2fX z)vpg3ssdrl>iev0Xoaad@25Y`_=ZiYhD1KlC{S?6t2~layRcO6Db*Z(ed-Fw2zwj+ zvUaR;&szBnjSXRCnSTE4T=4Ge5F&(@+>$@W9wIWv@cw5?eB!t7#OA}_iw;>ip6DcV zW_}(j&Ql#W)=+z6_%W*_<2yshN$?a%42@W>T@SV)Wr*B_pWrhRB-Vp^00TBgrG%>g z0{xj%ii8lVucim`a7K#B-%D=C@Kk)e81qNu-1c4Hf}P&>)Dw$#z;DoNR@Hc`>N-@M z-N3nZ#BG5y@0Y4IRL_#2m@g+rA3HRM^GM}P3Yr6dzM-n z2Mb6HtpbUm7t0Y!uVSbLedl2e^O z2eG2@31F&}fcmeQnWE>WW2W)B1>*LOi79;upA4p`jxQ_svY-|{XlqPgAOIkrXUrd0 z+`8>%lBu+TL!Xxdf9djY1XlxpC5&Yfv?l24A*lf!)k zjbRo|j^mv_gu`vhV;^{g56pLS*-+*o`xQ35UtqC#)H!ZJyKkP3(Rywzb5T`>d$+>F zadu=l$I`^Q;}xNj)l6bO)yI3b7S#yYii|nWMB{TEat=`CyU~<)=eQ?z9M!L4u{^F4 zDGumE3R3CQQzjD{n`*vds6!119a;Uo~7urN6E^2CxDDX}Xi4>^pHYd7p^>=l9Dnl!c> zQ=~702h702NhedHx_#A}z2n?;nsqZgz1(m68oGCgi9ei>>WLw#Q?2s-O;7EDcNXaW zP6#DoDZUI=cpXhbzD2)I`9-png2T5Im2=G?$H*|H>8Dh3lQR0;yc3PdU6r_y(UCqu zH%0R?!`lHK>zBn_bz9kfc6zdF1a1eF6K9~n5h_8Pzh-c<`++&Nl$NkryisMB>ceXXf z2hBpQnzpIJEy<=8!B&TOVBE9oQ7G!3(9vSa+NwsH)&X?O>deac(eo0YRt}z9@ZgL< z^@Ex#{}APBXfV|KNw7?S~0NVYKN@Uxu$d~XWFvp7Z7X%1i2<}+wr)QtrWU3=)Q>0ZgNjB@tDMn?(iL|}Kz;OH zW?i66ieMa!GoYyQX1QG&R`tABg=xav$so{ZD*&FdR}@X0C`@x>b!equIsH-%QxG0XOb9$_EtN0TO%Z%a9} ztmPvWt5yf!Zu{1Wxtr9~8(7Dx7!jIK2t01KM-jzN5_Iq%mV*@@o}E{3mJ^%u$3ziM zm(tdka)s_V4ohB)fh~nhA>-zWIYq3@^Sn>hOtu#hczcc-^Ey@n%sl%mNL@9-Qj;?g zH}#Klj_ON%r@ldAZt1(ZK>K!YiNnr?JRb)wIj3Te@hzbd&XAnZoL1xcN;FL-^BL{A z&@Wa~OKU|ZvbvU|wWtv*Y~d#xR%!avt#Gw!84uNm?1CkO)$h_612F?eu&r41gc6TC z-LxRZh90yyV(t;St2Pxa;zdHLX0s2h1+nosahIRl8oBsbB?n#2hbb8W+Q3d|u5nhU zlamd>%alG1eoa=%)7XcBv_3cb(rS*jdTXB@R-13m4~_B<6yOoQ^Y7d4rmZxVw|?5# z`BQzKQ>IaxD@Uk`Giv-%cFf4Z-F+>s+Osi3bN+!!Uw_zRlF2TDX%*$!Y>xDJJa(;O ziiccCFqUj!j>0JodIV`gTKjv~$$F%g!?HF%Uc=xBS9ix4o#8;iz7~*irZJeUil53wC zTbn~I!`txd60u2LItTdUi6KR)w#O%vwyjO|Qmc0^w5yffCcDGQ{N|kWn}oQzOw)ey zI2SEuU#u%%Si35?s7*GJfV%WoNJz7dFO<#k271W)5WDwy>Mxn?JiT{}9X;~eSIgLx z`n#6UMbr%HRFY2Za1m*4-)z(5CsW~OYOn;^S2+~oC0!M@%GW8`cGHbls1z0U7_ZltPeBU?+4{N>2bTylE zyvXP1v7Q{K9#nw)$?-98@x>T?9I->_GSKCx=|JbK*j7QK!411{Vsg(GnHs>&eWiaCU0?_=pF?(LuZe3)v!l@-^o&!>o(+jSKg1 z=^$PO7gu#=OJGdhY4x4{5IscLZCzL-YG{Q~gntQZe$>NQC5+>Z{YQJ<5oGnpJk~|U zve0MvVd_(N+mw#PP0fJkZ|Jp#K8pM4^d4&`UoG9A8=JzO7orH=&p4aPEIWc)S&_(v zaaw6PW^UWZ_mQ0Dkf1P)79M5{-&M&G;J0wKyG=`8!`WD03(C#Ln@_ZnyD7_dHc~X? zFrP+Br#KfMT+%T>E{{*J)>2~_F+ttNZikyi*v+GEH{VWiVda0N;Qnw(?J}b#HJ{o096Z!!z|qoxNwn9u4>#Vd01aYV*h{xEP|7v;HyS43ukn*>*947*dbt4 z%0*$HhqIx9yQQ1QYh|~;p}gFv*LX~B1f%8Q1WeiFkGrHAIW9XW;nW(r^-^#uU?1=k z-q=5}X?`o{^L$T*k`ftQamYIJFv{G|A7*g0Ej0bZ{ODrpzQ`Y}DG%y;_aqFPbN`^r zZ6+wLAjAQ~*<1oCc7gx0Fh2IE*7{7P?TdDn>o?tW$N3Mcaf zX?ENdR=#P9Z4G%Sy#FPV8zU)*bZuawcH0O)zp(PT$D1+t@H2)(JcjV@9)4<;jcxeo1SglsUtav7tDg7&E3%bYTO2Y4p}4#M zFO4=MCp$af&#}zkbn?GJaXbq9k0Q|KSemuPihbjnrNptp`Q+l*wrG9B9Uz_3cofIl z)UVpUW{IA}Z!%YI$v`45xTYv1T4F$az~7{4FY|En_!Y-tI~>oxM%sOk_mW|MjkG6J zf9ZA2MrHL4KmH+8e-!#&VS=F1RI(jmn6z;JgGviBJ5l+TAx)OiibfwcZs zfM67oR0+8Yi{#fEUnt1TYZ@@F@LzjAA(?NA$fj?IO_q`so(+m=4|EbHYAA0RsOLJ z>2(I;alNnGkZSXvyjnqLf*K&Vz1pWhar|2fyg)ym1bh1;N-gtZ@*WxFAmn1{>vI7{ z6?6z$bdV+4@re++hiw=+6r#DEK)~r8Dv`jdD=5>Z`K6Z0iML5?ny*Q8_?l@$bhz?` z0bq7W9nsK!yd^dIsO)6~7{!8PFhV5{@(BKjqm)V%DbeQVnI-yzNY-NaEK3y20aKD6 znwajNcY^jptle;2Z9zQFPk;}kRKkIjO1@X6k}t^>9Ni{o!|D`hH1cyw3L;_1H!Nup~Ao(8e zUCCH!L@Y~<>FNqr;8U%>e$*j5Q)N5tbwkyig*0ffo!r;Au9YyBO!A11IvLu8HSrG1 zr^h)iN@U6nmKbMrMZ#Dr6|Ut!wjoPE+mQRJ!sL{0;Kcm2PRE#Nzlu*(UyVvS!XI## zFBDB{@76QH6V~9wa5`lu%|ss|ge%cSz^#PeoM}LR6@&@iw?z~DlJ^Gpqt4gG7C{c8 zF*RRnAxC;prp+z%5$#2rlC}VI@j#+>aQy8w$K*dVZ5NbZ$5!RGr?qfWgJwxDJ%$yT zayx*WEv6a_G79+fPpg6kP1SoJ$#bWtPZi){!f52f>=U0EV44WvKsb(AKF>kaXE7Vk zJmS_q!1&RpXa!}055LY%E?|6O0r3zu`F%4$o<&9Xw@^v)r%?Ik5212aP8v-w0~t&9 zpF$$+|4^#3-8U&k*c^__B1pk>tYvQGi^FO>+h)D*Z?8% z4BC&4G))_lz|EG$pv*f)SZ zptxQlXd2Rto(my*BidhtJ@uVx8vb-&eD%AXWB-z5dXP}52NEg`->_wUMfioHqE;v8 zlBD+zR-_#PyoSy{rt-nD%N5)>0uXL?S`a@1rM%w_08K;MLZKk_>`mvzoaEEJyFquu z-GB$tI7$$Wn|P&h3szSeIIlGBKD;h6@ijv?tRu;w8CVa<3)iP|FB`z zkS`Asig{e|j<+PXE-bctN_KzHIG?vkuQZOcrg9IJjc#Q*V*boZp!pw{g6L=%-V<>p zFb73q2|&UJm*a7NiPOKt^fWgNi+*2~X3v;sXROWemMIz5=^xXORZi2j*U@Po001+* z!1D&aYSbz{(5WExfq$F|0*8_H9lDzwNJEhNMw6cQHJX<|6u2-+oR`WFrRLR&9|;gU zN6b)yPez>-z?@~IF}6g6d0bWZiGX@m5Bnr*#{z+IuXfaaG|9FeZZVsAWs^$?gP9is zu0xB$1lf(?X~`IWqyvuU)*pNzYrS7BIQa*To6z&@p7&Ky79$^pGA4SDkdV?MNI4#` zoc#xmQ%{5VN-Dc1+-pc_cLF7z%!4Z75HHQZPL(2E=CyQ!uTCs|oexG<6Z~e|hYLXz zR&sTH{42l*M@`56oz$c^dOb!8z!rR}Wx<@R(cazOp*nY(?O_}>h3E@xh~#e^$CN#x zn~lU-Jt$%8TS*{DE-}^dvl1)Xi~zDq8mONS^&7{@zT&uHp=^RnOhzoul3%fUs=>## zIu@IPf;jRpR^YMtBfQ-$m~PpiX~^u?X-G0GHa-#!+U!54At7BN#{=eHry=S7n1*!4 zzr})UDto^KnuerjAgw#(Rt)<3EQ+lEisPmm6&F{E?Ze27K+}+vdmtQV-e3#DalelR zIsC?P!lIDo`XC$!u}lvEnuZ))tc4P63}9@tpQyLc@2RM)txJY@u7~EkJ|%nWnl322 zjCWZAEBKgpxbFmUbG7U|dbx9~o`Rxn=6nT#a^n86?5LAzhAvE6mLh5!=0qAprP@_DaoLE z_({(F8UHf)i78ef#!r3(Q#rxi+`{aencla;Uo0o#*0!Cg+#Q%wx87NJu`(&VE_xc? zE!tVa@>AW}p}m(u6! zL4`Bk9lYVITA~?>S7hV4*mX*$DpPBoCzEP2zM5Cvtzxh<51XiCrj@I?tlTMSvOi8e zPD3%(G?`$o%|e@J1OPT$hX$j$=d!O2?fb5UCp4Byj20WU_&FCCBg`#_w0Mvt6((Y1 zrUuN}WqS(kHo^=D>T{1HI|NJM0uV=eAj7m4v#YyTczBl$Z;Oi<=R{_xT*A^(t#t1R zphcq|XnCF{V7Ke{7HsH*JT>mg4D zZg0-M5TBOe%QpfWfKHYxu0ri-wrQFwkaTej}TEJE6 zW!w;P8g1YlJC);)CScs+H|~4wqlkQp?;?a!E_e*^Rf;4QC+;Ow?q{Q$Y5qE6;hvAW zWe2yrSUip8*W%$Benl5C37zR?yM1~4#Uir6#b>63{mm@F*K?EIS!-=&HkesPkqq z>C_&yT5%a+;0@WVz$3~~OS{wA$|onwV`6YRPXc1CW-r6SY#%AYH=nLits`oN;BQx% ziEV|OD>}VoT{P;_8{=}elH{?h=XTrCFzR~GoulF%)K2kmQgZ8uw3ogK=RQi-D0jU> zva`9WV{cPpBQ6^p)Li8gU()lhI8a(IMD0_(KqOw{fK3-ZXdOmw+v?^{yPZ-|XDWZ{ zdDxjhO1+IC&Z+i`PrFxtvE@>#S)-!ldgRUSo}6cBo&}AMnw$@;xgz2fxW(jgOpVSu zOj|R#6=~{w({obPx23HN$MoJ5OYA70SQxz@xQ$UR=V@SAV_a+_x*d*bh<9sUaXr0@ z97*yCK&X5GIy}~&McP5`M&}8qYHB7}^I9GH1t@Y4pvsJJ_%w3$7=?|TJ{d(;Enmo( zGr2e~(CEGS+UbwyUZnOVA+z3jh`T3MN;awYleORT;eq{65?)cwbTf@!KJKZ-^7M|t z;Q=gQHQmW)bfi0H(j>qUy)9<~LqE~wL*%65xq6yfVYnSV^!7m-em+;ustgcAMz^cO zSAsu=-}JOpn-g-vwRx3AV4sIAZYpvpvA(E(?~Op^sT+ZviQ?u3_RP^+iVn}5B$ z#NTwor+q#v$|kumSgFO2bGSdCW{c{u3r(`#k+$CU6tX$4B%~VENtM0V4NA7lZk*vP zN~PFPz0*1$w~;f|PriSl5?Adt+8uMiH;L+RLTM=ito~?8#b~v0oee|q_Im=G zouTol##PQ`Gb%P&r9JU8Ym4H3e3YC`NGWtP^ALMjkBa5ItK&+l%Olg27${nB7JR(Y zCA8{a5xhZ=n5c>vHnVg=SZp~Nay|}f!o6y3@g!w`)}7tgnG?fhNVG2EKs zN%z6Wl#07fW}w0--^)>uxK))Y?V$W>{)cI*rc!_RDg2O=qDk%y*W5x^^$vdyci=WY zCR;MI>DSpy1AjFn#nv?J%!co+7c|#~IVzSOS_>FUTX;17W~QSOPId$1Ot$mOc96;* zlXmVb?JikCH6bTH^;^KaTWFfFvSIsK+;m4F$aI@>W#O zp;I&J+cR&W{M;Svrk^~eO!3RwWLe_*om-uKYox19_-6(X)?tv6w2{s+bsP}0$;pyc zq><(tYk=P~#6L<4*X}&GmyeZlzjJ-rPH0!w=7$S*=~8>9v=%xrw2JT zPMkWm0MI#`I6KLD$aX(+cYQ|6gioVvO{*27+4=ElbN_7_8eJi^fMzJLEyQE#scG<7 z9bDcZPfrRG4XOC0|H^jKsO$Wo#%l~yvWQLoVw(H%n?;z$-rY25jJ9%*0qpKd5m#ZB z-452!T6!|YmCXiUdD$>N7sqkcaN$$%I}&{g?wLM#RVHAWr#j9q9EYXNhv2-K=I~9d zUON5?e${UbjMi?(&&QoEc>xa^iD*@GhKAF{UX9{bIVVd5IGQc!vN<)!->%L~YSPbB%S}A|U`|+*G#1QK8O(E?>`TFWOMfk)smOD`Unq#E zDvsedV8FHZD_^Zn&+PPBK7Ve*u&{By7u}PeJO*oKI zWPOG_y%64Y;bm3Yv(X13aA;LZMWdsj-_VP$MP&nd>r`!{cTXBPX46E7?vJJ=S!2~r z*R0G0oBP^{TrGg7egAEsRtKa()qauVvJR?cQULAE6r$(X};vF$DBK0mMfdyBpxdwB8*wGENPx z6YLMdYkPQZS(aB}e|hobfy=}W%bfm%_4mu{pc6X({Gj|FUCA~yHZ%CkJ^JS^BojMF zxHn)B*I;knK)imkzkh{ffiasuZ@-KDFTTRR;Sl{cVd=r&6ZL%4_r!lHiE96=Z|3)p z{pa=hzuXG?n}^8s+e7^OQTPAk0sZM_&O1|q&fpm9LCqCF_G#hP+DkWNHQr)~ocMe& zBeX1Ob?qHlk;s$F$Rp}LxdkJ;#2yzdTfa^fPQq$%T#jV({)>m0nNl+`hXV2tLGV-YfAb*z-_PMM zjw;jd^FWOMB%c4tGdNa*QbAj|L7GQa__$6ryPzXTj4q2}l_LQGmk|WZLHZU)jER^T z3=Bo6{3l5hb#P!H33dJ$K*e76V})m6%g;_&%)>)o^ z9v?{6)S8A*D^!#r5{hJ~OhXEakaU+ZE`mgh-_&J@3?WiR06FLe9aHoTk1(W@43G@a zWN1Dhrs+rwsoP?J8bC7Q5dnhqz(9{(5@gYb4CzI{Ly1Q6Rsluy290<6!Luy@3_=E) z8~QzjxzbE@SQK(gnF-XMF&BXp1X4v}X8LBrD7*l~5jQjTXP!Y(j<=|z1qoy_G-MI0 zB$CY<5KN1BQh&t$JMh&bGm;{i^r4^u*vIHpFx`Q#n z@l%Hb)^ylblbQD@!kiRuj)oJNjEV?F;~KPyfF@KZL?QYsCQTAXY}pnfhd<;#AV8iV z#70O14&Q6wj{?vS)uLR?YJjv+f>>ubFq6Cz+}}O#ve#GEnspq8t$-_K>6! za3CZg#e(Q?fJV5_2{qqzkkEc74ywC)0D(mYDMtt`a7id{tVE&?-ia>Ap$z^qT)ao> z4hvYqfxI$pp8^um4-$FKd-yl|F!$*c%_ss084;LrNpRa+M8r$e|HgjHq6!qSgd2m}RyGC(zX zc2VYWYWE2r#I*`6rvOc+f(C$rK7P@_R3VA0fMK;=2X zo*2!x#GW7@8s0YSzUKP5JbQE(2Xr81C>*C1lmHJXrGy>0b0uQ{lu{z8*GU7h-~}Nm zG*b#><0^P#eDw$etfDXU8^V24IYVFqX`-Oe7WqX-A~g_TM^E4pTZZx{KVB&Yc<;Gg(-Ssy4zxwiUW?jgp0Prx7dP^7s80qvI{z&+#!bW(Eo$I-cm zA_p?6#I$Anj)K-x!vdhu)yrlx3IyS1qJ9i|<@f%&?w&siHOGm@<|>suGPezB2puQW zQ>Vf@6LEGCQWHV?7wm-m3wEYTD9Y_B)nX*XCPCPU{Qj1I_9s0~2Y{VHN`&V~9u0*4 zqTP@%bs>^}(&J@(C3!(qf&`hP;z2cxC|{+3Vk^6Xax6%RFV#R zZqp~7J}IMFY@6jz&c9B+nz}-0zl~PN(Vl#lENxUGZdE3fA}k^o`;O5|Dk+&5JnUv7 zojF_$*ujyU<#G$i+{<0lRvN}7XMnA@1sE5a4;I~j-270L0K2Ly3${<_PJR;>9OcN`hn68`ONo%tt%nBSJ zwg@FQq8Paq@(46taRWy;Di^#Bwa;6$IZH*wAi@M&yy*Ti4Vm5WbVT6bY8lh~@ zbnG?P=5E97GbhVB_Vx@{hXc$*P+~{0ll|}B0P_%~#psu0VgSu=3$>Xg9Hs}GrXN5s zpT$^#1(ueOOgEHg0+%!3+tHQ_d2>F$|Deur0P2J?0@(rYYD$!5Ve}`4sU!AE)S)_$ zZP@wbT-fue#~@&4qgzx>4ub)#;D~vrpt0115R&Cp(^GtQq}Rc3ha5R5^?hCkc1FvX3{I?}oAf z%dvRslFEeMI)F4)MFw{PGBRJlHx=fAU?Siz)EWK{^APw)uta~E)!+{(kZy<{dyQ{$ zs$`mwNP-z48t6%Lc1dD(YT5(w5W$KXNY|q&Eb_aLG;ORoUWr(ODi8Mkp4cH);sHrf^v2!7Upk$eey z2DjPv&hLnIH8G~KRyet7i;3!P?JRAE|uS?+b+$(VjxA#We+tx)<=-{n4$s&{-M3dz1!Wx1U$BFNr*= zi0i3i_%=E|#G{Xc_=6W9bUD-LE^u~h8{>0szv6kn|K3{`_5X~M&ftGNnn~y8z*1gzYt4}oJw%FRaSX08zv0C{6?oD0) z!AU>N5GhwZUh-VK#!j^LW43q1At7ZV+vomc%TP51%=X8utvNaG*h(i8@6}?J2Ov-} zsrFS8b>MZGD=|{9x?^jY^Aa=6!z^WP4e@~J#WupTy{oFq6Tg1_Fjg?~exk{xT)EW( z9|y__Z}~FSFqS{kqw)4TSZ$b3JRH}h%#yjvO>VwVG0Q~dv(Ih$-RCz`O$MK4{HT|d zV<#WvB~iwd{G13{cwX z2}>L=uyVMKp~c*Nrk*IKur)Y$lyR8as}H3ob=yg&Y@+rLnNG4>4vVx&_wiDPc@H|S z*Gqrn$+I3Z&@MOmdLk`0G-_3^?C$64p`|NdRx0RkvOmQq2;N`pL3C?UE}!N^mxb#x zQK`|}4#QgrMdf|u|Iq4)JXgM7>&(dS&)6;w=5?f1?%kzdgPGeEb<0Y56COA)_{nxU zRx!rZp#J3nv-5<1`Od8vD26u=^?BXV)AiR^ttR`7!pVW_d3xLtFXBVU-sUl8Fo&;_ zA>-8u#k0xivUR6cogAM{r!G~tL)!kWE<+u>hj`n$VSeKgAKm~lmw<26bUgLurVEtT ze9lorVe3>YbAeIi(`k^_!+{VD6&DY$(I@yg>NKff`-I(Z9qU$=3;kA?d)^{D^Ef`0 z{jl9LX;vn`^`O4pu@T*Tz7|^iUaWC4ht1!o_sloPY4ZFW#?cH73oA3g2_erbI0u|- zV(@+;jlz8%r}fKj>&b4B9+danj?NdtLyZvpf#^GYiz>8_L!Dxh@vfDnYxR5?`azTj zzYBjCvkaB5Q;}Sp#GUp^k6kA9b9f?*UxNl(E#SSkW2~vTBKWTC>yFWx!wkt z8UO*#_cA_im4gYYs&S8RwO-pc=IAU7@9_lemR)cuDP~?|FBuaTjpeTcO}_`o1}IO- z`CB~X>$&RfuWRWzeSa+Uw`r0Jw!WmM?9Fpl%V)cP`|9d~6UV(X61{QQG&b>iyy(@g zsUEHtq(U}k&yCF}lfyjBK2bn%q+%nbHh>`Leg^bt6lF8(bB3vXTL9r z@%q+hn{;UBxlblzx8<8w>y94JrIm*tB=O9KnFj8Y#eI5PIJw)y8kPaSpNd>rAX^VN zvnI85#WbW34%T~(JbEMDPVd9F_eQ1zbXwaYyRc0=O_-i&_v~9h2)AKO32wlpGG5{q zX`?ulxuLgy-s>`YyQ1CYY4`)7sY8`@|9KT;G@)#e_h`L5@7C(lXF7j7cyI*C>tJ1n z$84W;;G|Bb!aUGPAuZ`e<37PFzQT#lCRB!D87n@K&2X~$S#5TpW=2e{y+ujQ$MZZ! zxJpM(r@4E_e1I_9T|rEMkAJk~7RAtTL^3c^w*i|N{9RMkeokkythKhhpV%)VcA0Bg zX7dg=dv~A!M}Fez*ITLT#)_4U{O84WiTcrGE_!k`wU)}Whh251S4KDoJ9G6K>oFt0 zQq7Eg+h|x;&(IQ!7_@vrQm!^a6NMZ056o!?AN3DH`089788$tG1*nOz7otmH$u=yXgM@>ceiQRdX9@^RR$Fre_Dg zde|||%40qd9Dj9Pz6)LC&M;xV;Ye}ObR~MdFW3GOV`>FfEz5$XzyD!K=_chq#9=-b zH{4eim%DOkv*?VT_|)p0e)AgoE*3E=AH|k=z*2K`Odw)^FyDTc`!&NZ*-Ld$v>tBZ zV`)2XJ6ew}V~rbjStr=4vtDM#>2|WAV)tTo<9Lf^lyE)D@m{o@_l8$l{-G?Zg`n** zJ26Ak`g3@-5`7l_)cA%XXP`f_=!;XOV|mv`*R?(FLv6TRGr`xRfH3#+f_ zBaB8`(l@dB4}}i`$?-tSSGIA?YTbIt*YgsZ@v$=XML7n2Yd?1Pg*(+?F}nv1nx&Lt z|0A2`n(6i-+ivAm+hyULzVvL*>pg@(T8?+7+rE*d;>WQsKL>SR6JLEtPEN60>5hJH z;B@*Iew834!cBRrx@8-cMP1G_T5~IE*jhI*oH+eroD&oym$g>g*Cej77APP2Elak; z?IE34Mz-B&_o^^=RBbNkK;~`oW3f7iUaq`k&LjO*UFP8qr1_%f+#E1|#b|6@1AII~ zYBuYMkFWB0mNs5x{`h$7H(f)y$w&L^v@hceUg7g67OUI)tySkYbr}(Utrec-JMk1; za_N2(GkH9DKeZG;^~n6cis{a$lkjtS@G{KnOf zb5>TXuX|Xcm7{CfJ3oe@cG74`7kM0+h8E=q6=_7HS)D3+SU~sfaC3eY7&+V!3Knl% z(fC))!!+ND_^UbodA`s8^4!s`7zyF8@8utzpUb`6Xz|~$znPiv_Dw|l4AEu|6dLVAF8WIo5M?4bjkhpq zNclZ>IWUGt3Q}mmD%Zq3C@X!9Fh|OSJnY`m3b-PHW*iC3hs-4J?}r43>~DsQt1O9P}bNhGYYNV7_~k}`#g)NJhy zxT6Vlq+Un?qzc*uoLaR6eN<34bmR`#6p7UB%TwHt3g(9>Mje-2*k5^EP8kJ?;2kD= zQOOR-Q3e#@B8Er4kc`FvtgVuR3@6M-@h(LMd6!ZD(@{N&26u+~Ihp2?$Qc7sNLxX7 zjKHs@Sh!RgmW)4D35_M826bu^1+a6g7*4 z--|h}`Rvn8-0=1*c10+^fI5H>g{UeWJA4-dWVCFK90f9HW>zsi4PMd!30f2ZlSFM? z6`a~7U2|6$T}Q0DBr-_hCj?=z1%seHMb%FNxMm5+MN6xJ-K(vFGxJ#QeRLR>g4EEzD$!7Dz=x@KPcUKOMrr;G%h$d z6H;nWeYORIGIfcF31Yb_`~r=u)m!+i1a@Cx4m9zNuD$RR9g+ z0O*PXKc#~?Q8!RX&VlAJ74)|px*3C24y3zLo?9U2j8q{&M8}jNIl_e`p;71I_|u)O zb-V8p5b_xDaP0}wS&fU#p{U|(o=QmJKu7m6Mt-;BDcAmq;#T~L;wq^e$PexgyiTTx zngIc)S_Q+&S|ZBw12G9xVbrgt^$4B?nd_9&0Z;WINC{3S19`{^Tz`=4k$>^8kwX_( zO;D^CK7!P#{t1FPUlU7#3)9i|{Z6nCR7Efo6=*Hx%d$ee>4|~y zKdKZfY+SfSAa5!dOG3M_Yaz_FU?_#({Bmo`v`VVzRY`@vC?Jov^V?vlcC5N))R=_}Y!BqNv=_p&?>`y8>O!uQB7V}$_yltcr z%v5!VRUjx31@W3n5fxJ?wtUM!+MjOIj90qpKefL~*y3OJP=Sm{rD1fru9u~8;}CxL zeV(u3cXtj!7)gw!^0@1Y?3hc+V-}(He|i+(K_UnC06mJXgN#lMK|lJ9iZ&sP%lbP- zlnpX7O`udAs`D2$geh>lSM-HdqoNixBc@);Rg1Y(nr#uC@!%SMt2S_B<}BA5Idolp z0(SIrsL<|v#Ae6e1K{xSNg{3k3_@F#Yxc@8B7Inoe5~m5j+5`j$syB zn(?w4$<5CoF?_+|frJLDU#j9*RY4Cuw!tpr6MuUHtwK0;!b2f(GP0|QDIYp+2D9xV z`SxX4QKau>W>6X^O)X_ohjkbxrl`}BYcCRc>=##-oQ48co7SlI!>aJP=ujy!4Ot>EOktU)cIq4YNswRw7|e9xlaE9XvDk+4UF!Np;C})tWisAbqFU`d@YMK zspy1~6LVXF3yV8N`P_L0?qDDFQcpujF+lz2WstsW;;5a@BVn;0u|rO~DJ#T!`(yUbnu(UR&&e5F>^C=tcq(|YI&iZZ2#qsgcO8Kf&r+?E;dsI|U&RQn2 zy+!YPex*{5>+yW#Y?d+qP0VWbXNUgp*{ZiU;O8_##E*euzvGtyLt%*CiOFiW2o`fn ziD=DD;$q{$&IOv)k351#H*DVNxUMGBSMqNTe01+o}VQ>9!|JLZMptldui!E<&_+{>pkDNy?wdMYKEYWt2`!=Wc5XJ z8oEBrW%p{^%=dn;qu_ak2s!HyGif#B`EcJAI3Q;EG%Qzl3b_gshk>tJe0?G`cb?hE z?(ekIYN+Drn#>Owp1HLhjoFE?f!A+gC#zBx#6Z#6;IT1)hh*i+`iA936sqspV4eK! zdIptE>@XQE7)QFXW&hFgVSf}8oKEyvEwQ=X`{}Y*j0@KL=4eZ2zeW-MU?S(!_Fj6? zupBK9HD=?Oww2F(pLr9yb%RrUlJ^%s3Ef88JeDlK!o_XGe$EW%WBjIZJTW^G2>IvA zK*v`8Yt-7vcaElcFm2Pp^_Nubt1+s{;+z0+Qj!zdL!>3n-n&;kLdrbM)kaethQ}k? zHiO1BBOkll;a%Kb@H=vA`a-;Um`1(j6|UX$1j5P0qo0|Fb_Vr*dQKC+_`IgCM5UID z&%y?e=B^hyqHiOmoEE>XWndBOSW=`XY1=zBWa%1dY~Nwba%Kd|#BkBF>Do72FLiIoKI-&sj|Pq<1RW(6A87w&!i2U3^%_U`UMHm%iX zlI)2VwUO9&R|qG2ih5sM+O6Q7*%#T3y~%x@@;{({t#b0Pw>)?svOzEWF`Q>TF(!)q zy1~Yqp`nNI()bwoYzwB6<9@Oa3=6NLxC9sB zy}@K2W!|@^%`b6?4vwLPYNOR`tbcQE`mBfhTQ5QXMBvU1`sB2Pd^@S?#&bNZ;p(Hc zRof6IA>YYe9DI+w6*FjkgN531;u}nYvY@6^yV+XSCjUix%lS_545y;*MZwtSuG6>C zxVxcm-u@iF-LP=JIhY*>%v40Y1wzGmT)!?@Zw4jdPB=~mNG%LZa>ufC*=)wvoPh<7 z+m7kf0si#+_AoshmUUv1TTwe7+@G9gdWU;wdUc+sz>?RkzA|dwHeaJ0Yg`1Lu&?baDpo6U)Kl!tD*|vmK%h zy)Q*SX64A^x4J>{y0CG`&pz?~>`=EAm)T+Yp#liMoSGJn8)5=z%`xR?b8^fB6waOeOM*{Ncz4KNNTlM!_j^6VSy1DqRp39leJAIdTTf? z3l+}xQOTJ0@*BD6=PUN)TAE7`KjDqn3%X2rwWiXK1wC@gn=lthS4VTn{fFq|5(kF`=g6a8g|g40b<4Vb zk$3JR+k_UFF>3t}o(*pr(f5xJn;PynqtQV+b(i$ctIT=kr%f1$HrqGdk2@+o)~%}x z!s}w_c3!qhjdiUYkr33gh3ic;Dr1 z>f63jY3;+EI-Mwt>S?z`0G8c|Osv=LJDRvQBUm)skMtr&nBOn5-wxD{+4q%mw{eY% z6W_lr+NO+W&j;b->!W;buNM|SUi@A{jmK?4vofO5o*x_e zbeBKO$UH7ayH>KbLzSu9l<-KR(X}6(=D0D~vS`n7eYJ_Wdp>S&`k&e5ena26kFZj@ou#nBss)z8rIj-^%mzlQ)H66yc zy>+28D$^F{gKjXdUY=|vA)Wh?Fao#M899TvCwqI#9< z?iVe^Rr*gQzY zHcbdy{MQnO*NBU7*^WqyXOz+z82d-aid*H`RdPG4<1J$H)V6S1-gwKstrLu?sP@i zth;At^G8d{rcgz%;os6dotfenFHoC3GK{%IUak_o9enuPd{Aw^Xsgey)d}y-lb=>urzf{l?~MP-DuoIeym0)5e`A)RHgf852{XXAxsNwJ@wQUC zQyZ6w#_suY$LH{L*o?2|`!+>4ir+^ZGduE%GJ-qMtuMC^&|U(u36@oY zEY7_zGe|w^{JVj6QiZ=S1HfShfV* zoqo*!=c)ad0vVJB@**19)>pdbIaJ&)&_qGHC@5ecNMM2jzX*0nFoz_VkWI-dX0*`Q zK^;o-RA?z|3BpU}iELG;p;atU8|Z8bchnltnk?_v(+7LcZ|fc8RC77aN;f*>Df6bfdl1hhl^aQbL6bCW88_WabKJP>6N~*`vgTOI9gM1V{xjKoJY4LZ%EW zP(=!vu;5bLL82>59J@$_3)Di|mRO26eQGl?-a~ICu&fk=Og4VK*5Vkn)VzE zN=8T+U=XXp$yi2TFL#{a>~WVhD~ zIOx8p_7f>Sr_c>m#LLNAm)PjUV{=v)0 zKFwgD7`j0730S!)!~zT|QsyaQfV0sM|8&OUmsq~p-uLwv?kqr?; zgQy0z3nnPc&iFzS$@Q+3>qkzxa2dg5uLRn`oGHJ;hXzs?OuhxqKp@<#$E#PS0-3p- zhEJ&&*opom(s$_z@|Oy&2l@W$NPsOhBa?MjA(#bX z4&)yd=o$hVz%vSgis%k-$7HQ2jYB`Np{dIO%OgtnOus-H1XRJF=}s7t`P!mwAg?w! zG=AHx1{0dFkAE7~H_uQyMJR7O}c5%>;DC?ipNqL}6{%Rj>-A1`hgvn-WP#Yd9B z>FuPIYm(69TaX59Mf*X5Mv0NuQLnHvL8D!1QXuHRd`N=_D%^A%;j+L@9EQ7urHcg< z%m}aubJxCmG}L5K3Id%#HYSA{sTnX+Edt^BtGZCP7h9PAv`Gy#HvXmd*NmM^hCVv{HL={OZy($8X^?xz?z6e|dJ$=1gY94OS;wXzntj{&xfSyYvU zr{d&oAYA{9j>y!bnOxNW868mujE*3NlK!czgGQY11NFw+lX|2Om08KP?*H_vp!Bt4 z6tJ*XDlBJ0P6=R%4Fk8H*dRdx)TL2n5D~Z_Fw@E-)hU)uF4ka5<-9VMk;Tl+tjs_? z$VwU1SNmgM;J|arJpwYa?cuOV0sqLz?oEBu2e1Ftbw-n}0!C?1&7qqLD%QHpoA{f} z&y~HQXoQVt2%cOraeye}FLA5_(|PHNVAIoMxJaSVV3@@s9Yf_dxIjRrw?gw%LmP;$ z1_Q_1`2~(eEfgVXum|cvRF2nORYT3Nl`s|iX-cbC#W7PPN!*FrLB2A^61wuo2<2I; z)ILlGB#8P@S}&HrB+3wcL>C^3M=a12(v$8w?N7<2?}Yu=62UrN=SvKNKS{b`^O6CZ zBcu@`I~vMBr+FgpY8H2<4JI^T3>d*-&44Xqmc1FHldG!gg5|TemcKSf3Z(wp9D&^O z<#ZPbRGw|D#9(5Hdv6%+0_w2(S)TEz;&ep{<9)M91<{->qI`y*ybxUIgPqRUkWnQ| z@7n`)eD646xF|YnnE+py)#N7seD*Y+@M<=4>4megG@RcVu{=ZK#GIv>X^hW#tiDKx z<(YqR$6B0!I3OADLk1NKmMy)bAmW3=dbl4DaHeeAp17iaLtQTQK_0D!_2K$Fo%h4b zd;|P%VQE?nZe|r+{la$VzOQ)82G#y{q$fV;`}Wg&AKnG@P4lCp!@m(=j^*e@f#oRB zH=ADs?7{7WkM6jlgAI*hXH8!+k$CSr z?M7XS$!^a>-;?56cuuPt*X=u66uBKP(_vOwu{sTvk`1-! z>w?bse9dgBJDDhC*^TBFQM1NPJXBv!oN{_7cXd~d;+51w;&!Rb=~vWy%bDx3X0jXA z$ywiXzocBv?|gxVv(cXYdiF@@G0--0w!ZoJtjk1JZ0_b7fbPKi^vh0B}&!u?YPM7^j>e>k?QrJ zJJIDKWC9*m>&bmMjkGF(*(r)&n2CeiskazKxYRemW&;9^W(}G8iP~xq-7I_JMWxAU zozgdU>hm$9kG{H$WU0D0`Z+`Xtg-6z#k+mg6?IW>gF@@PPE+GaQoj{hae}#*{G%g_ znrofVQq|)AVm%I3zb!Gu<(6|VC7#Fmg+%x=5w<{JD(tXUS8v;r&$=?mDLnl)Zwh)E z>bC(RjTh4A?z!gBjG?}*6hC3QkL4_0Am#9S^jU?|?iKEO&^6Ma{3{>JxuKH#lM-5^ zXStQNd91GiO-@p~TM1$|gC zq{*>@Tho^Q!1eHG+S8SHhF@0(!&F(Q8@DMV_4n?Fw&Wjeiw**V%?#^NkbbZE-E*N8 zr7`?iva&>6+A_GGWtJ`3=2+%Ej1Pwsv4rl>ia*#}o#^>nsBY)W%8&S{_e zu9(&9r@RzR`sv3&FXDmy^j^6fhNB^{Tjj%!e7g<`>8fWumoCJ`VR$VkU53jlZ(yMx zqIW{YU3JN@v6t_ss7mbj$#7pA9CIosV7@*wcqNRi7p-DXNY~Fa!LiF=VYKd!_GPBM znAyBYAIfl@p{AW=#`D|pqh>hqvOc)lb|1zhYpmlsb7TKTIw8eW$U=h&8<2XWryir6 z#<$hP<}V{zDC?T)ptnOQ^sA@SI z=PVx?sB-yR*riL9cHE7@z**zH_RA^I|`@0LGfM2KBFB zvT!qu=sdn$L^hjM=5lR3D|;FJs}m@D32yb5oV@T+NQdwYq|zO3I67yUH+gwLrw%?g z(_PcaDb;q%7rFpHtjIhxZ17~Nau%Mp-s{$JtPXYiE93EUmY&{qQ{=Gl(Dmf!|;%+;7{Clg@~AED7*Xdc;*UsCrzG zU@vDboK5W+iJ`U5pVV7MD2yi^gAX%;${m}}eR~V(eEG9?9Uj!oa7?89>Cn|Tgt&uz zco2`6j{W!O@R|-e5V#Ymv4pxa5Ya3QloT^Cbl!XM);>9qw2_~oCEm2hm|0=?xRk#m ztzSt!U!yRvEtc1zMBe7ZrV6!|6Lz>$yu=1&M?+`bu5U4ZQ_WVLO_H;EdB%-qk<>hkBbYBNn5L_8blw$@xt z1l}r@h6cY@kAI%PSJy6{aZk`{@DT1Z9E$!>EID^rw3fQ?V7IEi?ot-dqF0Qzy7P>< z`BL6==bLS8;J3e22;t#>D6R$eZ8aTpywM9ugPHd+%F5&^R@^UFjuj+gDs7goS{Lrq zlDvs<$4Tz1^VO>f%ArSeb@rCK?8e-sE>|$9c8WWw&bSQpGv`WCzT0t95NR;Fu~AjU zCSj{q0#@_f6S1ZzIyqwr*^+Ml`Yep-+&;(yCew?>g>M-?2SU_1AX)=FHFkV~R zp-ZvzbY5HT@|#&NvFF;6m+hSYua-7+$HCAn-vewj^+;bPv+~PDTzXF%gbfz`dCph( z>RHYZXz&RPrTWUjVfw>LM~~^(dold?u>@kco(@@T?W-Y*O5>S&(}LcP?Q^p?tI6KEeYH_XHagu z9T(-BzgzG*1I9bAcOqZUe0Z}G{4O+qA9pd7TDKJIYxb{xpA$R|WQ+2^sB5|zc|_-{ z@@bo#KJE_6$k~dEdz6;TD59^HFf{y$-5e+wxx(6Z99=>NMz z#z1HO=coQQqw)Tq0ouQ!n(j_VbIOMmFVR8b#b}#ZoeTT*xn|e&8U3aiYDsEaJ>hOW z`M`e0y@^A?|>0x zM=T&9rT@Xv{r%YfyHopr;N5?5TF;Z8+2>Bb4D$E(VHnFc#{1-0wLYRNg!5NE#0LBTf7W&?~^l0xg&NY;YZl+ZP=E0zf2$q5! z1Ro!=Uv2V|c<%Ili(zp>&mQr}@_AQbzLOo;ytnfWpe`DL34msY5^;%`(l^n_<_eH3Zu4e=Wgs_py zx4-&_DzJEo%%}BjD1Le0KOJkUy*bo)z{$O@huu0U@kFst>#ZZicNYQaviqtx2J{mJ zI7-mbd-s@zSd5Hy4~kh`;vN@-i$t11X?(gt=kpBA+DlwQcM{^WLOQB(N0sPXNz+x& z_aGldG;p9LJoMMYRD27LmOJMZF9?VsS)9aU6)J6`UkuuqS1ScSD@;WzypZzY0fFTZys8%*END&&V@6gN=>W4R1 zH>|~h>kdv`^7u2}1|j(&KdZ$OKCystQp1KN%qtVG-dl~C(_(qaz`w!W58QqxVy2~H zrl(@#4iP8;5dgo7f-^p;vpHR8UL_)$eG{sc*~u$(wsmlC>&mi+u5hHB2b!|c2^;x?L|)bu3= ziOU}K5(kqI6WNm+^)~g$)(y)}Y7U2x+0C2rg)`ywR zSsx!_IDV}9yB(SP7eAt8p$UrK2od?}2)8gz2S!ufPVWg%@zdwVFF-C1XjQuhYu8H;=Y8*zkmE0gj4aoIC7aeZc?xx+F(W~tn=$yu8+9!plq}**N%-zg60*z!?e7eHF~+N-i^;} zV_awEEPodQ@42~T7Uqka7mZl6r-~=guOv9eY6Q&#OJzlaZjAd@NiQjn7iB88Y=8ef zI>R!Zj(14|V>4y`qAodIx4gV#uIm(p(|>-sEb=!hU}KO~Q7 z>|T6?Op9$K(=l&xd7ceL5lm^{g=)1Ca*IoF@CI2%Jp_4M0@mPvc0i6bky#Lh+&VSy zuKfPAvZG;?TuO?aHbL-Gb^BoVWh*U#RxKejW9@kSRh`V^{KUXK_fg2N^Bx&xJ9^&A z`aRKi0$<4E*rP%1K#eO!Q7ex$6Q;>TXjHRt%(ht(M1yX9ls<-Tkxk)hK33%T`6h;L zUpqdD_YM;kLS%8lGY)Ad&1iN5swk;*wW&hjR5Uu>b)i>-4Y3*P!+u|rk%wYmIe>6u zko4uZk#ol<7(@;G!;hg*M@X0_8IIcdAG+#96PU&iQuz0Xz>7dRyk*fBIcWBw`xXeQ z!2|(bTwLdkLo!kKXUa`<$5+n=%;(IyMd9pi1bYd3JA&i|jhI9QiL-%nU^OfUp>u7N zpejzn(@I)UHf?YGSp65w0_$%|D82IQ$jy!o@@4djN#m)m55vUV&898$YUnlvuSQ@? zP5$x&Q=Dm!SMnDoSRG9~VkV~kKs3@uDU!Z4Y%jjB9F|+?$5gOxYmA*$OvhrcZev<6 z9@#I!j<=F$o@_!rvCHz$cy225U*~1TIjag8g~!OyMKW49PwMB_s)A+NzmhxCDOvkt z>hh$@*1d+%l!Y8l8%JlY@eMuM)2@A#4~_8p`x)0d*wqqxbZk9NN~M05V3p4^M4kt& zzV`;9_z?SgL$D*uXoP1O{urt+P}HS~C$;o_GqEf<9F|7W+URqsc@1!Y$Ow~C$fM2G z8<_+jlf1a!P-K!oosv##d07Qog1i$Hx-+5kuCO?n3K^wNmdQiYG%LLnr(Se1cc94z z2odD?oh4&WWZ!Vh-?z-CtHPy#6mK1XAEUyA;qu>a6pSKgF`P6x^@n# z$&QZ;WuWAA(L9r#_dWfpo5AlUedHZboOx`lhL$$cO@MN>0EJ})y>f#Sif6qa?U zJ823tRhPAaR1tG`-W4W~|5c8?M~!wYn}=A{{Q&dKh6yb@ zl#Dkc?bJ6X74f7$@~~V?I=bFun2yZ>s;ui-amnMLI=Xu2r!PIaHWmHAot z*!{jWET23$h6vvKw_(guJ!5N&5UyaPW{4!W-CHch-E6dvl_ROynK)RgKqlzG1J02S zC22o8^Si!i`N0?IqNn}9O)npr=@0_3(dRxuSt(4JMX+phE|*+uxvagg4&wSTyOZ~WHRc(wW3L6+^( zT_(65!wsh+t$=d0!93!#onb8V;|%(_#u?U~A382-2zN69I$_EcH^SS&dJt7!h(?+el4N2XvyYu-K)^W&0o^EmC z>#~is?7cR%u5I|O8oAor{{*v^<5TR3WUhop0X?L$W4DGY3Ivy75{BNuYePLm%#II6 zAImr6G3wy1ww2kD()+qAMWF`2V=NBVF#y|lnXF>h`YjTd|Iu6hg-r}Yh)HbQ!<=94 z+#0t3k{No4g+qYU*Ehf7^9>$~Dx5+})0*sF5Tz8B4-#*lbK-T2>-#roR4F9l6UwdS^Zh)REqOIQG3k@28?h_M(gQY}PnX0Yl5|MJ&~09+0MZWpSJI;d@y zU;f0H{@m&RyQ?F*e`pzW_KtS;CIB~CMo{&S1{}lS_Ykzd9?RcNJJH^heK{1`<=kEj& zIesr!8>w|+S(`c~C^4Q8mD6Kz)}J%aTVldqy7X-B56n8&vfaGX{|&B5cg|IRF7ETC zuY}p9s+6c-2fgVit$FXD!ZYc+e6=006V~t4S?+1Gf~CuWE~cCXhYU})m;{vjswl3` zQj^Cz`Bt=<5&i$Cwkv^$YHj0l>qF%hkrriXp{`}DV~>hb6lD*QWh`TvYYAmn@kywZ zHQW*rNePuKCAScgJt8DoBm3HS#xiq`8RN|8JNGwt&V=*)&;ME8=XuZjKI3zy16wK8 z^Wm|+*1!(q>X_+G!{JSRaa&@Xx&)LwZclyZF}&&HSaKkneoN4?@REb;9i-$Q`*a$V zWFBMc<-RgxFBjGPVER&3IOeFd&-c z`OynpJCp6V!`&%mb9#S~>a!QL@iJ01>|+6&2G8w^O%Lhb8`l@mVOO+2v7YPdiN@nv zLhk>x`RtHs*Q`;rA1*tO&6{ZV-?iV`o(*;XP*UrWJa&QmN(UI+XXWQOMFd0^$dTyySEV`mD`ZZ=*_#;1Swel_T83;o1a}}e8Lfx zq{zzB+QSyO+j{dQ)&!aE@wGcyWb$;S!$ZpD%`-3o0?-M!DZ>MC3qkt~Nef3(V)Y+_i2hNep9zyZ(`(oL7aN|(NW^Qleje90VdcKfSoP0UF1`l}+w74e^^(Iq z^dpjtr+8|1RyplEbh_q_?St?dg{Q@1Ci2y1r~cU9rNeUW)0@mx-sAF)nE2F>2^+M8 zikn8ug4oK$uKuTNLu~Ml$(T`h54YxyS}W=wUj$I0t8Avx56Rq+r zG-P;~Ems?Q;oG3z%uwOpnCWJ7U)4U>@sZD1EsyC(QYvTM3j&-Q2dA3kJX(D#lg^fV zza6CW*j+o$!0_I>C)vsK+&4Lo34xa8+<}7H+MBXJTVs3OYonVt&S1tGMLcIdO|)lQ z{|K~s74qxch&OL{e5S9spPc7+y+;BK75>cGo-+k>qwaYqiD75M9mi^wo0!%wu=R}q5h|wXWM8{^acC3xqYg#Q*?Q*lUVMH(y`JWJujXg8yxRfWuGjM z-4hli`pQ4YwSgToS;P^u&%?66N}RoM;De3g&&vX`Pm;2nOTP`*+K#oaj}jlE)svg~ z7MxZ47GomH>@k_l;5k_Ad;P^|efmM=591S2Nj1*3wY^a{dLvF$6mNSsJgQ(gK|A3( zNk_LudNX%JX`Lv!}dnNHy_ww5Ytf~v;sH+L#F!*{G`SeQn_BbW3R)hEV zZrJ@2kBNTJ82!le^jE3cy*FM(hL&#*K2di3srjeYs!#hgktejbi(rp)UsOmN`8g7I z`R8$==1_}reRa8~&pi^&OCvcWFqXx@PLZ+-+YOYJA&Wj1DuzV%H~Uh-`1GstogE zs7>w(3Nf!e?E1ltzS$~R(P!YUxBokmnmW&D`FV8q^i(Wd^q4liOD(9*y{T)0 z|C4?&llIiL04F#8;zwK`_cGUqF-z0>@CRUbEB8oM{AXR;n${79_gTM`wtDQD9_B5W zWbU9zdzW7mnTOgR;f~rS9`oHl^Zw;q3Ez094@vbL^>mjHr{6p>#gu+W3oAM8xNA~_ zXRHJ|{IlVgM_4=5AI?mMblwlC*VcOyY*2H3yy~*; z&CeAqjVSdLgO`za=?#EQy~2oK?heH-NsgkS)ioMM5Sf172&QMrl?l2aS_4H)3 z-#4wE2!)ZJ7tdzUwnqa;A8~2*nAnBn)yZTTtEV%|ed@)OYPQ`P`kdMGE9FOcUT~@0 zl_O`b#|5t5FUQ$Db5fKu@0nEN_Is+S%J-i<>=e<~XS=n}T&!Frr0;b_lG5nVyOvgk zoemdzbIuHuCUj4vPBc|yWbm@T+?Ug{-%_k$==$rU{6FlSv-bVaNz!4D?t3v4dY?z zYAsWt7fwBp_7ude_HFB*4Q+~rcc$e-Oz0+jffO8f{8R6QlAb%o z{l96qYb4vkNBt{KV$ujN<%_+zlxFtuVkzLr0J302ijqM9+u52KGrIYoEWG7sw{*7R zKm5d99ho}-{*wSc)dM&t7`&pOW9Yq_4*RboRuH z{>%6i+*$sEFTLe+fS|uI|4HlSEdG>3*#I}ZzT*5~2n)Z%`{*H_3eoFwMn2VLJl`%B z>9Injrf9`%$HD!(h!yLE(8o-P0s9x`jxW>SqE`{4XqxA^f2O{&2jLZf(Z<~e@*Umi zq(gIIetf=k+x!LkRmPNOujUnCWe&dCvxS5Do(MpZfzIo3Xms4Ce(azBBkT!tcz!Yb8$O zb>pRJHFX2ah(LVF$w;5K1KclhSjDz-?HkU15(BZenP zBjJdjoSsbRv9DWh^E5g%CB<+$lw5yZbw;GHE7L!zEUqxAQw{w@q&fP>kG{@Edy51O z_TI)OIUT;MU+E?o+B1XKkGL79T>H_@@jh^9gq^jMqkHnr0mD?!o)7sW!5OigYkbhM zKCN;5uX67+{QN}+&7Bk8w9m)_x}yE9#UOE(PcC(SsQyajk|5*Z{?-b)I>$$EI8SRA z)L(3>E-|Zgw5+$`nTW|7vUQu{v(1UmdSckx?MZSJ@7A{Gj!o%dOkiYWZnJi)~_!ln6a=(70-U!joN6+Uv5m-BZIBed21o zbo4djmW?GW*Z%Rj*%xlW-Sjj)1nlFZzz3`8sR_{O*$ss z`4)6irAhL2!|2gCZkfo=u+Ow?Ec9n@b>-qUG8l!(`*{D9y%n8+%5;owm(*} zuO`g45zG#O*ofFV|B(^Vdk}Z(WMpq?Wb0^XgS9s_ap%?+f@RIV7j%b^|Ve1ee4S%ph zguEat#ORz0RWkm9$Y25zhox@>Cj)7W#X34T+8a@KODKrAbCiff;x~K3_Xs?dz=<$K z^jC^RK zduie2;M|)INFkRr7`z-LC|u7ya{)hv5f>Dc6cRz2qOBc`Ot3aKmh(?y$OSz@6!<8d za{VC?1>OVChmJz9<-x)Rf;K-14hCXrYi4a}ZsABiMi9_gxNJLT1)$L&LoV7p!Lj@eDVT*}N3WF&l4+{=UgMh73 z1dD_jQOuIH7bSy(zaV(rhv zPLlA#fD`D*X1o)QJR%w(ZjRRs`P(OGMWaxg_W{YgsGV{+v-3fuY?nM5!hrUy^a_Tw zD{bV0Lk>9fkON^K=Ah0j26x2pRW zMYsAJO6wyK`DH7TztUP;9)d-MXC++BZUCdU_=Xn!C6obKK{YM{AwZm$5(4mLGx^|3 z@c8uR(M=IRvCAL?Y|+9C#+B#ys1r;y6fGK&MI|9QC|sOnrH0avfgvdtcsO<%g0mh) z6+uKeAw%ZEpa2&h@EsNMX(ZvXNv;$giF6qrJi-vF;+Ojhh-9;qFa{9z&H`P>;D=Sb zd4b1NGWeEH1qm3zW2!C$BPZ~fj|z+s4HIP{Fq0wzK9`V^A!`66?qv?a_mnz(U=B$p zuZvuT5;DQLK2+haWT_$(xp?y|#EXHc|GBv?N-%;I4s~fGIjS(igGk<`3Hk<8Xb92w z5OwgDp%8D7)N%-Qnp+2`;t=FCB>WnWapfce?XakEYN*pJ?bVc!2-Y;zS&FFyx!4i~ zNe(X5VuV^F=Fp0XB&X@+K@yS)6@-|_ZIa0Zy*yPQLDoomf(9iDW)`khE**K~Pf&#u zp1$G2%ee2tLVBdBp$;C{Y36hdP09o%geAZ{F0XS!m;`ePjX-TG{|7E9jzH`9j}(wj zeVkRP!Urak%u^pqI0OeEsB?=0U-(2t>az?+Jc-MLBsjOr=Y0>i0nWWzU~#2TM;%Ba zrvV$K7)n$^7!o$&dJotj7v?bLiupiYs>q~tGH+>ZuF|6l08!!q;RQ*RHZ|q#BJKd% zWFo5j;-~`&3e|8z3^m$>hLlT4B;mbyF3IB~c!ki%77o&dpQG8z@e$=W3wbtq zc!cOITyAYjdML@VqzWE1vf%K6y%rdx#i2v;@)3prHYKQ?hIFWYgZWKhA*0*^nUrA6 z{_oQ-7C?x>y`WX#B`fMME)Ii>cA27tfv>ydQ$hJqTFZO9ejShz|B5;<1ks z3k8!2T{0EsObJ+|1sdp(Z;Uo_g#EbUY2l8~z+wpCK^d;q7d zs)wL0Fagln{A;jKm|(5X+1MI7I2t*^U$s@m_R8%~Kr-$HT~9DhxzOLVfL3AJ&wBxr z2}B)%=i#UWLIe^CeB_@cg&!9NaJ%9c9YVwn)yQd#5U1jFZ}u#7OF%<_ie2~}KtW^v ze5q@VHAe~o+xl@|jv%zWcM+iKvGN3Q06GRzUZAA_*u<4wDTfN7RA@O$$%j({Mt-32 z5{3zZm;vT?&q6T%;kH7IIJGasCR~&acV7)IY@=Uz<~{eUktUdOlw`Kt!?VF)%&KzS zK8;Jj3u=SSRKS5c3r@33u_iQYgTj>9U>$Rm4rzB8x<)y3WTDPKFQ0Q4A$}B!Us8|~ zsBBSX#_s@AR05{Rp{BUz2IWv8?y|@fAxMGk;vUcA-X3b6@(P%;(`lIaCNWR{HeqWKsiPfly-exAxap0l1E8!cwk`1AN%lW z5pKcF-cdLxXB+myu7#owl{+jX-->u?xP=w|6i8c9B|vb_L@TzP3lXX3ruOG;6pvYnH3Rdf`3vA48j>G z9?Yv0E;Q&@^UW1WbE$neI|a~IoYOTo#mha$5Lhm87_rq^LXHk_Tc4l(2ne2 zXm5n5g$AQPJ2`NKNC|DY4hH8VOaV9wjGmI=8`^LkjE++TK01g}$ub95rO@V=U|f8_ zF8k$Y3FjjPlr@k?k}@|+c!Tz&1mi=={SJJPf?+<&E{~63bn?h*(V+&#FvySqyuK=2=Cwstl%v-7R}R6T(z zD&6<(?$h_2bMJ$kBrpghz}uH;G>zsjKm6?<1ORLRCp|keJu3$tOKUqFJtsSBT4f~& z0O0SYCT2IUkE06|03gUUAOHXe^6P(+dmZZSJ2*f~2#sdliDif2>#sOL007kgY$zIb zJr-srJwtkGW_@->Y9=E(CTe|SJtk^;0|RzCdObE~HUL}c(9%G3_a=_z|*>-&cIN`zP_|t7Q zwYD}iGx|SI%AXHO@>fIBm{}Pbx#$=g8QcG#!~c2^n_my2Oa}x`{_C;)@Ol_&|JC7t znTh73y|vXZr_1{FbiF;lJMn=r^iW-N@PW6_VKGhdxitf z<`o^+c1C$UVw-id1x!zncPR3TL#g~2cCf4UrSyhC8#pc;WRDuPDFLBDOK>Ml!dTk1 zy$h~V8!Vd@TOPyE+n108G8lE7?79qM%{_pBgXP<5?A{J=sR021Jb(fKQ2f|Fmg{9)sVAe*KPD zR>m8D4=Y~V=CDrUoaa#;BhyfV1UtlN90)vl#g23UvvyNr%2wqGXcv*#)Hso}UK22L zeKhx6CENye>}*fVCcizC#;O3p4|TfaxSZ$RFrBx^2PqBclDwQML}3~tgH-0$1Sq;P zm}%R-dHk8UJPG1#v#WTA);`Qi(w-iSort!38mO1~o_To6%pcr3)nD%isL)%C(-_UP zFk#WPgm5y{sv5!O31FA;*>>>}{RjHzr2)>qGHv2{s)745B+q#t-9hA)!lN>uy_7Z7 z6|Z@68ZajhM@!E?IN@FByrA_HtuNc*6nCVB>IX->XnsaYKtkdCmUFYGOh^0SPC~rn z{>1KM@Qyt#wcR{I64bk(eyM;++wN=yoQgc=ck1{vonTz8mS8da_>CZhszs13YM(oC z@wdTPzw`_?U|0q?8)Wc5>W5{3Bv_Dj%Q^;EmeOA(eetCLz3r#Ykh& z%Ra6c^Bcp9tb%B%s zoD9NIS15t{@jYRKToP2rIH|4O2I7%qK~T{|{Nt%>SxCm#*S<#l8U}kOgx7+mJUMYruYdT{y3nMeb|8y>bznsh7z|_c6@3)Ql z%MJfM(INb+p&iUDjV#Qp{_lMDTf%vYCnm3cy<$gT006}PrLljaIP~mv>@3s40B3 z6TQGJNt2HWHm6;NX$pcudkOEP^zA8LtwKIcb8$RCJ`yegbO<=NNZ=#W=G{ce=@v zBsd7I+Z+ha50A_2;i7OiEi_YGy<^u*U!=D7?G;j&KAhkI$pl^h0N09rKNQ$aeNB*S zuFkEV261{qr^ZMn7;J>R`oj4NJk$spaBqg->s(D2YkB3O6TI1w>TA$et+eK@5xA$mRcLg|vEDqs+2qGWQQJFoC0 zf&&Dun~6icdRp47xrQkZ;*sqPl_1ITRJnO8e;L&?dL!qQHL7(6R<8^o^#tKPhM@^5 zn~gJo`VT3JjVKgtom0*igLqJ0j%+O`RTj%CNpK}EX8`_n7-=g!KmIgAc$_HUlUp5n zKu%<5S09+*XU7m^hm~B^v0W6RYwUM5YIC}Cdx7Y7G6)Oxbse1Ky1PJa>z3@9*W96u zprL6fVOC$LN5>kBhyy5x=AhPckSuCZ60n{PvgfG>mxc#veMIb<8_)c z?g&wncH!2K8m>AXNvli3FvTcHCkiPpwL(9~RDPmrg%ieN%b{`t1UKqa}aS9KU=M{~ut+ z&dAo$$lk%o@UKAgGc!`8+GsET&tdzEGWn;lwRwf?KZmW+XPRO!J{2t^7&1B^uq0@+ zaHd@Tj#{XIVo5tdxF{TUSWM%9nP;Jv!+PFHN0p~EFM&)DG8}CJ3?E_rhYWWMoloXg zV9q8><|&X_x25zP7gvw+$&6^%X*3gCU2`=Y!0%nW$D5=t;y7Vr=gB(3Ai}lBK`GI@ zddc99TqEwPtm8ezPHeD>FxiO3EM+UMag+as*Yn z<2B|;N!!!0P#GdE_SNts&Ry5bE5w<0fva$q_`P`x8Y=l%q%bs1Aeer~H3=XE@S4QB zs=IE-3GDDkU$=6dQXa(L7lW@A`^O;To+h?VWGg7J`?<;1o=flLEa*d4y;mq)Nh9vn zu_N$Mp)o_Ou4T^$t|S%D2Pv2#fLkzxs7bz@e21JR(I!o$cM0&_nC-jGv?8nsj!!T@ zW+v6*VL21GD&`i57p8MG{BV5~h1TTMX}FirMaRu?q4vjMI{Ppwr_dUV9o<*VegdNQ zAVs<8fV8f-Ien>02j%6;#(`30u#A1fY_i|R9E{W%x}R4DAv|sr^36jq`%pHAi#h#d74y9W*;b{ztCm0V3qA=mO*_m8HeAAqn_cGGTFbRAp4w8)=49t`Qdt=CVsT4+u^csvp2SKHON( zei)p#(wX~>GGEc%_cGj#_r{uT4u4z@xPyhpl?{9%2*#te4$TS{TN28lRpv_*0%*8B z&xa<%l^iSL;fDbw{a1FccRmR**RG7ON4!Im@N7g9@Wr4m6vCe56v zPsU4GY@5w5YHUcl;TMvEp-U4YXv%@4OOKiMNKXc8KdlGu^6k#;$u+w!j5I-yc7cju zDlDw%P}al=0-`o4ddV<`88w#_!Ul+bI?r-5+*q&6?E_v@7x8lDl0ZmG;jzjZaaUfa zRdAbdQ29d5L>^W%*t`O;WM`(UuwVHr80dDIo{Pq?#T>n2roGZ0%8{5s&PVud0!`de zsF+g{n4)ygGH+Vn6DpZY>79oa0uRycZuZ|hxNj| z_>FZ>GOLoKMF*s0-(xQAVlq=ZHk*Oedj+t5&2JAc2E(Ad+}Xf*0qON4B*>WUC44rh z>hYWby0x+hVsN6M53Q7>@T#`&Q7jhjRZz$Wy49gfp2Mncu|>lJ&FQ?a1S0Yc!S$Eb zH|jpfxd8H>Si15ayThAENHF)KrW=#ihPcZE48o)2t%IOmRW5U@Gp#*7JwIHJTgzs7 zyUud9-(3wqxb3*Y*fW3acZx8heOcV)9y(WPZ#gSH(7*!#$p8B(>g~M$>}An2(=+}f0{)(={={jhQ)#tYrh|8yfhCV=vd>;*G39)^XN${bG=Y&$=aj?6Xd)twSP33$atB@+XH z212X@jE8O1_U0`-j;XZAv7^xm2C)PbALIR5W z*|3A4nm-h&kLktpXc#Oky9t+nuzdNaGRAjI;|6mj$1CF;a)%prE8uFI-_)BR-lfJV zAQ*B^H{(qLe1rI0dBJxBVZUb5i$^ymr~|!f_+7yyz@fwSQ(_omKG<_elxg1yNvX51 z3w}IpWUtbQZ5)7>V3E63*150(;4BLSMZ8O#U9*%zOzIF2|bBvSQ0Y?QXh;7E8Wy^1_gBKh+=do%x0Y>|%KR`s~t(9$T ztf&w+MEK7U3iDY%=#wGr1RXKqmq} zC~e}K_v>B&^k|yC)StQ~9zxFl7)8NymvX>X&IG2V1rU6=WM0`s9K^xAylHkpl$I7h?|qvYQ@TkmdIY{ zgnv&hWHp{3m8Q(n4j0;rNEq9ds3(M4#tGuhrZvr$RG;z z%Htb68kH#Q?d4TB)>LdhHzDKx*jgguFd}0|WuHoyrPoA`@0&_|qa9S;~=hCBDhrNFzn@#^c+5AK1`k8E6{W;b9 zErj124u5IG{at^Wn%O&8+qwRW8ux#|ssBJ|fBu+njP~vM6JO=GB7jYmxRL4Womu~% z-I;G8{CsCJGO*MC;m&;X{@2ggoF*xwi#gPgF`%3kmhZiu7F0T;7dNe1?iJh$@=S#>3eAGUn_4*+ghV)6s*_ zhe&{>amxsxN3?6?3kJHs`8NUd%Kqi|S9t7c17G|hIBE+NFW9sa{%s3_oKRULg$<~I`N6RBlkpsku<>mt}5ytH(+?I>r4Idczu3GeqYbtWPEXI`<4^#YvL?q%A*cAgg3b};iBPb5XE76`f!_dzXYhy#@ z^nm@IH~mdcQ*qDD(m&V`vy6Iiu$C!P0~6KuapD}Bu)v5$x%uGCA zri2gnT8Goh;EGh`=0{MbD;ydhdqS?JrTH_1ZBSm#Y)7OF!xh3rU9p2zNH7^Ka=T%4 zQyK-3uHGuY$2AnBO5&z%6hqacc?zYN)ABl-sCsm$=&Hr@_m0Lga6;({ejTYOP!7Eq z5tbdv%ha@_p3$GB_)GT&^y8yKqP7_f@Q(<=HdFcZah;pfpJvWSy*2Q%ZoD2RwOj9= z8^#iJ(!=5u5^@nw_Uv_yYCT$Nb9ZB6r|Fh+51I z8Q60Tcdix%s=kD~<(sGnC6VvcTI6JZsiKJs{jt`D%)2E)?wo9@*p=R&7Zdks;N-M# zz~ta|F#Le;$LXC?=xOL|7qBimHjTZ!llf`wnIl#fw0jQraXbN1B8IFgjy((IrVcdI zK)70Hvx^!;62^SOUJH1yC{Fo20!|(G!iCxlvaCsL{%PfOy%NGvNOf^k@=|a!U2jk2 zdShZiLNg7EFB9Q`&)o@9Kce4VJu`T62yV+vtE|uqcGiJIRj93&A@}x(C?SCko>CM= zq(q5&iBd_Te{lE>@6WP8WSmm>zkvLk`}H&A8JOtk{s8%Za{JV&$XZX+A)gb%;uFAP zv)QKDS>@)LH`Kig=nG4!HB(-W>{e2)2wirw-`Z%@TRjMj2Jism0ofWzgj)CGWM$Lk zw4A4YN-rPH34QW9f3o0x$?{y`TA#?`ANYc>fV0!pP-dQa>yd}%7-TVo&0XfcF?7zF zz=7WSxjg(1VQT{*nBRKCpIVyC91DMqbKt-M$AYY%#t|MNh&mz!>7cjfI7bu@y&qe# zLS&s`L_QH1F2G1=L+Z9!2tYJI^Fr7JJTE*TU7nE+jC+y*yK(z8{w^9w+Hj@|1cz;! z93(@pg8F`TV2l97U_xo8WKyy{YYu0|k7mpk~q&Mpa;i}x!Mur_eoP!}ETqBvh4;2U{V!70C&t>FN;Y6ABlF zRZRFM)44-fej-E-*$FqO>fh&!)+ht$Wj3;!Aq5dDm|Tj>l#-lo%GZnSaNZun*XO)Z z$frzeR+~%)r!n%jUR6BRuRtb@p0C7$d!Y&mFFv(r_UG z9$s=4$wc*BP4dLy{HK=mUm^c%0r&T0<}b~3OCtw8Lp_IoFzRvts)F)+qy8_6(%;9X z`4Rlx?Zdjk{wY&3jeCnTn1i~S?A`6++_(qpjj!8%mt32Z* z$1a}cNJj?LNdY=%$G{9s%RG>#EjITV7!Y>Bjmb)8ct4lChy91#KwgrJpT0ZO-21Ct zrRSa;kw`FXjJ+V(JGT}s?meT}Ro>lQQrc?e4~B} z%*(NerY=Nf(YRxP)xspq44}Z*XDc7vlo#>wE= zsIChHZsx`Iy?;Pm$v+Q?w4oHW%_g1msX>$aqhM?vreUT!Ww4`0n~pb?|#yz!%z!Qmg78f+jYnJB)nj4SGgU9 zr#m_9RZleSw|$HkEqj+7Y%PxO?cz>dpKjLoqVgT?pEGm1f9o*(H$}(4Y)tsWv7`CV ztB@T3-bZ?y=)W#Yiu_k|{`YlC!T)al|E_TPmqpS)?(Bb6$z=SuKIq@2O5T003(L*<}Alt@EE3P2Nmkm=8*Fya0}!K(3i0p$kTfEPN!zBCZ{u zA!j2!o_$hnkpwb>8O>H`EXFhJ$1|IFobA?H z+;+GFW07~+&;umu3!MrNz~Npq8fM>U5nSE^kvb+!t4Z6-a*)@Q1i+Jb0Z?;7f`&& z2I4Q0n$qLF(#!$)%$1{~_s#ZJQ$Ihw0Ggif!+6V@fL@Oi<9@e2!2D(NDVgK%8RA%_D^qfD# zEa8y7U@<`<=c@4tTTF&6naOul#e}8}bn)l0z=h9W!JKt=G z?c}PeQ0@x3i>#D8ZN2;C=W%Y?ip9!XO?zy0zwWR$SW3|Gij|gPr{?`0Rjtm(PtjGC z)-s5;lhmESP}*6Vy)x8$32|#~;~bG6S{K_an2sYwo_3zk9zlOgF~2uv{=7BvZxrSa z*Tdf-``0M?qf_%Iwa4FN+grP4GdX)m+5ZsPe{hd}>SB0vYyKpp*Di)v#|An71G4g+ z4@Ec)h2$zAkYj&bK(Sb)AHEqHLKw{rDCtNyW^`Fhh-bs9(o>L^z)SA2mw5hIgzwY^ zgP=`yCAa+;xAWfp%4W6mZI?S}(XX)nbdLKEus(Q$wf`%u8DC+I_6FodIO=Aw3>B1|)J9npbFp{MmF`Yxp6K&Dt`nC?y( zFVY*TC)uiN-%u@5t`I@)H~`GjWN@uO|0955uD2nAMAOn&`{se3#(ik+3wOw5!enPy zW;x|+Pa=WrMxiY$I7F9j;anp1YQX^~2y49#?{@{pE-Z38N8ts#n6ku<4KKJ_sm7@e zYRa$8%MCC6G_0?vc6viK!!M|Y^E)0$S)l=0<&y63!ggOcZ#M?YL2#}LhQWGS5{0Zj z$d0u4>KVv1aAvTDK`qq@$%04rwN{7vs)C3LyU~)Fbfval!=~6gV=q-hZY+RiI|~{U zBIoxctxGrFvbrjsGg`9V9^sYO{`~c6h+@>eXE~; zO^U_7(cz4*N%47rq+s#vgc?LPMXSB{m!x<-!T57hoEPCloXJO?^3_v6j`W=KnZ1*T zPOHDsLYf6RzYgAj9KWIPn8}t@;PmrydhGzh&DGS0Y(grMM_6gzk_}7{&m(1uqLSt} zgY(l+A@562-=WO1??1}dzko0*4Yx*+Y+M3pJ-Ic|{GJf`Yf@}|O^V6|SZ_)3%69$@ zJ|%DPX+F?+gU|BqD}0X0?Pf_W=m+{FbZr`d)k!tYHiA&J*G|)44A8LJp7=BXZCEW+ zCBmc6w<^(4jEQmCEdsL(}wp*az?aIh4WPJOH_nxbrba1Om+IA0O zM#8nD0*#B8>sj%_p{PtmB6EZ`PhVFoyLit#D+pYTi=&+(rz3tD^2M^->S@#2^wVRJ znl;mz@jd7Lf9NK7)8HgW_Oe6}003aL008>`8MCDO{nyVvA{`q$%O8BiH<|t?<HThf zp0&J3u6OERukWj**D{UD)VKag62s1kYC9#JHYJCEd1ieU0tQ~F_<<(yoM;XwpWs}* zs|s2T#7A)Kvs?pQo7)%}sS<`&Bl7%Z>sF8!#z!VT4#2jL;Y>h1n%OC&Huh4v34p1O z49a0VoiOn>RR+5ujWH=24(?kk9 zeGX_dyL?AY-Fr~5IFBtN1H}wV+N)!iVnJ5iW{^;;9Br);ZCLAR?1^9^sV*m{gFs{w z<37SbHdW|YOusA5Fu8o5x_@qd2MY9wAHOvG%mT#?ql1KisN=luWDPB?C@in?JAV`* z23KjgH&vnpQhcmh{7`>)y0AticD30wy*cjAW>3A2%NCj~~X! zBQ@VCF*@XyvAfp}NMlKMGeqp)&4IrXPl^2DqVlK*$PxNeJWGvPdbC0V(h=t9VJS>* zf`Vu)-4Z2qK8pz($GY~6UTQQCtZWrPl;gp@>Hz(ia~6D?c-=USz>ZZ0;SR8VutR5c zDbtWH&nkY*zQ?*yGAfyRb+#*1gG0u()~}_*RH~&Rz6l7qpJmWaCdt%IJ5-L7k|jMW z;!^wa)BJl2wn3=3PSIivyO8S`0k^)EH=S!cwLQ(h=e;}RZF@Mz^S(V?e|b99d0xC9 z|FmAK&f9}nqEUvo+}^_dtNKo8MW;i*l_k^ozup; zH~Zs$==lbyGluoa4>)c13l)SOU(ch0s`KaX1q7W&UN&l9TV~i$hXD9sLvtmWaB5Qm|80B$s@U#3UVhGHV7Ln!ufSy+Q#OIx>@c)U(^;Bv25yRK0S`pJ3}o3z&j1!C85RQ& z;K-b=m<;UHh!mR-W33)QNw5|w2xs|LX%t}$DU<4Zyk0C{*v5UJinG84;O7x5_QDu@ zvZI03;RU%?uA;<(5QLr6IW)7l?gIJ;)z~d$2-=j0_8_SXup#MMq!t?%L{ogW&RxZZ zDZyrt)J?|~?B)jV{EkSYSH8v8w}=SW09`O`GEU)zj-uxD6=vf6S?BF{BFN` zm~R$2rHSV?VpAkDtU{HRVx|!D;Vy)Vx^q+prknKydAXXj`KuVz2vp$piE>qrMIVHc z4bqGp#Lb`t%ICh$-3P2Hu;PH^;MbqI^5veB+{>zV7_6AGN*n<+keyt40B@^lxuZD) zbzFlOZaZajxy&EeGIs!UJXajU=%b-=$=QXX^DV3{0KM?So)iWyUZoX5!<0K^I>wY> zj}uzFL`V|tYL$Sj8yLVo64Erk1?%%mo5wbzdnM zio+kG(d{q*A_90o+&O1Z5OM6(dF|5>7rK>dGI!ndy{dWqNfmlAH8Q?v?Qpx~5J$j+ zxlE&=p8>`3RRT7HqRn=I^Gy!-sVUKDql->*1RxZoHoH^FuTak_%!^RC!F{Zue9tdX zj+gdj9b8k8vY-=W*j@rkwWL~0e|jTW@)(ynJ&17B+>4h%GAF`@h0G?4EzXdyU1gquR7|S2z9+|M z2w9i3c~_IWy{9l2E~kMhv0QvfYS0~-cm*u!y&5^Yo6sZBFxb#k(Rk5hD5$x}ztuav zhrI$eKx#h^4ueh#4N^Mk@7{T24_-s0{h;2DeT<+Rv;G(c4KQrkyQ}@AJ`?|Ed<%ZfgS&x@XAv-$;%3uPCD6y^iTPg?W`HY8SH0DjMXE(7OE3ly1 zRJ~3F);`{azT`z4d6KZUq8JPH-4$EBxRz~QB@t`aU#lIRKCFj|PJk_0Y`EPhP#nbF zrpjQ{!sckXeyl!^_Ewm{=x^p7Oybg>c03JmkDIz!3ZP61?ZIK@BTP;ar7QH!%f3{Y zgg(JafJL1-`$~=_)>$g2WdW>Maii!xceJE6XIzj@(-fm_|6O>Vs7t6vY}~EiD>*}U zch*SdjIEMTN<63cUECA=YUvbDE38}&0Wyn4`(}?>2SYeaqI(RZ~sPwL+*_A6TuWXQC|3Ji6mh$~OmRpH}Q0lpekFb~10Qz#y;&>JiPU8)r;9H0y z#JhrNGviS;Fn$=mg#NwbO09-)Cxb;mwZpSaN_$zQG&lK_$GJ+}mNTp)3Tg`+rT1b> zj{C~vdGpHYV<4H6*t%qN-ari#?BUIpayzTTG9}kFx#>0Q-Ur^Uj&80`n_HLPx3<0; zK9j`!17kWAOLv?WaDEf9WNN-&~g8clrE}Li(u={c}kFq~`NmNWWjD|Aq1M z2a))*@$*kq;%_niewqHmn0~)=KQGhIF}`lcj;<1FnM;O52 zts`Mt(dG9?;@Z&OF0L2knG}oDodLby3!PwEU2!8}6%nS&wkVyiga?ra#Vp7xv0D5cd{Q7?1?;|tjw~*`ZT?*6O zM)Yob_Zqx|gqug%;7rQf{wu8pq-q3dqG@)a%YCcOF5_z6y)fxa=NnRiysA2#S@BFU zAWOzW^6-U%T2_dTNK1+pF+n0pkHtRp*jw4r^e&FyYXp} zi8BurHynVt7=(=%th-iPCPij(z1?tZQHI9#gl8d&L{d_N8iHish|dHtXROM3<)r78+U=yBZ)AK?WDn4 zGF0r9xZk8;8BrS``xlw2og)=uE@ceFDwB>SeEXnkD)v1$D0>AxXtbbrWEPD`%^V1( z$gsSPHebejgdz3$i=n((`RXG*Tr=0g<#9;@Le!3?h8sgp8Z-tH@%LJvoDU;v+V2(6 zhXP5l&vNvk%mjiNn2GT~nQ{ng{TvX$OaV_CA%_Hy$qXvP^omz^M(e37hiJXRyz0?# zw-g_daJ%z2At&Ry)IcViBNI{FG?4?p6Z^1{uVG3yyb6bUBoM}cH{pPjDxG4ZJtofs zEc>oLCb&Y8wF6lhBdIiLv%g`R`4+>Df0n2dsmC$rn7^tATK)Y7D1A@&JZ^{~vV6cY zwvFrML3j+eE;6zwgiBJ`1X=aLWy+O{Br``xVTRG*qmYOhl?729H4s$kM|kvwLJ(}D zMi#Cw!Fae&L32z}<|^QnxJa=4b8}KP5OAchcqo?O(s3c24YbtfVl{%N8J$iAcuZHX zvO(o`y-INZ9q0&@8Bv8HqXz&75}X0rn25j`aR}whdjM+Um6%0yzBL5RN7}R%@OeTd zEiUTA6j5;Fjr_2iKr@v%1Db_aKUFcTM9yG^*>W7zTF);gP)504=S1QvbnD>Jr-FjR zjcUD$seYACID+n0coH0Aqv}h{qmZ3`{Wpjd^)$@I2X;~|Q8eVI{DaqN^CD6Uy*A6j z7sx$xDesu>wLZTjfO7Sb*p(EVZQv$)s$1xO{bCMhTL|!&+L*5TWr z!^GKh2I-Q$14-{=qSJeG@v`0JrB1d!KjLWWyW}{>UvAGWS^2_Q97+p7I3v8EVDaJWt89o0_$%2k7V;_^oXRz%UuA>Gt8B3UMK<)6irN_A z_yMqbF68t?;rEp3H5-EE0yxBN9Fr^dIko_B$CMdVPCt(TvM)?;8isgN?5$9 zhNqg6SJl8$GNFn|v0QvaYA_s`*zLR9e%SYs&ZG*N<&MQU80r%s*5(i_BJCPW@&*eZ zOKOXV=>VPHMkg3&qjg%FA06jjq>chrNEQI--K9MP2t@;mv*fi`h#u$2(ZHzmO9I=c z+X3gTfu{}J@o0lJKfEub=VD!I8y@Uz=ke`H*T-(3?b*9u_OR@Z1KM*y9_;XuLce~J z+qx<#d|a@vqf}!GHfe8WCS-q5jr1)n4jSNr{zmB#F{RL4Gpugq?ff7HZxswgm!w~T zZw=5k-Jv99vqU7?mFmtn}Lb@h4P^zN)kFYLe&%2Sv`!wY*x7qKz;kLR(Ro_wo`WL@CpJigd2|reHU@ z!p^88XA=o9c^ehtMI=U-o_8wv5&sxt>)f zb6&05N)6)=rzL(M=qjD#wQlYR43b-@d4XzSS?F zV6XUqv!K5=d>$XW!OT$D=$!Z*II23xRMnd%Kg#co$p?Led+1Nwd45+95ZxGnAny8! zbhO5xiJuOM&am<^vv~d_iUXv zFMo^T*S@j;A9VN+HOHT+=RavDc%z=bb*y8K1C)Jw^`UX-|6>;TQ*HLoTG&~BBFMjV zU;LAT=ql#$s}DVix(5$!H$c812n(YX2T)Dr>f~hn^?f@kWd|0P*Qnk$w7Q|>5&hsR z4UEI-=7YWS5%$67<00{mk!&Lvpl`q*BpD!3UL_U|LoNzV1T{wGBnrJQKXPLpzsGC9 z;w>;BUmC)o|MYw~YoN#*P};PfFoG~;S)T@Y{q131pfLjnVtxM}tDs1dIr39+3C9UQj;D!(5xbo6z(8Y@BL9D+P=cRW2 zYb{=um4_jO15wz&2iP}IJjcc`KspK-3sazVUER+#X(%I0NhCkcm!g%Tw`QTQ(G}tU z2y5h;VE$s_@2lUNp)g!0i_%TEq+02I)m2;1MAhYzWF3efbxft)TzXjCNZW}${k&Nq zeR{QFI0EF^`$QPOBzAH5gt@=~$%H?r+W2)zXU8@ibRBy_G8n96(6}bRFy?#?-v&oN zP9vz}lu)<>JBoSgyf#G+E%3oOk}2ZO97jF{&#Km@fV!`XV~vNqY-l9aI;0IAHCEXRR77LHFm zCl8%?&%Yjj^=iG{J6&c`Ds;LPG%0-8TOC_lW{ork%$qc|=zQ_M+h=)9N6M+Hn@xER z27tLLqF433LVd3KAuWmUGIYF9HeIFBS%hWprctmhxp7_nt;Ku(AOv8SgQllRslP0F zfxSp%oAJteYdCsu*^2XFTNBouKG@l)xfMW!7SsFb-4W{55P6`ie!f>7af@9bbP?W? z_!DpYYB{-=cvRy6kCv&Db09GOyPheMu&7i?Qq5==zV9Ki0Srx_J*OR!p^kCf#bnPF z3R%*xYsm3sa6d3f!3se~(0LI`Sp=qzsd1Bi!KK_E{y424dI}0G{}Bk3*_y&@N>8OKBVaZL#SR4lsso7nM_kZtgX5KcPAz41t+QmzQ|OJJ92eQ-;pvUOKu-We z`O1!?%@UH72>?CaO-0!gymGjw8N^LWOvS4ijR4(}29}n$Gho;QbiB(n^nSyv-y!47 z%JBVZDcMuTY}}8~{XRs`d#>zDUwn|N?J2vQ6e2Kgo(Vt3p4C-0_`!}`bgO9}$#_v( zn?>#{zfxL)tU%(KD=CLwdbECxya_{Ltxj%PBOMAWM28)h~@xE97vz zAWxvwhbPHaM$qn3Y#`!TjM5@27>hs1!~{*z-By}Ct?q077itFIDelAUZTCJo zo914%f^4PM#(t=V-~mV#$L_mfGY4#n*VXLBBbdE?A^|Y zIE3hg=2Dr^&OQ#@NrB4J;iJk+?8+1f!02%!pC2zuhg;RQKHq819S`g< zVEu@FzQc({pq;`N|GNKbIRcJhb*^!XQUv;5pG45QxX1!no z5yX#yXpiGK!=)G|^lFyQ78Eq}g~M)W5@QeoCzxqMBy`a1co3cF)S3F_zzN_rg@CyC zv1+sXQxBIsw~Drgl-@E0PKCGQ#?EY8!)mK^+=%jWjtyq>)Gncb5dg@!78w~scgYZ0 z4Ul>PMTJ%srPVF@5=RnH;=#HsHFl68!?RjA*T(=AWWl4s}}jNHH`-%6-!7;_HCqTu44|3o>vTpwzib z)#C<&zUc;wmgKhJ8F7 zK8<^)rrq`-!hwSH2lHwS*84|n_LO1Y0Fo&$p(t6-Xz~TcD#}!5Q7JE5a~J)~B23_% zupqHD-7u8{7YX~rF)H(>yVWxy-zE&m4%N6*zBQj8QP^5^own)N8&OmAmZD$7UK~pA zpevgCa?r3eNSgaH8{b)G4}#U3(B}EExPDql;6L6-ODrT@Rejoy7-gG#60hC~rPi92 zSx8SvQ~7971r2)k1rV|6$BI8Y3^8atT3CU>T6tZit`nB@ARnYv;E_IN%Ey{dJyg8_|1+>U4 z7QFoXV!IKeVe(oBQo*bdhs7=S{n_`sWp2t?avEto^=w(r^Y4ktzKCogyQ^U6EI{Xn z2q>0fC}Zyw!e;Q#l5$^={NP?i#Sf82&w_WBSLNp1!vf(366Cl#S27qpTkvNFRZSi~ zluvC^%fypRCKW02P%W_@P)uFspmHa%Y`LoDJX9VnK;*Yi0)~b-gWE&^GnuPr4*Kf3 z^pVNS&eO})&h~oqack%CamVFmHTzc&{5J#ftII%s`u~tR{>dBTOl&{7;(vc*{7+uQ zT*Z{O-tL<)JD_j7R8{TlR}g(y)y3(&&vMDoqEo%P(@h2ux#;bTTr{sEbD577O5 zgDmwEdexk^B!h}r#jI5zQ^)x_#HOtinV-+`d1?XfER$<7;Spg-j57}8C(%;IFzz+_ z0N_n}Jb>VgyR46=PZtRa5npCB?Gd!Yum+i1CX^_BYJnanAPw-~8GV5;JG<$x4ZsYm zKNTf>N5BQ}#eZ?Hd&J0<2n|{>@w9FOmypC=>$J~Z6y+$$+$n6G3w*lpShjMt5Hrwe zxPgCa%2ZU1o)guquDTvuQVp=xJyCH|7)JBah<5AosHZqn2HEWUU{oGh(8kZ>;z8hR zKpv>+D}Z`zavT9jv>)2Y$y(K3{F!eo6a{!20}Y%b@X?{o7ghE` zCOaItUb=`m)RX70be2>w;dEKM8bl$-O_GT;=FJ>ul63vG-j!C0@3^>dYvgn;vq$pi z5rOV6fGjYAG2yYOG__90!y9+~tYHrVCA()0Yt#f72ApZ}ZBX=nBaLWz>t^&hxRAF` zP*Ze!70Dr8ex_xCjOO&HbKh?6T$PQ_JOrTwE-+w@Ejn>)^P0dsnyg%_W^O4`%`}VI zRV|i2OmJz_1``vK7NtVK?>nw~K79RFGQPIT<_*Qe$@XCXxXb(UjJDs8$C}Bi)7Ta5 zf8pQ7^{tQK!g1JR?WpThS^3q&C|8e@3F=0-jPt%{!@T?aQ$b71vS)kEvhUH%q340s z_1dY`>50d7tpxtq{utQfOW}(}S34*|MkI{MJsuF-UOc0w?-kf9W7JU#`=KJ=uN@!N ziYdcAd}!GqA!>NS=KaIjKMGu~Y+E41p2LX&hn1&)Oq_iTZEt-h8E>@VROY$A3EK$~ zwv}Ra4Ir0wjQ2UPndB>^#Aduu(K^i_P2mJ>AtZ(ZZ%zEY_e-NXZTA=)Y1^(1JNCr3ZQHh!iEU0au`{ua z$;6)6ww+9DI}r84k#bwukGz8LEq4YC#%f*lCI6uI%9u-bPlD)L z*vS-f&(dY6RJgDs5^F1TBB`5#q;$xyI`%6xJ!$K}u7w3&xS|R^b&n7NR_dzEpC<>kS-95&~!)P&5SBmf-*ru%;NxlFgOJ%D$v&I6+Q!R1c_EIKdVZc zL^>W|%tAaKI) zVAN%HDAI6fZm=$6M{S}*yk>GRNk#8$Dhhw$wa_#-U*76#ep1p zFe9!$O#T35LN0~5vQrMeD<9yP6lQNX^1?sU~6&9IF;30PE|| zAFqlWu)|CkZ5@7^lGedI{>t;TfFRT*)EvgN!fWFxfZm{kD4;?_Sfoj>j|2&{)(nde zv=N?&ZqI{ioZr#EH^9J}i@K10KHLIOK#gRlst_95OJ7({W#qt=3nxU)tl&9Uey8Ai zl^ELtUUy}cwgO(DJe{=xO{i*-6<&n_>XabF;TR2RZ(MVg0fqy*aXrB^O4eFUGV*8f ztJ3xde)FZwl!X=ZzEuqgQEZjv88V$?fjU`m^at}L_PaG2?Dtz1xmLkV_O?kokUm=e z=uh;0`{+-Qa1WVCjLR2bLXXvFiDR^H3^jxnuoq9+p^iSwGG)=}u#g$C<6&f}O5>?f z*x1=5w|&RKz7yfe+7u!e>6FR|`g z>Rq+Q@dI7|ifzYY3gx!c2;*H1=}o-^ikWh@>t~7TGjIU20yA4tJZ$3^Ii_L9(Y>9k&SvzD zD(JE_k&36f4cFa?RAd`|;tr&a9iG<;c`7OeWTA8aXi)3~J!kBM#VLsiH}@ z^q!BmA+Ro5NB1@+^w#~`3;AlJz~bOXgW?iYZI>>L)2V`CQsHU0EZ6G{>_}Ah(z=0& zL;eB#gN=y*HVQ{iYAawZIyHZ=Q5e9+hrif35+b=_AgvByBkaGk(N^e#jk>zb-Adn1 zccN$5Y$a;`uo0%RnLH;2Q@fyrJgW)eZ)_|GfNz$v9r@vMIX0paraV#%f2E-Rk8Z&K2PD$}#}xFxDJuMvvh>s*iJAWwDf^$C>Sy^+QqT{^{wISx*05y& zlwCqc=cDzYLHxk1;51mQ>2SuHtM@fcC{Pjm?%=;`_RCQFBk(Jkm_NuX5#P<6F~K2O zk?3mtl9TbVV00=T(W|g z8@$km-umK7>Xm&`nFja=VD%{#J|+x}ZM6OHEMK_JI(v}!7Bd3Cv+h5w&V8>_72!%g zJgXD#7H{pEQ=$%J3I0%sC5lg22=d4p^v(EK3~nf2K!xfElotrZ=kET?ql$O|@~7rO zk8>}i#AM#4-~;hm1ZUZm-wMy;u+LjZh4ao!vEtwD&PkqBX!5G?ZKIn^j8C$ns-cKC z7Ap_)3n_pav2Rn|ZPZrF5PPC3Ms#51_yWBis>uCO{&*Jqu6_u-+5KvM>P?z3ho%bO9CQ{0pnsFiuoNi^?1khA@iWDa5`7;>bIetv6mF5~K zw`_Krw;S{y-u9ibCBOt!hK_>j270jLQmcQLKF z6Uqgn=|97tkv}TAcFS}82=W88Qz@$q%q3A^+3S`5ba{fHMp40|W!?fY+O!Y*etMj& znsV$JWTb&!jS&82hzYCAV0P$KAo~vGaXv-;K`oT43)cCDWBMndJuePTv{;3jf7f_cib9y4ykhm%Rjmm)mUL zx52wtbAji3g_nW*4SNnfuVW~NtH67h>%IsG|4N?yYkA1GQ)lcAJ98q}m0F;Sdu~sA z?zMA_tj4SbK%UBDzoMA6ymp_Jb)M0l)xOYa#R^LpC2kD+_P!;sa%nFgH1JE^7$92K zv7!M7@CogQ!EFvYu@{<3<3{;GJMbdAv7-f3?hY_L1{~jK`zQcue*#D=-Obw03Q0!Al)|~Hy zU2EY38CPz$_XlQSAO+z=|6n%d1G9br%tHSSv*^Eg?qa`xV3z&FTtyI&9Q_TmFQTc7 ze=w`~2eb1Zn1v0=o%;*3O@Co_SAwYaZ55Gv-|+eezw1|S}~I>2q(zYB+8<6#UnyWuijYZ1fT5+S-u(@@FLot}?RUD3eiSPjFua_dtc9 zj0Ow#gqbAZJiLB@n}@cxIugd`NHDDy0;iD2=xx?`@`8WX7v_bPDn3X%3^%5aZj{Uc z-A-hwJmF+!8*qL(5jO}?l>%Ef60cHpENE82_knuC(VAIeSR28-OLc&XXE#faY+|YT ztrQzF#dy=?^1RBA>iAuW+q%j;0ikfhdTcP_`C>}^fCsvd#ge4L1jP%8Ie|0s%<&h< z7mDttZ~@^T%|X`8OTj}tK>Z(>HSk=@jTq#@n*9s2?Scy*nC0yRV3sDBCs3jTzSzBQMX0`f^k$`JGuyjEip#hk+`a5QEJ}|5Hf!THdW^DyOFgx;rS(iVU z)jn6ekl=b99NXewcbx-ZmV@$i#|9Lowk1;J1GC@$VD?obOI-zkS!aC>o<7?8CX#*- zb09-EFVL5FMb7W&S5Lm5i{m~J?_22dfIcXsmnJ1HG zs%z0kq{1M*5nNlvonwci)6bNdj8d(6qUb$3XIigqhun1spTL#Vm><6al&aUc2r|e~$ z8Wf2vpNpqy(i)wbvWAwV2do(jO_5)d&f2_EC(LEIFf%y4W~NGFw^s5QNJhxlCe|&b zYRjRSw%tY&Z>aqW!e_fa!sC2hvMU1&c;hipUg3mhw8?+efkAU>pWsT07Hyo>HVhLP z2!5jUHG7JQ3M9}Y$eyUw;PF9k%BERS(qhh$+`wXv-wgdI;d6xGAYpaUt-Z+88++kO z1~JbelPa8(agAaTRwF$*L8X=;SFy%A&MuD5kWr!&HnpU*DGSetWaCh0hlhJr<4}M7 zgIPPLY{#ux^FMW9FQ=wP)<7apo;(b~aD0UwyH=2h@S#zzt}JT?y?4y(FUPQ#YIB z081B+QX5G0`h9tSd4(+K>`X;ORUqr`$gc*nPnm$O)M6+LWwgoI05Iz?4#2Ek`J3aw zAIyHf>P!A<^?}*(56p)Ag;}rj?mw7?6`N)HJ7!J(j@kMp1tZ;2Yq#n2Fx%x_oqu8$ z4+U?HohxIcEIenm#^7^mfKV+)JRe4H`J6x&x?&FfUzi;-4EsA~!}drUuDdg@4Jn>K zFq`rRv&!Dc0L(f9FdNBWWA}fQf&Rhl|11M_|HwdnTznj7I=y_?1-JwRxW0RH9Qyv( z#m)bA82$g}WHY^mDPSDEk*T5c|8TvDAqFSm(G@m$7uq=xd#XX*xo|ncoo5>Q-K6+d!M%-9~q<)%9{Gv?s zi*7-&d~~1%4Z68p2K{xzX=03Zt?=~f{Z-)k=+&!ma*S*8xovXqng8ON*FM>&o3^0< zf=ztEUm$&R7R{?=YLXQW?+j1(6c0D^RQD_^3s2`v_b4T@1`Gm;K?u_hj;g0O-zx-9 z1TJD|!v}LfB5oK8_}UNzUfDZB#sFU`Kq@bd@qu126$FdGw5Jv@Tp;K-5)t99F|Xh@ z^t2Zd?)Rw69L_|XboOslD^eeC3Go+ zkSWT-bsz;y_AtYs5pT`a`u51F36VMMUY}tmKAU&f8Wz69;TXfbk|_m4vdMG@`G-T3 zsdH{31&JvgB2rBeeJg+gRYv2*r$CGY5vFri9w84P+uXC!q!e*Li<>uI!=UX#moTg_ zLY6ov4Q1$~9JhQd^3#BMlp<0K7I#D^7KIAh;N(E=4T7SzQ`M&yAxzAS-k$T41lmy~ z4G1Ob@UkQ3P^w5v;=dW_1B(oIgaY@F6k~)Yhl=hq0m}!&fbTZquSfjZ!XO2^36(?L zmV($T+C>dUY`d|0Y+KeNUBVf0!&2Zt`k4FzqX$~x4

    B#=r_ZO+lnoU7T{Pyz3PT z36fI!%cQ9rT9fqA9JadOFD%tB@$?5LFmYjfxu`qtC96VsSJyC!2{OMdSF2Lu5V^X_ zCdGF_2l28e(Oe@tf#Hh% z;Pk)J`bHpiyrISTN*Y8NB>{mOY9g0LCGU3y#>%fH3EGVuhFGw+9#V41m950RY>+L- zk31 z9>pC><{<^8nDom%ms3H%>>&RFo(5e?Wslhx4)zF&H0DMsvY4M0l3a$8P*?cOD+QvE z%+m$rwQ5YetVGUi%Vtj^6sN9msH#m((pl@opRD5TSozj7@CwHfews;$UOTMLNzn;Q zSo4(2*)&|XNKbE46_%mY5XTcEf&{i9qhb;(ItmiD_VtZ?!A_APO-(H4wIG=x!!bQ5 zbT71$0`Ve^53@+2Ed4}98!et1iy;dFHP@$lD}+X-3JtQw3rrkv_JdOq#!nXc!GoQc z?9c{75TTr|pHx}ZU)IH)p10_h_kLqrvbFFzfBO6~6L2HCs$XFIcrJH6HPVWPt(FHZ zH__+|BS;@6zLa#XO@om}Lx0-KqZ?)d>zNb9kd%t5r;Jte4j6fGp8ryDasuXd-d9=A z)xdsUS+EIJ+@>EpG{dT9-p4g~hkz-HGBEg7+@DH$xh(4ZE1`#h{O~8uS0KO8L^W{K;@3`B*pxul9`mI%aUWHdh>9{96LtJsa208#uH*?dWO2X$AeFI6OJe2tmgzlMLDaYX zRxv}t8^iXPO$RX)6-DEmWWFsN3!yyiH?gE01VYZLW~c&|5ARjF7bXJ#<@Pf*yU4JU zfc*4soVJ!Ox6nqugH}6)U5k?!DdtihMB*+ru`1LtD#3ckT(E7bB4Nl5WtbSZ1Q;3;$dbyTgMlz`)YUd!j*Ub@$xIz% z@o_L|hC)xd+WX=PFg+syu8qmlC)VWOh{1!o`8Q(nBwx`q)eTyyQZpW=^!ini;UxAX zoOm1+N@9#)N@PlTL^TVj!uv;#T@)*eLevGHO>#ah+osYwKrQ92phL$ex)8-$V}9y&1FP%My(l zXV>-RteXusoS%qwVS8P0R1TeBZN%LrRx7S>!*coWEB!a@qX|1lY?irn4@waHYM-uv zJ7P$vtU9$yMP}JUmDg^gqgn2jnjEyMqdS&b%vlzAQkS2rG$}wLdEKiF!;3Ug#m1BT;HmY1pjR1pe-2##?e2gn{WDgI zP3i9Ydas!rEDY2pXFw(J=FYT^u3=tFTj(R~#sju<#?b{q+fuGi?EK;^xRD6#tdQD@ zZY`}eOr3z1c|XemT}#}Pqji0~rB;-s5;af#4?$z!sf$~Dx`h)3N85()PCFi2*rmNG ze8b<)E?~)z;ijjmc;{?#;maJi$ESeC#s)dNs4aulCT;N67DJz(6c=O1mzR8gE}(1N z=r`LAhj@3dIDHpr-=BzyXz8%~TxkEgS=IPDCh@sXJ>{&^$zp9>!~GK%j=Ytfn~Te! z9o`M+OX1Vh0&tqZyUQZUg?E2+VCz_lH44G84=g;qexk~lZnB|ev+t99J8&aT3nW@4 z!69Q>YhDN+!-Frq=N;3Tk~J>_(ZLb){?G6NpH9npPM?C_ifIFeO!c0YeEs&B%VR_$MtAGj;jwZkF_GWBzE$F-h8~ClV4}$dIQH( z&Qavff`=??%2$=UB)p&2o8#AukpP`Wo|+7Nww?}$qs(-AuNF#UXXDl+>Tdf)(`HZ# z`3u?&y{Xh&tNp}LB0seD{nkS=nQuBaUiOQ}{+3Z8i=KBR&s@pKXw`vEa$!vSR@9A` z^{nvDxI|W~{X6?9h3!uG15X-R8C%igDcWu}Kh4N)1(sGk@1)p;oXy5XH23mNl{b%d zMi{=T&0IIZc74yc-mu5TLB%!3eT5oZ9kGDpj zOL+ps_SK>RQuDJlNFkCy4jY8UVgbJu3Y;%mS5>8h`<6%)2`%HNXS(jQ_8t0-)%?Mu ztg~?`;PEF+R2u}iIquzYj*oIN+x!RR`?O~#5NMWa+NO@y(ips`r%lAnW1C#obdsOy zTWpH>-P>C_#QCLb8O^W1NGcV0(z&y6?RpCnm)d&B1!k6XVI> zvsixtlXbY=C?M44gN_A@f4FwY!QmUs^>#vXwj*=z2-6Phc9W56^Cj?Eg!Nih@%2R1 zwdIunyC~bk9ghZ%hkxJ(vkjHVV|DlV$Il~_N&*Ex42iGjjtgfIF`Biae%TSMXy9np zpACp07sOrNpAP$wi!Xj}s5j}`LA@Wf?)}W;@II~m@}%Upvrv0CW?sZ!bKc%L8pqN8 zcz^RNjE_2DzzTL>V(yJzj+{NEdF_@U^DWTK)#&2Q4X5qgWU|V=u6AzP?(L@1iRiSb z{3M>kZme#ly?hKI80`*$*~tpCbH$QFzf!&8zSzVV_0^NkzR zEKSGY_HZWOS;Ul#;lm)QC%u^7FJ9gZ$5FL2bW~xIG)+CY3uUxP8}H4(xkU$Lah-n9 zI+nU0RwWcw^1WWqrw#Ja?MJPq9pQb0Qjlj%f4x=I&TH?@ytu!ZU>B7I{M5E5hQ#By zolZHedak@2pgBx9D9y^CV;!jca*Lw-0xR(5&obiJ;du%2T8}R>`WfR!iTFACQ%@ zpuN{iL8p7`*%k4gVhfqHJ>Jv@X&Aw#i4xhQcb|n;CcwEeKNe2WTmokX_}5v;eq{yX9qM0hO$z-q7Ig`!XIML4l8*T*6MLinT&`vh*|s+#AFh){_kWa<+C|^YkXD z(X;Gs9ZO!@6OQhbhCS_ziOYTt8^YR~N4gp!8_g`;y6gBFftQ)4N0fk%Rff6c5XAMK zS~r@%dyc1gw*J{;&4P3s^r`Yz+>zHxg!Wh4`)9k+ufy7qY1Mk#qByI}{5*9AmSEQv zCMGx0f-B`!*G~*)cCeJcI<^~$t`Jt#)>yW>@99HclZ{rc`xDRO{52NMHjXD+tK?P~ zW-O*dOTS*%+bb(L|CSBCjCf1sif~-J?dq__u4e$-ZxAPO?HKDko$@^?C|#9^z&sMu zj*(kWrHhZM$a;}?_R)?ldRq<#{RGPwU*00VK|RG$?5o;lF8(TZQ8n5=#?Gtz-CFrs zBKmMg%rNXEfki6}fk5D{IAdk4#^c@p_qp45d#a}7s)MA{_2L+U*>zF=IX(6mjMwXk zsZwNs`h^QTBi+@zS9e#5>RpInpy4{WSfqeY_}y=PB9*b{DZDO_XWs>N#S-bwv^!$Y;0`YX6=)| zm6d(lecv5}CZ6RP*bL*w(rQ0peb%sCwpg}~oWU(RDncq|TC0BaCC|fDS#LF);i@5U zM`2>>!$FZUpn}sAv#@fGo|xw1mN+=X&B}B7my4bES;b~m%-3VzKTA0Oc|wL}hD~S~ zkcCO{{a?t!nEpKr`*VipuPlrSu;7`Cj+vA5zdIpwrUk8vqHzz23RVEqL^Fh`Q$-LW z2PPzl9!HpQTEk5WiY3R)#A1kKfu>WfriF%B()6PO$0&|kt27`02I`#hTnq`#p?mH4 z-i#dDa6$cj&BI0{(C5eoJKlU&6&1~X_lTUAnhvqQpp~=HddNx@w48Qc_CPGVuF_IJP5b_ERbuaq$!RI~-bK(X&{IE#e%moSEB#upF z*#-S}jmSs@BK8$G(+~@l0*xqAW*Bi`;K3^qibRng5QE|LA%?o|a&r<1gU3Z89(}}M zV6^*4a5C|v-9K1-IGOeRywOAIq7AQO7bkn!L>&ZRN1+nG1u0FwL36l+ zn|8&GaTq9`XTwT#%MO#sgmAnP2}J{9u#q5t7$_`tuPzd(h#ynkDY+sbGF%Y()L&`k z3B9O-mXAmiq7Y!aM=4VvWeS_fKCH2ugzySV#Eqv>N&vcD;%&t7>3YA6F@usuKw zCK2=v5vs`4jx*#pqUKa?hh(H6B^O?Q$s;1hK#l_#k|6HzYEWzt{W$G)4k$UHfvTSf z<&;R@Hz`IT2TpQwLaZhSD0GM7V24L#2CEZ3k&;Sd)Tii&XChP!ZN@L8!*Or|UkAV25G4aKA|b{` z;-QX`or1lf5Mw9~XCPu=2 z4pX%0heJa2!&nrd98?4mr3`>faIgl>q)o2{!zB$_1;6S5`#KM+3{xjduetDG1>m1!sbJm7Rf*8Y1*_t>R_N|-1umdXoR7O&g>0C=xLKY7F zL{A)E?4B$rpUaJ|fX|)YtIv#RxMN5?XQF^!EZK0H>{ zz+NoZCiPxm=`|V(XRo&{CN`1$C7@@ZUe172=24PR9v0|3tiKJVJaauO8;A?c4`8qX zGhLoO$lcF&l-fbwefWfy6jxN+@36sS^TO8NVdz*b{^>ESfiI}H#nAeCgE3H-KQT3* zb{b&Cr0vt+`^~CJRQBZyOlIrRs3_om3$BZpwI&`4lDEv6xKf4$4aEEu=aKV03|jN^ z4}if)DVtp>5UQAo`Zeg8(OY-cJ-;7nffKFGQ=)r#Xz|S@eB4zpg$DZ+%-Ka)U6dG1 z*dSIT03;BB8Xn3dB~?&ee#a;W5hX4O(pI#^B=2;Udoqzh$}i@@i2n@PvYsdaJPrz_ z=9?6de|l;ue{pUQwGd(ExY$D#FzP31;8-aTeF-AsfC*RNo33SISxVu8p;PL~U}NqH zr+2z&R*QB8tyB0xFZR|zx;IPpB6OFoK2`0KsI?MAGL%_J62VBrgnZJG-2B!WmWkbk z&fe7gqwhRI@>j_hbS}eq6pTo?&|~8DnP7;J{Rb|CYZ_ex51X?6{;>Q$WYw$X3c7bB z+@IpN;{zlxvLM{Q^IMRnR6*{a1yl%N02iZ71A;IEDE2&)Pn$@ZT8;6Z=_U{`5?!A_ z>MMXqc5lP{+&O2^b@{K_i(+i}AEYQ;mo-R>JVQB){iMCQ52NKV{P@JOLNMhZCP-!& zx-3+q;aCgYaM?8EEmC`Pp1c_qY90n2KgS_dfU|lWpF=C1ESU zAmqCY^;96rknr&JFB3o|AcbY=YUIAEqM_7 zgIJi51O{(Fb|_CwuZ=J9lLfPTx+_FdXRL{%WU9*{0SRaq_`Y9Ylx(N}fDt?Kt!7YY z$B-p(CF-cMO(TR84FUFWnsn$^@vg-=`T6iooIKHdEO&56S)$IOXgFhaEO$8)riA|s zBhIN%CUmtQga3x<8c2K)JSHz(1_ipSE>D9zA{0F%Y3ntQTA0aotmdC6tk|@yuKDvB zuFN{BxS`UizzB|((-Eu7!yJtzH_yeP{L@AHrxcjwkuL@zP)+(!dU2bHjkX*ia8f^f z&zO6OnZrr~J z+>XqLU~Pd86yMM&%l~36=S&f~JW=U<+~a>0U- zpKa8tih&lr#BUqbT?~Tmrs(e^RNa7Yya;9IcO8u;n-F@ercH8i&RLn0e1Wcqr)I8r zpY8qnR%-8|*492Hyky>6@wCLT-jwE$0)Dz&e>>2p?LU5zDst1XA(|XzNv8CjE+q;#*$@lT;G=_xY(}MS#MCI^-eQUph4L4oWa_5nY%AV{v9{Fqg zeY?+t(2?kC$!JNtMQs%>Xd5rDmu#1W!ln7jfFH~YI%v%St*Uc+PSMQh%S%PVFh(hE z1m80_p62)5j;AB=GwGam+BZ5>8VG3o%uuL5NxSwc5$gMxKI=J7 zK+4_zw8=-bCHsCe&TjS`7{Gunpy0ULY_=Mo5q_xz>4HdkfE_a-pX> zpm~-Z&|VO{JXrWdIP#LYq;|~r|w!cyn2l%_vk!?8Do|^!}=tgH!Z== zUMsmgyD8VIw(P0Icx4Xa-SCX}Mxi1sR9%di??$FG0=k$U zX30_E5;|A3TeDCVjTp2$m_<%pkH-xzGAA2g6tti2P+N;gJ?vWFJudhCF5~;dbFkqT z>{{r*xMK6Y{>Tv4+O1WKsRZ**Y3f(3o{@3tn;{}XId^~McMXys`P84JmBe*yotZH| zzK*NRHZ(4ouAY`hrsMfsidgHOGUK)9HW5%AVJ#Dm!IQt6k=59Q=5bP8y{?gQU>$L+ zM`@4IuRHTTT=!&kHWG+V_Pdkd39-Nyt5ASq&fSF8N+&n7hJS_fnB88s!gE_F#XhNY zJE8+Snu_n!>J2rwi1xTU6F&0Aeco@YsH?o!L>$%YfGiEPu}7JsiJLCy1)Bix(FU}8 zYfLv@d&(L*pL)>Ijl;rYD*txxrS7hsUnem=rjplvCw8T`=x;2r@p|aDe91#h3r|mC zhZ92P7lj*aT#0fwH3(}cwFh4>tT9F*R=nH2Cc6e|V-j!$B0bP|Gp; zPFKR<6X&?oTQu>i=^NX9B9KT#c~Ymsr8#w~_HFM~*v|W9{F*{zh7q0VeGu4t_A2y* za1h19q={v-hlSk7?fmL8t2R= zFVpk*<1q?Y_B@^XQGxNaI6gT1o33E%z9##e}A6D8khP#ICWOYY-8UhWSG_~cZa1{wbSh4ozx!Rs%hxS zhT|yQF8WTa^&)U&E;60dxB2Oh%}>9G)*UD4)>g)F-(>r%`(SnAyJo1xe)khDV44_y zQa9BWO^>_KBZr$ULu4yW6;PO1*`N73@7^+~q^Ek8x|cM0B8P2`S~1<$U@Yozj)M-{ zaoWla#-GB>Wjj>dZJo?aZEG_>@?1n9X`|V!_`7>K&RFbuF1o;86}R5~Zro(&Vw7Y2 zg6^=L(4uaooOZl&jGgn5pgI&b^IbZ17@^@(V7=p|;F{Fv0xjh+TyF_Xhl#!!~; zFOSEHn@Bbqw>2~-pNG9MKdNQmi;VI+J_iIw%~LZKq^pDJ?9ohGA35%htK;_Hkn_r` z=X(~BjE;EwxIp(y1UgT)`VWoUFzaHoE)Pvu-d_?5JG67)i)LdF5RN@UX^G3$S4XbFZJwNVE%fv;|cblE2Y|Tv{cbzfs z@Xj6ubrB^ugkP=M^fv#6ryc`qNk&$eTE*q~7LcskKgVs6=ES4K z^((u2jtphQI-f5B^C2`2*dnAuTuXmnor-c^{JNESvDWWc{xkxHwh!l4rW#Q}O|LSz zjqTrGmHoUYPP1wLS_b*@n!jPv%%*= zt;iA~6E>ET9}Qm>wxcics$s|A6#CH?b+ESczEKRGeVN~Du5|enlVRK9G$Ms?vgf)m zc*u+YJi=`aRcEwcHE!)|2Cm;IeYW7a*)ttXQI*BRDIn0&pl9i})X&7fBJvti<0gvT zs^gXQZcrk%9BXwQuZe zE5(h-a)d5h1IYSfPkFh9;|0hAjxBoIvzl9}yxzI{N~(}IL}KCN_l*aSx( zbP77_`&CbcA=-219tSyk0L0o0zda5y!lGWu&%3b=9*!}Wp|k$q7M*p2Qf$|iY#miP za#m!GDGX*uquJ#+ol(BTL^7A1LHkBr&f{HUlv%(Ll6@4y1$~FUO zKJ(bX;$FryI|;l4YlsA=H?E|5h{cy@eiQk9n#DtW@?S0n-iwIbG7N0*_5V4B=HpO^ zHa2xM0T2+7BH*d_KN^Sr-xSV1Lbi|I*}p!R=>J|g1Ju#}1Z@AsD0G!OD3{-=W_~~v zgl3{WgoGA;5O6ujnSx`H`G|15|F$E$=0s^v6Y!A@08gp^jgS4;-V=QXKo!u^+0x!l z|L;A)f1Gso22|qyZLb>N7!!;iBOFjydnMF^e?DfnFhy1-+JOPAnIziqI}%`o#n09~ zN0)YNO&6+dB96#1KMkhSlT%B2LTS4)5b#y(bA zOhI!&eRFWX<~8)by88aFWc4wbI|L;gX65iIiW<{UpcLpOGaI;M#m+28ols&sC}>zi z-z2f_m~TRu%-j&cL~#T`S*XczlwsSL58~ncP@{;dkzmsw z$)^iAj$dA;{XpTG0ET)O?p|J)4a2@n9X5hTayT?j{}4p^B_R1k$eAWm$`OHlb`9X^d ze>(l)fE2)_XD8^+d08hI0w@6H?I{z<^0X?U`k<7RT%t%3gSD!8KTtx95)*}k1eh{& zUh(BD24MxneFUGy4t2<(ilLZ=o4=r%LM$fwgCxP~VduP|FC#|@HE9c{3;HsV?Y{(k z@wBTDe9Z%7V64>3C5V7IdNDKuF#}x|Ae+4x$ zBJKhmxxlFV8%8X*Gu z;CJej)PDW%32PUAP>RsB=YIKcTN0@3rS2ekI^jh@oFqVB>A`Dq!1|rHF`X2PfUpG( z4CgyUz7kG;5dmjz1NHRF#UslJd&hyHDGM%qijM-kX2q)p@yLHXjl$m%jRy!mIW=31 zSAxk+*~j0FY8RELoHCSQQkTQyQ9}9%FLG*hwVVl2RII*ep@~TC^s46B77EDr*bRsa z1`{atLlBI|i2j`E7 zxtsK(E7mU@6l94knk{iQNQb)LRCTW#GmdJ>f`?o6pTHvM~DiDS!&Z7oR}o-uL59i0kdlG*xi*5XYOAGz}i0r zK!-laTAs!23v(Bb>5`^QyN?1OUM_au?iOh%6G(WXaGG#E&C0gy>kCgLRl1uKMTwN& zaeIMlGYTnvposs{VKNE?v=Z^D?rJ*|Fg>t2HL%Xq@1$L;+I6=jcxUXyezH$7uH2uF zdEG6FL+c>-ZNHt-B7rDU7;A-5vSZtb?|l>iOYI8oSo{B80EDBMzlHYi2b;mw<-J{3 zfy_YQFAcN3Ln}FAF%d@1Q^{6u$&1EZRJpMXWii?0&3yrh6h9aeR52TzcA5_g2^MbI zfQ(3r@C_{l6t;Sit#;R?E34BK&%ms+Axg<@<|ed)X7mfV=bQUhFml%Na>N=-O(}K= zT+1Oz{jNy}MoAHes$?wb%GlJIO!~X83Eu@)Xoo^PlTDej?2yNt805l%6>;g^o8xT}RMPlr@ao!W|O&>8Y%-jwCrXqmDy<3M8qCzv30$NH&wXQT5KAo3~ zJ(R0$Mm9d4W3u6B9s2vyw~UFhG&$r4v3?+4s_ruB>NFtt^qIH;2Lwh*19HzVz`0kNp^?o)mSB}IqtrGW;7%w6 z>1Aos;V8vd7CsWI)>9H}!O9|gsQW*a-3?Kwd-s%|s>BUIz^{5isvx1^`@lQEeC!X= z6-i~g5O=<&>MZ-5EwTy`igm>XG+d}A#RLAzBB}gbls@mMTe{XRvnRJ%ME2AS{i6cd zp)n)-_X=PR@}CMImP03MTW(j0-HIC~GhzCpJ;o~!yelt{r$!3W=Ah|i^Se4%mz#EY z?z%-Yy2!ISB5oU?R(3}R+u?E#U%AmWCxJg-*5B#9PI22X7*ei(Q`Y;dVp2Acz=Zcu z0}?0Sv=S}ik$A&OL`m~#tWjLo3ldkqiJ6@pX?{I)*KFVMar6t!)ZBP0Y}1M{3X{V7 zY9EMZgSiuJW-F(C_274q*6qcB%)Z5dMBIE%GIlbSUz}X1pS}Y&a*H;3tJg40a*Vz| z%VnDvnV1)u!H?->x7%guZ~T}Wb2(@jIKZS|c$Gb18NV8q*S5**SbF$o>rWoBLSlX3 zz32Prfak?$KDPNeppvD_T-vFk|5%G5vNy}a6~Wo7IDmpKnL`@m(t+MQ4DoyXYJ3XCo~PZOx(M+@>Ho%jR{UbnDvNa_og!lM)1qpJgNq zYtXWD*keX%xR&fnqdy&uk~Z9JrZ{!`dsYo~v&UD{zh_L{Uv}7epn>m&@V)y{oA5A@ zxi?eta`Z3r-@vsgJ4-0l2)GH#mTp{HRN@yUS{w`K@Ai z2d8_r(=w%izPS79Zv6a`1yR-#XJ1n})UL7gth&dQRFFS5aew~nbs2ivLba^%F3{0`1QJY+sDIV<{j`PB})m)+1aQQBGENtqt*79nsY-7XUiHcwXC z3B2wv_Q$=k*x+B0^9k%ECn{DCFe@apm{GVt*P@O$kq$X(r6_~Secu_WL*zNj+|t)p zcgA|~cIn&z*ETtI?#5BKpT1hW4O>;=xXALVeiYto&Zy3CXYxqS8=UaaMNfA*Od*-> z&C8O1?7%`teviogL0h=o`+Ghde_kZ_vMa;#Gs^e%^&cr@qtj0&U(SVHLw@xtZ_M)@ z&}O*v&_tW}Hcy#;miuko_`|6@tJ&xI=UTJ&SD$;X_Kev=IIWkb%^IpIAFX?z>xvPV zmLqjbieI~#@3T7wcMrRPw{HQx_@CG8HEzF`p^V{cf1x8gkbSwRA-f$wT0ntx#tF|J zbNt+VFMG~{N035cOyA=1?N{=9%h0e%oL-wsHbdzX{R#Rub<6ztw#od+PnqDxwrzX3 z*r(4s+(!}}IIedt8-r%#chel5SdVL&Vvt@=tSS`Xkm?%c(M+1@b^V>C`ln#oOfB~; zZLOrbnSp@)V@%TZpBo3z^fPtP0sC?$u+Zvxw z#~!g?eaAJE-iV8N?@Npqm_gXExU7bSlJ|~(7F8@<#8mDcPf(|Ik9^_c+!bY55tM6h ze4{dEMJ~`z_ARr;nlX>_5$)!P&wS%1Lw|LHSVre*rTt!KF}S*BA}X&Y+}EyC%N7$| zl)(4>PTI$SMN$cDG*`8}VfesqJ_ z^jbr0cUHe%bMMFISWO=*DuU8_RXN|yqyH;0irZA{^Rar?eq(CNMiU*1!@#*WUY@>VwbKA~yK%8`h7V9)W|Z z3IkiHl1*7Gt6%5K%VHKum9lS}Z1^gc!kz;HzVozfs~Xw36zfIEE>sd#iW-aXXI-UU zO^yYjOK(yz!M)xHcLT-WytMl6?%lexxi3i5X=624=blolvUx(f zw6@l61?kU>NDopJEhf(e{|;M2Lb7i&78~?^2<9wfHeZe}+}!?7TbuIZvB2|j#>>w) z)v3!=*_`J~eHBgPOgXMurkc+H_onmV)frvcU@#_i2ZMiRAbUP?NzN-J$}9{y=}fsgAo&F1!?ees0q7Y+>^hO8z~k^dgrdzvjR zAc!w=lLs1%r`ftEZS$1M^%>I2%nf#Npeoyj)boT{yd!-`{+)+qvLky9ZPgI4M!te% z@h~DBpMa@?O_a5fZK{xp#{*z+hcaMkEt+%GX zKc3ea#3Ov^2Nl2Ck3-qyFVk_Zlad_Mz8&feq(bu)7;IlYDH>_1cX6#!cqzJq*<4w7 z5+Gha(zI$@ya}&Ar+IR~sS$R!cO2O__=S9JK6je>g$~|3%ldQ6$+SYwR~56QAabW-(Kg!jAB|V~42ycplV9 ztEYAS_O!RU&UGeA(xszKdb7lYdy}$5DP)z>ew&mL?+sqf2 zNm;87_B++_hU9FC2iGUjg!foo zzxalBNb$UgLniA~Xhp2$QMiLbN$uZLrU$R`-vq$FZNRtepq0qpWV-i1$GiXHVgJ_& z%3`gWN|Du8T&gYhZ#dTLvFaY5}z*Gtt!;l zh#WI3bKaHt@$p%Q@{+{FWwp7Q-;%~=VDTw*UsG76{$#v`$cliB_hJ%|@glPY_&m}B zyHj-ykkUt!^yKGIfAYW}p!SW3uHB-i97NIg5Q2hw{Dc@&O_)X((5DZD4ZJ`4~>TEFB`B`Rt@(xP>kUA zG}}f|r5c$1`cXt`^tEoAK84i#Ti@?oJ|W7hAI6j^^zsZdKIpVAVETk;oI)@T!}=xt zK1$%((WoI}z&RM4EW;leuhA#!1j6RH;WNW^XzDE@gdU9nfWIh*q8swZU-bQx@g{!w zi$WntP+hiG^g+-9tFfBo5XXKk$*F9RfJJN0@vOv9@7=dIQd%N}#n_1v%rXLmEyFSoHL^YVes1JO? zKTyh4kcRU%1iIzH!cZxy93Nuy6*XZCbpv7`c4L7diPRO1VK9Gi14l+<%<@+Y5eb@CU8I&YVE*H|mEckT^)Qw%!)M zyyLkKgR$wuU|da{5)bLg*6&1$JJkdjj8?h;gAoGbxjw=p>|7H{+}=u80s&w!4t^Mn zBOeB%6Tn~`FEYyapJhY39*bLa`Y3s2zX&M!cA$q{>utY(yXPX}Rsw_N6I4jM#D>A1 z;r5NC8%Pw)zyRV-23nTO_-1aX4|9yh4+=G2zTymp`C%~1hItMG48}-dw5#{P-fQ!m zl1`yKxNILR)!46Ip>Fd5)>XSEU@od~)UU3qfkXvtA0V%KB%wu`($&FIcobStAm%=1 znU<6@CX|*%y&{aT=qjgdhII~~Q=iQs_ebwQ;_+_`Gr4$&yrZj)Q+i7;8x}62wBX%B z8FOS#ja7Y1gSx=Q#4w1)luDW5FtIRxD7b40IlU160bt)#6jZi^xn?P%p8Rk7U$J9p zmnl9#Ug8JHiGQ^<$@p27o-ppODH4a->iJH(()9tbTukyc2pMiRoop zA3Qb9F!B6ikkqsNmdo~qn>Fun;>fh~c#*cgo6fZL9*lR!GR1ICJ;u_C`SEk55ObvX zaAbnPOECo$BmFS#Wek(+lTCf8S&y_2gHhp}reO?VFjC^cd>D-VNcaPX9${@wDZ!!L zsbQd)9&Zwi1?sx3rzQ?qdjq84A0%&-yT!EFXA9`2v2Q`u2WVdWNQH=s6*yhK6XFu8 zyjXLDN172dka!EicSR)vdGsluTL8x_inh}3;)n@A^1?ny-pJ9p0xwme!UxGi9rzzo zUX*{Qyew%T=l@7~SD+Vo;gzJ9C^8$MO1cCbYwbfD=mgqC1;eeB$7BZiPSs}Ee>}Xe zMO;5ar!Q;!41pH+%u2zGr1Lyj2WI}-xf9ocIUabK&S~A8-FipK> z4(EuJgzbhittUl;`^#WF?YMt0osiLC;5tu zcP1&TifQ^y@iAgAUaQ9D`nzCohFvd(9r&>we+|h9m!B5HD0{9ib*Yic>>7^UUA8qm zA;z0E3xxR3S)^}!->X48B+~=9O4Y*^rjEQw)H0}Gd}{JT7(%dtHW~3|dcmToL!|F0 zT$NS`tq0!S%SYbL8ThR%v~tS0Q8E3PW5OUDCO3%CstI%hrIBMeFH4fh5mHBfIxE$F zmdJYE4kL?Rn>{N){VfgC^z-{xA(u%m7_s}jPn?!;0L<4+2oi8IYg^OVK88B_q!r?|zt%u=mdj$x1gT=VJx z6vm^Mf1$jB|GDP1Sd|5!ys)>1F(6q{*5nD>OoMK{Vw+D9ld1UopzZbK-gv0&-yEIO z?hM7XM&`AiJIC&Z==fr+?A%Mur30h*zle6vO!F%}O?52Yl+$sQZjduY+f8nMi?lz^%cI;JREq{2yl>;5#O{9>oPx%&F} zjkh7mEuOYcINH?*t@GI$8N$gj61Tr4xMs)kuF6W>7CmO%K7`t~<6_?*It#UMK z?TdU{&KK2#MDf!<*UPckt(32qtIt`NU}i_P^Ozlkovw@>2s=l3O}Xff&)a!+JWo{r zC<~_=&sMQ#+fbZSygUC3@zao7j zR#`Vc3!;W6n=c!~3(r+GKUh*{xqgAoRzSsYKkJ{_9w4@&^L#5)$*$dX%$-Oix3KP@ zEJH&Klf=3bRa09Ez9-ZttfJym+VH~o{%{r3!_)_oH5z%(QR^dFV!N^ZIgR~2YYNBO z>F^%kTTCwyquI?et%HDa+P#>1a^}cgJz8wf0?tWKsRv1dmgD5w*YZq%GTh33?o@`27!@g#RAU*_ZSz>iEf zYL%zAa1A=Jy!ZGHKH?$u4VqN`{ERgY= z0q|&37v}N>?^aRw>xf*X4Rs=?P-CB7&d)FXm}JSUxADF#UOnzVA_mT!irZ4U%?xuU z{zfYrfy^=5hgNKYQg-X1Q|Vy2!D@Tcy6ZhJ4Zc2=oGM)BxZSdNo!qR%+zi_o(Oz0y z>z`db^A#Ss#blEuTPlDsp*{T`%eA<`_N_OZXT11@)_zx3?P7>5*~0y1tuuI)Av&d? zb?%f!+fauvm3CbX^ldUeb(a_2o#d|w0 zq??S+%I-IX+gh0~p6orac(gC{*R9V4Chw2x#%v<3G@f{cPxh*c6C`i(y>>e1el-Zz z%!53RT6H9E!OW5V=vcn#!j5nN5PyGgfwBZcXu5uSLookg9lbKJM}gr$@v?w(On za2n1fCYb3lk>QFop79ol8#=KEc$}v@B9ZW%9S!r#U59O){VQaL=p<@`)bBOaCoA~r z;opt(QMw=}LfJ}MnTpSa&w{^;;)xuW&OhqXWVE|ce7|yi{dKC7Gbq^ioN-(EQ{&~w zR3}lW#+s4H24%E6K;g0c>CBi^o{Ee*PqQqybx$8n%(HoWb&o}c$!%R#CXsE%?2<&_ zVR~H?t9ZWJiGvY?y{yhBde)}nT#%+Yk(_u#nEzcqlK1`v zy7H?;=(71JC)(t@HFF#tX;^C^{PtIc7sQ=#P>hTI981go4D$jxwyLgrKY^Jl1}XU^Lp7Z zQYRnOx#K(NNLv&Xtot-{(aZRL1-#F^Wr5Q0$#Ckq`ea))$7AZJAZ?lq6R(d~Hz4M? z01gO+bDMlE8D2RUem<>4Rv+NYy5cp)Ox&0w(AV48E#^8{jmF3Av^GNCmetzgT`7m( zT{gUUKk}~8+O0l2awPUQEIea)xpKIFUkl_0A**_S%=K;cM|gp3SJ`4qyBEE@qe&U7 zieK_Cw3cYw-b-0zA?7tlHx$1{HyEyzHy&KCyE+Bt#1R}6d0n(tm2N+d@B73z=mhGq z5T5OFv@{k+&Uy=M|ET^ddTB?iF~wnuedE>`eG9YR`{g?;f;Z9K9s=T3rnYHer>}bZ zMQ@fRI46D{!~qxnT1Nau<7sS(5Chw&0B$ucDA8v5?D(Tqrn)WzH54>!03=e z?YRTdmq}yq<(;OoH3i1T^H0{?tacG?jz>;?Tva=;7l&j^>ZcMcDhz`C2&VFhxQFvQ zn_7!^Qmw7CL?DMQ+Kr`k?#u=QUK?WK*{Pu~H}a3E_8ji4mw9nTX1Xn^O!kkf>qpKP zu4I*WYI^&}Nj2XCCSxt$IMsD(^bXNYhlN3X|ai_(_A*Ook zxAEjww&&`o%<9IWc_)phQ~SohH33WxnK?e(BrA&RhnL}wlHf*BdGO-fUoq9|(4|@9 zh-nz<)M%c;va8~BY_)hj{$N!=4Cx+tRMd$;Csx`*)qsd$<&n_eVVI1>PPXT%s zmfISp%|l+Mqr)WAHd_GVPB89mYd=gIyjO=A^UNSm`?DZ+T}?(29(6F^DV^I&Epc5E z4Xc6UdL~e?rU>;p6Z1WZp0i@~(A1|aEcqc(c667V16eixE(~vdwO6}fQMSJESx>s- z`#QwxX#PQHrxn7GK$tmAM{A;#4KEIo;?a}5i)&5VsnccZif_^h1UW5|xl-klG3(Tr z={@>FoT0KwmeuOmGMB8mk;12%fy!2VRoO-=sy=1aVT0pxmrF$`XR9>??8t&6#VQqP zS3HK5rOqjpxnX%SXG9!Cw&TTDa}&t)R!yx6A{8Tjg`#o$68rmRyeVp$OKK*^d|C94T2n@J6zQ&4%3_3^6{)kabLa|Kw4CV zPOQ}5CdsK)dxls=tqN$%^G|HWBVsnw@kIv?EgTNYqP$`}N@ku`z6B$moP4+c3GtHX zZNKp9KOlcNd*H>6TeZ`}?f zGm$Ia8DG%@hdvf8sdq~`1}4wYAtGvqfm3AY*C*V6et6FgnvLvzn0^QOx8m3aP#i}) zg$pzQB<}Ek<)+?0I%4|2JL11?>irK(dw+zw#l>KdumLKy(w#x1m;wPcC8crz7xJSB z7@*pTkjwH+N_c2+bZ}vG#Vz6| zSXqss+#xMEKFB6+10IvTT4KI6b)gmo8FL`fReE zP$XKo&S0W7xe}oL-fgf+^j#1kFeq?#{1|otYTb~4K4f4tUxgeB2>BeS{4KbvEj?rq zGAMZ=5&{k<`&@J)us}veuq)e|PX&GyN>Xz**OS|N3dkJ@;>Z*jz0gYKVsw#E`XwPd zOEV-42e?=2#uIrsquNa%!L$3HIWsrQg=4Iif4 zf4Zmh>C>-#iNzfXenrXa_fIX*>;1}53jKxq5~?>4jeKT$&Mr3?`cCTN)))J($&S5pN_8QFlgDIXIIa74QQU@jqdN?N~o$yrf1Az_Ycj;j4IKP27@5UG|h!j1v`F?~jC}H}AF=>iu zpsWK_D7Xv@%0wu=!R3iuu%_xX5I#m2LXI;4h%=xk^f!fpB7;)9%aXI`f$O2@hCL-z z(xZeVB0=k6UP}XBM4MP=07enN05Q@9;ob!U3m1a>>zBMaw={Jj3$(=%>j3iMxsp80c6 z50#$+Tv!ccj{{^tT5K2-)VEq2Gq@?H#w>Umxmob*Iz#DqLU~RM4;SDg|9Zu3?r!;? z4GzD3?-=?G5k_V9?a{Yw*v858M|gt*;C3g49eN7%YATJ;=pFC{#ggm~6*y-O1pq^5 z(s%NWF!r>LR7BW@tx2))f9rHyA39x|NkjR#>6$(1LR^5?}r&pKiuCGuAoz?4o;gt zvS^rKQlQ**|yHwJ2t$Dwg71mT;3 zq_kC=!LZrOl6igK?_lYr2@Z;48}S0p4r#1;qIFZ(_9jj#O?<72sE~#2=7RW@fT%ks zwcsU-UB`ml9%REZ)K(%QBq-mnI?yn148`mOG1@z7iX74sq$n9bz924ySd0uVwHRfU zXvG2HhZYztkcOH2`_v!>z;g63`|wMTE8$DY_a+b^jUWp5S972N|CJF36N3#Hw*k57 zS%M@Z0U%uFev8ewXV>$tf!G6@d-1yU?P0`^b9$Ro$K$z^%?V~-8fv*vbJGAe9a;#3 zM_Pi2db5G5Q-A6k;GEtZv$lQFFLB<{jslsMcr=-A!fp$xF=&4VG=c6_$qow-ytCa5_Bq!rojL0LW8kqOspL4A`K8qL5P zLHoh%S&?Bxe))Vwbuxflu{#H0tDeY^E45shqdH)w!-X;CZ#4X&M<0F`RwwP~4BTPq zkA4_ZRn-RD|C9kcF@UoSd3s`O$7$sYNe>+vL_*PP;(2tojgZ=yFr~#i0vi3po7Hjn zGkMC)7y4w-FjBLPncray7&Fq}7%c1J77k+SE;?C+T>86yeqvVE6r7H&8bLUvwM0Ho zF{T;iHhGTKvn`|m8YDu3MPdW$Ebu6egw|?~AzshhV=@bFM}NPlDctfO0BWfb53IbF z(gK_1C&Q}da8H-(4tzmp73M!C(9_5w`9`C6|1}%|_YoROPFA zo%SmgO<`hku#zLQCHu6N}YxVPHFRnT{{J=(7sMK_GIO4d3_-b zX4NJqm0diS8Dkj-SxE@$+rW>$G+0-bG0DcjS`NuVsky-)qnL5$AXvs{PeY?PHoo9n zdUs3A-V(#*vu8dNYUv`=xHS04y2FeHKn zyqZ>m**Z~$tqj+2B)Eq{YWPilV>(VUxTNn{N4##?)!lj4d(k3aU>YL{lnH9f3))54 zUvarn`x7I5(5p3OhXJN&R$lzjZ@Qjn)Ghl`dERbk$h4itH*@Kv#+jYoN7-E*2g9;B zt%XGWd&$9hQ>4%`Z%Qq$hhx7U@N=x47JtkVeCNjYx#?nWSsJJOQQSm9eO(^Xrr(+D z9@gH=_x3!|7}}3VhxX8XTHflsCtIh*sNaqAyz1t4`CgV!(`ia~>8{I?(Y<+{cZ1L~ zT2|Bq6mBQ4%yj9TZvuHoelc09hu+jfk4gR1rRJr?;Jegm*;`jB8x2``ZE4A6QMJb} z;JmwmYf0a1K1XDySe-Z9rF`tVX>J2^ad*A4=vJk>OkLC9k*{-md8`RP(SE*5V>1l0 zPU~4kCO!+dW55ONKt^OFw_swAEz3z}uoI%&HjZrVPejF$9iH{XonYLxw^w!`K^P-z z(zd4WYxwDAn7*i}D34cpj;FY0((D(S20=v?G^8m#UvHhI#_$Zn>X6~&a*CjMo6O`~ zut$q;v*#0ZF3s#Hw8=ItZ7|i~!g7BJ%!SFpeU;^W_UrXB*Z*g5m$voAg7@r~$YPVG z5+i4j>=z8iYn&XX1tC`br2)y^NsG$Sk_xVd^^>u|HM(0-^h@P0Z)>FR)c0(|Zlg@) zG}LSEmd;vLY>{Tz&4l21yp*t3=&$WuJ(H0)yRdBqiNv+J_nF?mXN_OK1QU~XWWJo? zjQ;pGrcSqb2tA2MJxzhDZ;GLCm$!1KI0h#m^X)9_!M-tivduEoYzijFUDPXSK*7uT z)zk5qcz4VJf`__Gr-{*Se7Y8UaV!{n-$Dh1`+C10jeEwVcm)J^Aft50=t+TIMMrX^5M+MGd zUy=)NI)66Q;Oy$^qQs@C+G|Zv58i6=`9p=Ffo6>Z7Y}HYd9g3gq+}@<$%5|{RkF2jpxjh8Eus!J}f9kGF+0&386Nsvo zeVf~l*Xa2YHF=b$?ckbxKzQQ*EbI(vmxnne;lgvhVX;VhA;bPOv|qCOoZ*HooWbqH zI+U;9?)2lZhZrL}#yw!#=!E)K|H#$Inbn(hnPSjt{@|1pnlv_@B<zmNg@m9^qUW--(Y?o4w!^Sw}==nWMtcs54fpW`JCL2p-ZC1jkLpO4MC88 zM_}emW1f3za6W)p1-aQ%*!>;4(%*CJ-tc}lEU9j;AYqcmN4 zfO~!9qpG(=MDBLdKY@AZ!=$ZF%G1(u8n;+oA+hIt;&~8Uq0O&e1`47 zK-0L_oT5C?c{Ka(t7AC+!=oufTJ(nJ%NrWt)RBg9-R`!>>P){+9ikWPg6o;0a`AOS zQ4jpX_rnK4*s({-^ubTsV!F%?rbRsk8Ge%SRP(wd?Cv)cV;%`qbKZ##2dxd~54;$2mgZ-Vi-XvgYxk?!??;Nr z$okuBTZ(uhDVf;@ZXRMHK^KYnEcV8y3>S+{3_H(2W}U8QX1g>E%lh|~-IIM*oGP-& zhaPWg9n&Ehd5_C_yPYBj&-0y=UW~nFb4%@(%kqQ$18h03eW<^?{O7F*+DiMpTd}QZ zjPO`&hyTKBpxwcYQAR3~og-(0k4*Yn`( zk~LD!UZFRg1yYxr7`1(^6TR3}XwsBpEth@u?@&IJOs4zQ#`IG~+a)(Y`;57wHQW>P zpkn}!OadVo=)zf zI?hE$@s}{6tDT2^g&2f(mJ5M0kXfHNW$u*fW8Hqz{k^*Pa0rxNhJKU1NZDgv^E!_~&IHdW8f_a-#563!9&)i9-J@BwQVmceeglAbl|S< z5^1XVSXqp$+y+W97jqng-@o!tjL-?!b+%zS?pd07dA-#|Ch1z~auu70LgA)ed)Nvu zHEb5Il5IQQ;Biw~6_4n?sxIaAHIXmix;SS%*YC*sZm*UH>;#X<9-L9VN0%DDZjA>V zw>@pLFcYxEA9gKhxN5y!jB{Ia#_oQSUewAd)GpzCNGL5SO&oPmhE!%tOr_FYt$)J@ zXmg`&^IELfPI(tNh0h$j2GCO7TR%ON6aQP$`*+^^;*BM-2(7F9M)z;()e4|quaK^K zpr}AV{{J`pv;Vdv@sErYK)pUPQbvw{rL!L?>3>OQ&om&_P?zr@frW+n<$z%$kcE+m zwfT|F1oB~kmH2VupaIwO2$_J9>%z5b{9#yy2K>M=$EcPT{lLczTrt$-DeshL_OBB5 z*Hgs_lpIyxFW%3FUU?tlQthXB_MNAA4%gtWp>MyzE93-)qrE7Cdmn%3XHo@4k%^K1 zi46^ziSdaY9f|&tJg;%KG@)Nf!jOZQ1pRF2p_BDTka`5J+Yp0n<@@mo^=PHKdq%Bo z8O904>BXs0A+jH#;_3EtrPy`+7y>_GA_ED~4g!-{Ax&WRZo34)lKue0NB;_j?~9f} z$cjVE-wlQr2Gz{o$3Y}cD8QT@N0{ah9Y}#J{X-xwDv8aHFb-xIFe2d>X7E`)41J^k z8IKwr3C<)=$aoop4<{5=e=~rv!tl2P$yItaY9GqShy)lf@{4tETrfGAUBc44_5 zC}2c_=5xYxQO^~<;@fa4vVeEY-}adtGQd7#1=wfYFt`HXoO{(u2K=RdB@UYo{sBL!ppB zwpk}5d+32d!m)lPiy=di7Ui+C;P3RYqxlmMg?!2W5+2v^{Qv^rO^*mfM3^1Wkdg?b zfT4m7=_!Di0|7=Ph-d*L677GDNKm)80rdrZj7U7TkF0($FNgsiaXv+up&09t=)ag( zx8|(?Dv~EUfO)lkp%b1AX8*&yg0M4u0(ZKakR-uULZFI$aTB^lh3b>B<+6adCt?Zc zZLXn8QeYJ)t(>_Z?Vwn_J^73x3yb3X6v<9wb|;0>tg&%=)uiHnaFiN-IR?7$G- zzK<>>%l9!c25X%mSt)4LcdnmX(AMvf;ttk26VCTqF2U0SNuFsn135n`(kO=P0?}%=d7LBsQ>+C%ZKI zRxC@V*Bn$QNeCD!Ev*t6>sSaDakezPiwT6e14z|6;#^mDtK)o(oV*6y1u|Am7wr;L z8`PPf(CWzI|G~Wo|8OsZ+#tY?MA$$|z(02+>^^oR5@5_eb|j#w!-P{}xc>8w1P(7z z08I|qUpo?kshNH3`^}z)MN^ zdq)DR|C=FL2m?-Bf{=RqUpo>{s(>Ad>_eHDc!30W$uqklXep2(kUDu9a^!yDtE22) zJgOZ%0h_{|ZgAbsI3=@%McQX@z}-As@g6{XQ=Igquk3A_0bWGy#1sOdO z=wIaPpY~Y*z&^Xw1CC-A#nf!M{~b6HOh3@i`W&|Rn;#kzUA{#2%ep)r@{;1bS;!2a zBPDtTbfo)$j?`^P-aauPs70%E6B2eW+&jFG|3`{3G>e^e!Qzrsv6G*p8Yo~$LgL>; z5@+Gl=QGo}nm=pfuh_U$m=P%6yc-PB*M@VYmzW*wyY&0WKW)d-a3Q&fTVkvH;Nn-J zFG9`v6ae>$M4!uvfM#gt$;a{_3WcUF%b20a){Hj=m}BWHgC52@dQti@HEoNfA=;ss zDa9CSuIFR`k+9Wo&?i7g`WeuXehRiAWC*h{)rEMj`H>^NUzZ?z-G?$<1&?bt3VpB# zGr#X+ARgz8(voZ8q5zs&{V-UG31}1F@LqhlAgcpO6+Sr-Sr}H<{&~w6)4)a56M$e#C0-5Vv?%4a6d?rN! z^$+>1oWenkDX067eAbx%AM#n;0IB{$sdqUwFlGuY0xRPy1ePR6NUtF`v}f!bln!_j z`jQpQ0U2+-0%PK*KKaL)m(=FPAOC4dLd8#f;6IimIQ}e2fCH8!Hl}F^#DDn^c2^%M z*giKH&Vl)EfePBKbSfrFvf_&8O9PjOOJFrg)TRGytU(jdM@u4kt7y-mOlDOKuIh*@ zth-q_&wnpnwh8NbBjEWo5zU%%=;Bj{Y;0ipm`F z1R$T`NWO84nAZx$7gltoJ<)Grmg!(R0G`BZ^X<&s)jxztevfuqRNS@DUOSo+<-RGF z6pX?+o#WH+pnZ{Eqn-%nJ>t2=JLwvCvUA&aJl2;E95sVS_br{Avvc@lZ6@!18o68B zYJ{qK`jBj^m)*XqY;s^wkPLy3+`Q>#U({MoOW1l*v6?x=yRK;4{cuSlTLE%)S?77q z`TJ`BHa!^omnsB$lJml(Jt`FA0+R|0`Y}4#wx( z7z1rnul=<7c!+1b<*cNLgx9U~X8?%T@stQz94fUidA^5Uw;!|~;5ljKJRXyITv_R# zx?FO6O%6>Cz6;>{c2v>gV#?!wp>;+}M(5=LxD7M1It&_-c=Jff*@wAmly~>jlf`PW z3rnfezO{GWM(c@m*;@YYxZ%RlXgp7Nwcmd?^Yi3AgL;@4SC`ID(mCheuPiGW)Kj57 zzP=XNmkdxZEv!uKVbwF286_j7n1hJN<*e7n%L56TY}`{2j?wJ3rUeUO&P`9ci;-S( zItdj6lzZoxC)^`X4k2SR_v~e=Osqr#MvKA64V6?h7|i)gteVrEwqIL9t{I~C4+mgA zL6Vs>Dk(u^nzG8zHP{*)<(C2bw>!G71{bQS{Nl_<)OIn&2#;Md%<5{C_4ZX(M;!C= zD_O17$#Qy~=CoX$tv4Ceu*y>(EQDq2JW0?ch3h;sNPoP0(jFu13rb%W5f*L{2@VR* zN=tfdrR8hA6Ei&%BHuli&2Bq)uYUfvC_c;MS=$rFpB zSLP$ax;@|2ZNoPQ`E&DyG{E`2vvoGQV{5VEbgmQWa^~?`7hZ(3_=>i-AlH@4a^ECH zvgT{*aS|D4^U*kO*7lV6o;Rhst4;E=1=0DaC$rmq^z+1z%yVO-SF-fQ7-xo7DGlIw^WqBR1?Rd-Um+| z<4;O4y?_0NX~%WQcX3nab+)}Q;klZ-De+s8*~ROH5&p!!AowSy?U=v;&or8>U3Y7d zsI6P#SnH(Ps{NSw>0nyPI1{qFb+HbX9?R*OB)+Pp>_v>{AZ5|>1;Zjq7`0Nzj8e^4q$z?Q1s?eAHKi^-kD99AwKNny3L<+AL=0=J%03(njGx zN#1i&Z)s+gil}yP+m|@s@jarsWmg`9U__~DF3cwxz7_?#`?%(zQSXurK6I5yys@Q+ zm#ZueN%D#G6~MqA{#E^gwnDz`0Kc=4?IrP-hpZ<9=QwmJd zv&SpDq4_2txeb@*PgQsQVZO`g0Ehnl{t?Tbnf5cHF=nQorbUOjb|j7g&k4tc%kR|@ zO6}3T%D3+Wo2g0dEoIPVqNc=WH`=X|grX%f>mRqIK-SBnL&;Up02i^j1CF z9+v7&*i7}8G96RZ1Mc}=ro zzamClhEiP(PAnkbtTzCsnENM~Hz&oh8abt>_FAqs9*ksjm~&pkV{Uj~zWc>aLh|jK zk({-)?^~F@eOgwb3IF*z*%i}K|JmmFSKykkb*NbG(v7PK{u1ld(M<4gI-7$xo}=;O zbc!pg6a1>32-68P8XAk&&+T-K6+96e^!5n+?e3DW<$46e%sZzOJ_epPhm1qlNFFo6 zcBM5CaBxmGRg>ST&TQU0j1@E;@k1vQahWDi_d=xD);KmJ$2BbziHEUF+A>;Yu*ZC6 zOM#M@x?S0!gLLye4Ucy}Y4R*^*jFo}Eq^r_Uv;o}l1d&L({WZ0O%k!5_eLWgYGPt4 z?+*s1w1}uF?cr%WDG~1;&qf~GK7wPzo-9SNj^qiLIA83Vdz~rwT)3{kJ{789WN!@8 z=%_nZ^%5vDlh52+8ON!_^mIVO`5(1P(tS?6)p{S|?MugOcyddQ9OInjYCiUtmvhj; zA*|qSRy53!Ijw^|v8jr~e>xO3MjO&0Lx2x*LW2%QV57S7GS_S1HA&PI>te^l==Rp> zex7Kne1Xp`s9D>UIG)Y(JvmKwhL>JdSFi!ZE+jAt|bxHHqd?;}+JlfED3OAg`z@aJu+niW~ zx9a>VlMUUP`;-gv`Tp!1_Qc7o^|J?&Q_pssx@f5DZ;~Qn` z;K{o??y1&fZ3GN#b9(2~ZN*Y;%C+EmxMh)1ChXDg6+`>Od5+}qls9~M@L#FPo-gvz zR=JOWoVd1hOWz+BLwdwpV%7?Q2EQ+CPL){^-=oWY;vK`tu&>zJ*y)vtFfpHq6 z^U93&-d8o~J@|yl9S5FKk5etn&fN}klJc}&!fD3zZullHK{X=c+>ofMs!lD_07Iwa z%H^q!IX2z%afF%nxk7rLamR_PQ*``sk%6@nuYA@U1EN3 z%Cb7G%5o$A>zQhY6iw!*IBMzolSVSS*Sn6;Dfnazz6ytUxAIm6rI%md^sc5R)89WS zB#*(4jLbIM8MhdRj?lT;Thu_xz#5YVZjPnidJx<^_8I}R1Ugq* zL-XoI7ws>j>MX;jF7l`X^DJUxfHr%ah| zQFl?hXMh3Ov-MMLEgU{4MA!~L7zETiM5`5}5Nh-fBhhq|qC|uh(nCuRjbiIlCWaq5iiLIO*Cff8 zq8}B=6H;R;CY>{j=Sqvdh%w8Ab)g28N6p+!2RQf2K%fHhoMtqSJfRDK?S`5+GhWx# zC$0FE#U|m#lKS)gRN$_D)OBo|5ydm9ygww9w=KsC@A zee>l)VFWB46voJWjkb)qsgv^ZgN9dX&LJ_jBm!O9Qb=mRv3A03MeJ4HuQ0tMyCKPgm!WMQGuAY>Zj$B6nCU$W)bAFK4dp* zE$U=3LCO@OcUlk@wFZBF;2R8(s9gHEUa+&iXtK-=9eU3%0mYGj%WmR?yYqT5mq!cHGI%0|sNMGZd~b7h1xX0!~bnJj0$g%otv= z1{h4JsM5)TJjD8YLOgpp^Ta9Hh(2xl>=-h4d=$orhVWnhV?73$VLz-GpP0f^cQ;Ex(_F;IN> z4z1WIN~fqkW}}Q=g5*^Kk_kA_lo3B`M1E>3*Ye-nv6yEF+rHdipiq%qFzszO&bneYT9XxtiqpveiiQ;K z3=jqp{`gsnaMiQXDE2wNcNoL8Li=+x9!jOUru;-M@og|1*zv*ojC>@&*c;AWJTmkx z6q~}hZg6c~@k)jh%j7eGqOa;&jF;21X488keFnk_nDkW`joUm-b43Ml~X>(6(C}O4~GAW~Soqm~}^M?kDXDpHBT9??0u7q^*qm7 z%ktxdcZt zvmT+{azWE~#nQP9ag8=`wWuKG_0qNeEGxoJ4!1Jf zyOd)u9I5;mC3y^JA(rtn@GkUEBh-SypMMtJWNI&4nfjY|vdo}UODph>hKm}1w27++ za1^SB(M_#<-I|gCZcV}GQTt(htXq7{^uOc4?ukBs^X%g-|_bIAlt)@kT; zLb_jJ7LnlO_{g=!)7#y?erfwet9&a#M05G3`Z5zxqSSnS<(hzQ>^ExK@37450NOVO zQaq~9=hrc83i^*Ut_j#43g@qRvRF`D@{P#_;YG>4tYprD3|nAFnS$1GKVq}SAm$>e zgdVS#%C`sH5#Q$Sr0&ZTD`40N4lL{Z4-s3R+ZX$w6nJfd9Hl&Cln_ z>6)qK3#8u@1${O0EDvzT!KLn050L*ze3&Y(&E_sTkKlPlDa%MP(qd$^nHzP?i>j#I zoJrmZe9m;yHTc-=?CB(q`|L}(ks~QgnhelnR#d0bkkaB%X0n31wwcev`-1_eehgQv z&Okxsqki^K+Bl`=+`HIgGry}oCV6_^)%cS|V{>k=K6^8sYQ@cHiTFWNrtx~YacLPq z(Ptils`4NmBJqMB4@p1egHLZwMQmuN5Ih3AhngGq;!=sw`U$Bb$j{SSc}Yt4a4psn zeaZM2Lw&^={IS_n`=T(8#xgFzh)-CKkHVK1&2oFD5B{(i3-9tovA{|NRXtYt*Btf5 z$(^4MaN!Rsuwl&DM3SZ5r$=X4SM1J10JPS~gq-jodEw+e)EroEMOpTb&fXeg(|1O%peo_|d3PFfX+2)z&e@5Jk`g`W8mp0M6)#lL zj6qxdZQB6;-Zm4g>+usW@ACb$#`WqrT-e>rYJcx4(er0max2>}Os1jnoT@qPfIw8~ zjZ{IDEJ+u9Z5>lD+xlBTC;^CtWv5sC^pMSzUpK}>MPWnQYbl9CSs-??%ff^EgVC1V zwtd&~_~)v%{#_>@>YD zCH}(83%nLru)8&nOIbJ?R-DULO@Z<+!9(LXpr5e{%_RnETg!zZ$6>h>)EBRLLq&5J zQdl4TVex2Ge^cnT9|B-dYT9SbwXfba3Ld?RG4!#6;;sDhU$xuah&PkA4QAk@O)YO) z$~^>d%7`e&Zsj#6(F`%5M;&jw)VaEyUJL2_R;OxTQ6tvR(vTRzK3!F$bo8T&z_?Sg zQg3hiU^17(<+qRR%VW*n-+u5ES3U3CrS0tP+=y!2&O7Tzrq6PJht#Cdpj=uqxy11s)r!{C z@X>u!Yol-Oil^lBW;aE9)}VoMLzZ#CUD#6HnO(~?xKw}J7y5t^JsMp;QnroWIB2;E z$z;k3`n?a!#uaRcQ&#-})lNtT z|I5UW%*V&fUp(xsM`zKgC&t#~a0}GGa$gB=ghuKW7t`}+V@96e^(=!aK#_yFK4?pf zcrRSVcq-%`&z##iIN`=R6bMdSCkMwmgV(eDoum(8oy%11W^4i>8l@BQp;nuEhA*}t zcEw{(;kYvAS;+yr<8Z>6v}fy8Zofa_qC3f6nKhhpko%n1BI@}iv8kL@XHPx)M6^tS ztk+o3uzOX<@-Ft3EV?v&S!6HJxC)<(ynv==C% zo6WL39HOOkAsHFl1Kj3^UCqvC@($4gUy+Fh;T|GQuJDx7+}1=yu{R7@Y+kPSH<78g z(vR!PHWRp`gn5RKik=$(mOIad&n4kdYcst?AI1%E{kZ#KR?EKzhjVRLXEkoY%G6{R z7_%$Q04JdO>KfCFlvH==rR&axKYWz8hLgEI<+R4c&wjOvHbx^8(KW$s#`Yw(W@z=W zAPw3B^jwEcGdy4hfOHs<4cjdAS)eG7C)0aXDvH^#iZvoi-lpohWndMq+*5ond5_>CXmK8}#86 zsU$7nbY=0-Oj}q=3d$x{K-QX zr!OH_JaiA8le!9mg_locT7cf+7d;s7=vXMz()EyC5n|>1S?2Z zFx70fzlu*y=h*Ujv>x@?Zda68U%ZhNTURk{pe-h1L(woXs5S<_>Yqhd?_~#RI4tdO zUCgnW`1a)3SX($PE9Hl7<=N8Fbd}L~`NUsW47}EWa=nk}_lpZjT=BGsBX;eXXFv;> zB_G=usou~hxkN!tSq=Vz%Cn;h(7YiZt1jj-yn8wr64^v@Y$wk%uVTwuXhW6AzqZ0}_}hno$qt;=#FieLcJ7=vbeWVw-+dSvwwm7Do(v z_TILq@c6Kvn%v7@#EOz=TH*ZuQ8{fGqptUKVt{>uC2yZYO?;}O#&5b2L>09&;a+61 zQWD2b%0qOnpRxL^3Y4cQcp5%h&J7uXlX5S2{eBQ+9{{GE{-MKg*)(z*Ck)ro*`ni_ zh2cEilH;}_e)w$p?y*x}CNJhCRzaPN7*B6&_|S*r)}?jY>>PtVWZk;%wnW)siX{9_ z|4as|YRxXH%hDMIL5qxyAc^q{A47RyC8hp#H_#~wuET6gfE5e!uN}Rmn#MKD>x*&Z z7D2*l8P5v++3Y&){7J|}c?Czz4NoF|lP3$HKFl>sVV9IMG7ZzzG_aZ8o}y>Q}Ci-c9g!z?QP{GhGs3d_RG+MfqZ5@ zACKF}`H$p_zo%nfFd|1;H|4z1m$Ft?YIJ3$hen@I^1V1Rc@mlbhldQTE0^lSsTVq8`Z5Sg4RK+(9WKAtdP_t&!`Uzk{E z%zJYq>Rhk-cpplVs?!>o%6rXUk0z1qeyrW+w>l;DuFQL^%6Boe!my<24ZdqZR4i`F zZ1%q|mK??we&F>@p-Py(>~1I2UN-k7KCTqLb?Iq=XUM8XLesK7(d_QF7P7S}wLY*{ ze%?&qFQw!#tk&G~{Lp6=UZFmyHZ9>Yh_>0b9acknBi&_emuhd zw}+=sM#c+B;?NJM|GsMWWoVOs7OTYuaILR~g#7i;JGyy)JpA^(IMyeR$82&dx{6?AxLQftVQXmH`h{7~(4=|rRd_IAn zLp**AK2Q7Yfz=z77tKwItC6#jG0*GxS(v6LmN}LtgZ$5L-|At1a6OG|YrN9>?3O4H zZ_&W6%^a2YNx1UK(NB==83+cXL#swg94AZ+_s?Ti`|qjD(5&_pWY2@qe9!T={!Wel zw(kt~4Ib*-&X3O>Fwyr@?VmJUi-i-AcPULNIe4k967A%`Lb|xjH7qedFbVK9db{4! zsN^Q8^A}4lEP7O8RDLS-R?JLwnTvmCe#MGZWOQeC@#=qQV#x0Dpkc62@(yQuw~5Fy z{8mp8k7JWkYdBYF4+rv%2~1k(n@lhL=Csl0;CpQ$J!4MJ7BYcUtO#7aSbQEKMxXcL zlQE6X`o?4YwDgCpGwqt6a0%(V1ri(uyd$pN9r!nyV8Kte19!1<3AI%5g>4}EOSrLi zg;LdQ1{QT_*)XaGj`fD#=6ejC&Mpj$(^a+9H1#q}sRTHJn^R5P zPj@ug8XedmIr4^%;*ei+T8WY)g=bu!yd9t-1WTuaUd+!?vKO@NUB4ne&4KlLprM2O z%1{K|-6JF7bI*Weam&iyQ7@%aa%xh1>1MBF1_?pF?TmK~CqsWEM65c$M*ZUBdmYW+ zIlwKt9YcISZh7Yg{jDmEPBI|bu^Z$+Z4jqI3VdH-lHAH`z1Z2lR#|UPV+N!+cXP%iPT_^qwhzc*i7gm#g-^g zzj}kTPx}-{fOWxW72m;_YsX-yeM1nsl$muMcR9D4gUj52Cv0T4Zw{t>56AcBOlMql zJ0E`(gHS=*aO|&f*m>14Mk9o6zEIjr2DcM#uX@p@NDl}smExwG+)M+f2hy z6F+aK541G-)ZGdSvZUPT+&d4@$^L9N&}2xtWPv7_kapsFxp1Iz=AP@PaE28QP)0E^ zG1+UkZ7P&m9)O;L@I0YpKaV6ZBs&e4O==QKqMTBq=&N(6KyEIY5xP~KykhJkBbt4f7rAA3q-3!@BS>6NI{)-;d9$APR<~BCAqt~GJ`Nzd$0+mQU0gCb z8c8H~YGr?iS#(71EEaT3iJ7^USqTN#$Ifn$JKSEIW9S1U$c@eR(%kF~{rI<4+b~P~ zZ|l-CD~+uGSOn{<5%i)wW&1X!DNWgZB&(?#@=Rh)1m_y}%3($dblKi%?RyPao%MRY zS+Wyi3{Kif9BK{)a*XO?aTciv;ga<5MO0Gb7+AR!L@pD(Xik$$aeZ1DnHv@Ba_>CF zz|=j02hZnKbbIP6J@#U|ucKTOa^5peQ*FBhQx={}47usJtU@Axw#mIhIQBG^10NhN zzW?-mT8nC=ls*Z_pdjY}S+&4kHRrs%U7CZg8CGAzZuU1^m6}rH{t;N{xB}Ql-ofrT zRqkU3lxo-)-Rvp`$_(e?-ia(osTP}Sg-4&tU_hZb+4SM6#*ljS{au)`e$EmsEQFa; zVK>Pw43&~dZPZ!YSAVz_FtM-bj8TpFdY!OeF_Larw7I0nH~^Qvr=&JxH>l1+z52xe z!uKV|KYZrFY)@@9lKwTIuLGBNhJqF_Mg9oMD0l-W>M6B1Up1yIf*yUgA-G4!nUl;t zNc}n(tnM|P5{^~X0N=lu$&J_Pf<>_gTj({5%0ey8!gun^G_s5?Ba@kkZB zD!>mg2DEb!(ZF_9pGdox?07|sT6XlrmtSP|_p~Cmla@<_KFpgVec0eKSA0oO?o=X7 zPe(|kR!1drc0=_%hOw9Q-%0b_`mtW-N<2|}WmshWm z5E`-^Qa*iclW+O=2(Rg&6`)s;Lm4{0VT^HKim88WxAk>rY*yU-m_RnFPs%DO2wkB%=SXKgtn{j0v zec`j#^tmOXl%*6arj~|%=byegAM*mOaz;g>-c7fdneXx_l~pWEA#;vVtV?T|m`$c6 zjMy6K{~SP|aj4D^yrw0_+=6S(zGfvDGvae)(8hqsAh|7K6>{aByfMs43U@pt7<^ z3)|?eHG#Og{|AkzZwBg3J-i3@ zu@@q}ilKn)n+v5e#+3BtVKv<-kUTZ8<*DSL@rU7OM(xMVPAbkpF`j!`afS=BkW{>B zdKCXwI2cB?=2wC76qnOA;6?<&!=Rp~F7utg)Vlt#oo}tpawu8sDfF{Y&&!&Qio>DI!I*>h}km0h1C0cxy{ zGm{JEgFyvrhr=gZvf9 zmrO48s{o47rsjPEof5yi%c}{Mopa?C(c{)OeDoj)o23~Nj3aEl>>X$+b)n$`b*Gu< zP*>V!420c>-oo8@>nLiRca4rDM;UE8N($8AfZq~vR-6e-2M#iR;*W_*0s(T9^nrCo zI=5D;1!bl><%!8W6xI6J#TM>bic{{RGO$#s#SplG%Mslk&f&s*JU=CrBw$}gRCswx zvUaBnW;MgIa?b(=!z9_Y%?2Xwg@zFroEoxTm^&dhX`p_|9Ime=+P2iE_5iKA87?=! zd+>OqP1l8jVoOruur9!hEy5V76L~OOXRPf>muulQUmz-L<8Wi?*7|3l6V221g}QB% zN73KA+VFy1)$BN`ILo-)zOd!IoUJEt08Gq*S}LApaNNbuwRDO|Y}NQGYu(Ga)!Z#n zT<`C(BGd$CaXN41S=5EWeR2UV+NV0k&b>$90({hQlLx%L@+rMNAOr^_ldp6{eqqrl zK;{FpgUy2C0g_T3V!X6MJxtRJ@3lp}`(@Z0kQ+ddc*{OI%Hc)d@o~HRR*`Z}f9;rt z&O%*5Ilwq^Y4G`f`^9K7o&DP%6y`sE8T>?X_;*Y6*SX75-URMnz_i=;|G#PXzd4ox z)9!y9%m12o|8E`35GqJZsGv!H-yy9?siA+%QU+xB6@vg1Li)jL{6>%U4JT%xV-V$) z3~R%$lmuCN8$T|L)_{j!p?Yzhv;Gp`ibkU077ZJ%iqax4EyLI zCQ-NlaV)dKMZ5eHN;3q+nH`fFJb28^Gj=ATv(@LTM@JAEJOzn$RTg z4i5we9VMPD;sD)M1hNNWB3&{3pZqLOXckG3g>Ko&Cf%8fEZ8xf#+w;hCDL2LoF4zo z0=-^7I=P51#qu^CBf9{(1F3FFkB?kn2#JV5zMl+1#(eD`9Xb~>|3JBbX?RtR#rN*- zn3SN_h;hK|;`||)cm zct8@oGvI!|J@x?#P`Yk%$AZJA~Q`})S6=j(9;GHFurM{Zw}?o0u2rMMS?Jao!G z!K9Gji6UE1Z~=N;M&LrDh9};0R~+<3&7u9@6^FrLj@oZ_nBaLLLD^Zd+z+PVvA;PU9WeU3{HN`+XMdMMM}DsdM3|LBbj+ z39<@bG5P4z8^n(#GoUJ#DUnxPi8*0ILh? zeoyb`$i?Gtmtt_#Mn1WqD+Wl8Hr=mCTG_uMX`uf^(vsu2{yA^4v6;Dmryj2RE}IE# zJ%F8UVkyXY?ai-i*dU{=T`LuYnmwp!7>$#Q&9!}dx*Ylq$H%`wy-CwV&S>t9hB*u^ zfu4neD5|7_lucIz{05tRwLlPzp00qnO_VO45h>ZMe&sG$blFk`V0D3rVDFIqr9iV@ z?A^}7DuIEm0TUS#wB6hp_bV{6&`G6k*<#NzBDV)hzZ9tyKNc7MyMH?(GCIDDEiZ0c zKb<3+q{x7t2A?8!I6_KnM5FyXr+ghD5I3TzV7R!#LJ;tv9s=_ojQqFnOmv!_(}ED1 ze!RfaJ|Q|7BC!9mx{Sd?j2Qf5b%CHx%qBN58?)bTE(EBRLkTQTF5}qS`fRhtFLc7Z zlNXK!g}3EeE9Sc|975#6Mg~OT^<9w30_juCl$nQ8UI11Xc}A+QMp{m42T{cu1Y%Nm zO}B*u!0PfuN|#4;p^FZO=F5=qjoRd0OfEtdr`3W2U{v0T4GK<_@+X$2{zZY?;p^H^ zok7YAxNVW3Ko~gdHd3er;{0BAqGTp5KhC=i-uyeeall4SJ_}t%1Q%vzVBw6bq4t^f zH&e|@Cq5IGNUty*`@Lx{$4Qtkmta;9D?h;MG77M|IDA=MoG`hR`Aw3iie5ddWm=w! zOdx^({7jU=dL(v>fC-pIEGH>0IjNapHAfKhPB>9GL|mfwTg=OmZ7Q`T*(Kmmf`&Eu z@NUvaksHcYT`q2G$*SH)^6QuRI&C4dT%e%E!evip_#?x}2Lc@`6NW`PX8ZIt>drIR zEid{#UZSq{&4{;hQid^kz+oVzkUd!1a@Ayk4H0$l+7~^mF2`m(%(c6{lAve@;8jM7 zMj{kKz}_8;{szmK-|4|TDhkw`{!(%gJttLNsFY0urXViYZY)nhZed(Df*es78+0I3 zYu&*V(EJ7%b%&PL;1z$3y6ePG136k%!)B*azDC{DfKfML2-TNXSq^(;7?|5)IP66B z=BS>3CG_)9CGK-X&u4ZyKupSp=7Yimiy+f8%haXAt%jwcxq@D@9m#wfV+-F4U zcrm6ZiCs|8P=GWJ&?=9ZZ@q8~DTm8*nemEN;PVA$OZ|xb9>>DYm@=jlgqoc3JFXNL z#0sPcDx=$krL9VCsc`CIa8Y3;2nJ;L-vVh|5wKv)$O3yNDTR79_U3(+0}W9cPyJgm;m(;Phz21PTp63-ALn0+ z3uVlT(|gwx$!RR~h?*57a2CNOvU$Cd!5(mXf>{4!)-Biz42>5#6&>2J6nT#XEcXq1 z#QfuH)(uUVNe7s9lb5WL!&oV$4at#4$f_WI%DVDA&~z* zRfQdo{`MHwLjjqd+GyDmjR1MjrTJpx+1{+R0i+-7heNg%_n-xsbxVih&v~S9^S^TM zNa!4DNcRag!NtPqwXV_IVGdIyD%PQ{c{4#yK7G7cbkr%%C3;%xU?*8{3zOt&`cXN5 zb8^~8W9j7ZaM&5nKmz4Dq0*dnJ`a)7M1>o7#_>Gtg|;7J!s1wEM}G>6f6Wr_)K)F! zS?Q~G5$I}ik9}>y+9I}MkAiMyu~!k$tyl-~e?$;=xFI1#$tU}i4c=uSBP0zu1@GkB z(qEIgLU8oE%|$%0Ca0B1EkJ7T3Qn-q2kkJe1%=i)L9bt$!SqW1raj5ZLmT5}-BQpa z@c!Y5R3-hVzpC{Y*hHW>G)`|Zzk+C{*DL3e#WC{DYQUy=k`@!jE0(yD1u_tt<0+4Q zgZI`0M_s$nfpPiTXWqpbhIJ&W(yRUAszhSSfKxlY%iR@F)D*WC9uM2i^}%+-C95Gz zQ3BcMu0^Ak=Vy3_u2R)IO0xW2Xudg}>7rkNb(fa1#)l1x8VX`_EwSa%1GV`L#Ut>> zB)hBO-@c+s;lwL0r5f`C)QZ#EPz^cm#-SKer_#L{i7J1R_fH9?c5TEopu#opoVbcB5Ujh6$z)h{{i<~%NCo+uiuja`t~7dftbV_oU& zp9kcTJ?noDT9m?KA(<_weYboS)gKjBpLan@F+Mz#xNknL>10h*x3M*>&%$uJoC@{_ z3%G`o!?pMvD-F{5ov!)W zBUtxg`qE?Wd(pP?YXJzSEGGvoH#OP0;^V!RiOpoc<6ID-g48wK(Mz$O;YFLieN;u8 zg_DEX9ECs>YoDpq(p=$;2!!Quwftn0FJDx9-D_I~R)wU&8mp&}VlJ z_zWunnzJ443?Iq}7&k7Ymz}ob*=bmmxZZ61;Ck41r#CeZ%O(6rtzybGT$DDp!wt995~=SrOA@y)v6NHAoL9!pH8(z ztD>%s^_|1d4MAxhIC2KU?4V+9dp&#R@c*)lw(nZ<+brTrsV$$&ssE!rx%*2xqNB0G z>10+-a6lWP%J@n8G9ZeE&9glt;q?xA$~?o-DpIZCP5fh6s+&HX{qROpIytrDeQazQXvT1q>=l)xd3` zd2*G`TShX(@v}_6$x`E${K-(^02-%5L09c~13PhbjiE{zm4|EPRJV0uhZ<>2n+NK+ z59>y8XP>2Ic)K~JahrJT0;hBhTkc+7Z!i)u?Fk_%V+15>&`^c=A;?`J0%5tS%wg`3 zIQR+&OJW@Af#Oe=fvcgX?nqUf9z~(kt9g6PM!|2B#F3{Il1d94^BvWBPkzVD+w2T1 zDuXlea+%*l73S;$LPk&88CI=|G*0#UavFL)+qxsdj~7H%518-bI}$^ba03Xzn?h87 zIMo$p8FQzCJO^$o!H7Rx%Qw*p9+9;0xC=UsPNmusEPL$T^N>Quf~OVLEoj7urLNtL zRN!}6tTuA4BzZQUgBGf!9}3r3uy8DnZKYAKYx@UAL-@0s203z`ncg$mK}PJooDqZu ze}bkZMjEHMp_62s*pKbCp&fTj9b%+LbnZ7Xl^G9r9G3LNyn2Y{rT*Y)rM!6AeMjN| z-!$L6dt%}&cp5s=Rc^%yR%)))v{tcv9?y2(4t%RGOMm*>^;7Q+Z54Kr7~l2 z!{{w(elQ%j`*9ie11DYjZ-ymN*O|-qY4wMH+@rL*4UuJ<{mV42@VMos7t3oC%ZSPG z)v_m-hIq#Q?W*;5%R%`CkIi2Y<*0O92%tz+s4%%LjVTR=sEYlnGX8tGYMP|W$cAR+ z>KnoMjjpm+cIwb*m;qE}GKpIQ3~Low>^CWdie}yW`&yLC+}O)?$&uCt?z2i!5@)@r z_0h_z%R)~ck_>OJi$M-`F*FVPHZFaa$t&tr;X3RntdIOs)a$swbF8^C8%Lo%$+|p@ zsKSh^wT{i+PS6;`@T?u0s1%1*nTr~`-wlYjaV1-&){hKLS99letWX2LYvdjINA zJT-M)f8XbH9^Ou(o9)hFnE1SVONyzs`I{V>TU0J_`aHl!o%p-+M|mmsCQ_!0o&MOQ zY+4kG_!t@~t0!@|g|*pp-f?oo?ke2hHA#z)zpjc=KBmDLIM?(}4xF?LuE8<)Iho0( zCJ%uy!m$)Zq((+ma5=yw|zb#A{6G;N%4sddI{@WUw0g0&n! zOwl02wxloy)HU*Ea&;4gpWAi{w1<@-ub~@->0PLs;H@t=N!mi{9|}s>UP+I3UHirp zdyR3ITpWt7(eP-G=bsjJM^WYDV7MugcMn3`EZ@>p17kVTBnh-#m zbP=9`6*PFssll6@OceWD8%$(R{Mtx&Krc`Wib<_Z8oaOnbyQLQUnqn@vArrWf55nf z@B8wkZp}e>u+vU4x~Z$Gtf;D}{Z(03U;eAMthz!^ql&hpVo(mO%T}EaAWb)_g#gkd z-^FDEAWib{hzEUyXJs#m(cPS(z)3-Mx>uSZ)q(Wfqd|H3tm@&n2ne`)yPUi`z;qsX z-y`{+WEjODWufr{zYM~*+q7Q>VHtsb5*1iqTSx@HfO=>=P7O+-5qLxtc>rnhrIaAf zUBnY13k|IEL!popC-|;HjIRodEZS}={swNq9OR7<6ds8eBXWkOOX6EVD3%K_2*>o& z^vd~!M#I0X9NNe$%0OT~d&r}ne}J<>8jazwRTt7}?1Z(Inu(i%AMYxN#E(0N}@O+7Oy73>?g= z#c?s5)-801JanKOUbqtU!fxOm-RAIs_$_=~^xZiqgwog8+HbzmrM(fzGq+K#Rue00Ntef><9tx1^e1{A zG|_yKrc#tiC``qQFVdv_MVb^Svf99dS9Q371Rt0P^M971&7dUt4yYx?NY8n~;QgFD zDR3y93}m)I4LvKfKzY%SqrgYf2mCXHnD*~sX>qLuz3=G@6%>53fQnb-jn0g>;bR-1 z;sqV{7nh9x2_-PXJiU83X=fB~68u}HxQ_+tZ#u{7#&3?0*+9UFjVD;WAg+tZ@gxJQ z)nKG*40Muj-Z|CnY9_@rsxy0Unj_u5bzGvjC4uvRvGHIlA31@4{b}bnm)|PV%S#j|yLblyxj9lL*T% z?8^w)xj``HJc3+XR;#0Y;+(t|tVMk0G#AYR6Pw=TlYr8YeDHv7+$vy@Aan&x_+e^W z2~{a1`ai%)atUDi0)+;7=WDc63RI##QT_CEyNH4fc!4!G0WfTR$N+z^gffnUEMF2X zd=C=7-ZFA1Xu-E(Lf@)9c(VT>O=Q$DxsV2>0Me9v@K3}`65}Gyt*wtv`ybMj3Ls6I zHb2*u%nY2_1;_^Wb_hf0I78$4RO0~}m3{zek_M2biZ9YMr{FGb)`CPx9oXl)a+HV! zdM%4z%XHxdAWZ;&Fn~0@iqR*iX0=*i{NoP}3Qm>sCkK!wYQhX(*M;f~YJqPj`Cp`I zyvHcM3P74D{%6vphyz54A3b=qQ~HqkciH1gS;C^GN<{K8=0-ML$}{yivQ<|{dZq|( zGTQexU?# za)3nTxJoADL}TYfP{y5+@mPuJ+`z+VD6|J?;xo&s+~ZWW9vs7voPoX;-d|}zQ7Blz~s4;~>jCsQC*FgG9L7)z(f$@((UsVa^bkkD% zr};)0`E&v=FHGf-Yc>qX-t=U!@l!WD@-KTZkSaeS@-|)P-Te;lAk02Pg7i&qUP;y{ zEZaFSk{QU$SHNq4BT|iSgHlQqb(4|8)B=|6wNafvK=@NV;VpJn$=wMsY_eQp*cnd4>ervx<~Lm`(9sxaGDqZ7 z{erkK`FTKbbu17d7ACE|h2SDsPYz>oK9u_>ZKcc51(WY4f^ouDrjEzJ_Hn(I55JnX zCvD@l6rz5~$d!f2A{5X0{C+C0|8BgbviukJc9P^RTo=jX2FEu>ZwUscT zH0tup+xqd6QPQ*F7l*(P`w0aTVce7?C7D1P$_Gg#6#n_GNJGfx0ckYxGSrJ~c^` zJNr^Gdt}sLNGX{|2Pv-4Lt$+)Tre=dEnk(Na@O9$aqMlz%!g<4W6bxb+&}VQU-^Z` z3Uqqb%x7yf&&o2SrID2}dt|^i1h*?7-sO=cBD)7CKBH&C1FD@o+u>i}bm*?7>S}(D z)Oo)pAhFsy80%5KVEb`dTSVZm_qt@^*j8_|jMwD8ThA!L#OF?}{Osw z*fvAGP+GkCdRN2Ff|HMAD!96|_FE`??^PjUMK`OFTc|kn@171a1#x)rP=w^ z!+Lecb1A)U@}jG4<8oIhYyrnj<*3d#$;A*~OD_M&U!{E+yZeLJPGCrFi>4;%78Z=M<}&a7s|@^C zX-=)&e{52mn8Jeyqr@Ak;T@sr?Z$$ot2DiFt)(MHlV{Fjy)?(E?-$kW1{~+9JHT!m zKA9uia+DvOb*~xVq=HO&=ew^aS;C1Uu+;2ph^bx_bQglIFUhAv+v>>8AC)F+Va3Heh=!REh{w~aYwvXL_3mmek+YIo^6VUx(ks~+_q{$!FzQOR|6)n) zb5I+1iBYOn)TL{@P2Z;Vc{nBB9a|K=7jaG+nevcBZNlJ;m9EITlCUJT;c}((bZ>~A zcFQ=B)_dKncbN+Avb>3)W`uYWrS<(%voN_861v+tgXDZn8cAyy1MA^i+1gs)zc7Td z6!QlMtX6F-W5r>IHPy9-VX?fj)sL}%v}@cg4$TDHXL2`lhJk0G-5E<5RgZ}Aa|Q1j zI}SZIHkHrb5DA}gTeHnvhNAuD5q6%>Q#5`_I(lH)SALB?63_jy>^C)PP~^{2O+WFAKfj~g(H=&akaZ~RlI#H2j}+ZE-Zm*X(^fI1qw^U z%pA3)*was#r*BU(EtYviYCTZXV-~|o4rld?$DxOFo7)#I9=sjyRH*p-^xVdduznFhHGl1r z=tBts=gNa|5{rcgb*8}92S;8re$MDJ)-$WV8W&uF{&~AsvB~ZZKIj&4j7$rVLN^=J zy$44KG`O`|K##LDX?VJ4&E;hXpF{IUrcYOu7r)!QCzZCmq*nP^rDvw=^-8hw;M+=1 z+0<>%c!s`dJxCUx)ty4GI6ga0Aa%WRZvmXpgxa`LB$63a?5U5d)*tM8&7OvGwowCa zCYc5?{WLWELmu9~7X#UfUG12UdIne5EuO;2cSnnLdQz8K9_NCtKa(s*j<(t0{KpYG zQUe=BGp0B(dbqrkIbMSG#xx9g4r3<$_Idr8*++< zqK>>U7uSy(Z=LaW7rQg62=g-ZUl%i_shhdoRf}8)AIr?5GE}_u<;yo_X7ecYnySRy zQ&|?=Z`Ler+pL#AXg5!n%5smz=foEi{1uL{s)jH=6R5-N3a;%N&7;(qj{4)CUX^-+ zVed8@o`8P}yrd2{TN!rdhVx_KgRx_V(UYH^Cz{P@GS3?M5VJMU!Ai$<$TCQG4}vb} zJ3pXiAt`{J*LxW*p403OU}cprd)r?((rxd|QmM%-#FANWpK&L}Ibhx6&q+QwCy$E0 ztZ?EAzllf1KEFcWtq-+A_$|3_>}&C;#OcAi=t(S2+z0NbGzFt=fBz|~n5@S6gX?8{ z8U|u+C6=@KJN_gU4k2^-|o^ zUX@Jy)}IRTn}#XA{n>j)kLOhz?v}4D;rHq{Z8u{pLuls&6k>3-)exq?Evkg)D1$lA z8Hn@vsRagYH|%O!FN=svQKnXDldQWOws~23+M<0Psi0F4s7pud3mPWgBZjAw3`{$0^iMLZ0LH^;7ERsmKlop+g_XZ}RupRBuPKvwDSCi7m zurPdMi}l+~48C9|Dkc%{Rcrq7;KMv<$mKcVLIt<@_wYyP>rRdgu;+wRqs}@zN(SaJ z+xm^?>RO%meEr9!RFkW<(qNt=SHjj-GeMRN47S?}k}Nf*y>Q$0U@mp22WTyBT$?fL zpfb2SKIV+$o3d^X@mNiZ#Rx^Is0+iQ&;FNAX-k?CO+kr>sr_wk-6*d(Y>M$2><&6Q zqIbKf7_>&BhwEdfY~?rJsEeS3o)v4#F|oMOiCXcvu9gKZo&mx;+iNUo`l{-!H71&C zL;hRL%^vrFm`cSku=rJ_0S!7%!Ty)yET-G>rxAh80|Eu?b@l4jN{#Z8$EW2#d%>HK zU>vE7i4n`IW0@|UjzN!^TT=Zqf*=obs4tx^@N^XK6^b4uE6k5Wn{Ve?OpO-ngcKQi z`7WzC)4)IH2dvLe^UN>MS^>R2YsI z=6#ZVw(6<=2uCDA+&Dw4{fdvtxwU)?)vg-U(tLd}dWfeIeV1%wuO+;#=WL+HM3lz4 zs#Q|5y~v-L=J5)lnOn+Ke&3!Bo*rgRBLpkA+AXH&rSQf|^>hwA>kX=hf(`Rx_kgH$ zeJnKB(b)n{yN8U=c_jI1LNF`Cb9dDFzG4I{|TxR9Oo@IEBpV=0V z1&l-?ScsQa+!P4$L2kveOh~L!n|w~`+75Ou9RqPblWCLQjMjyoc~I>=OX5SBdgV`( zU|T^_f7~!eFi|lMIwTh#=g=wY7b@<4f(qF-wrh_Y-N?a}zFr<*`h^iyBZF4)F){rW zel@{4&-1L0dYIwYGv%}p|E20a+)&$PPSXXB!JWxxr@ z|M>FI$$GMNfD47pLeOvO|JBR0=`u4h>Kf2dG3l`}P%#?PGE(Uo=`vE$>Fcx6(&@4? zvC`{P(Q1F`g=wki>DaUZhYHNJbbu!|S|+A{h#BC!Z5<6A{_|snO#W8|$-1{i`o8D4 zt4HdjFbG&t@XLFNi4)fW@v{E5+bDQ`rET&iBB%j%B1=``JX4(MLV?a8Z&OO2-?bM?GXIbC*(RBbPpqv;&} zyu9Gq-(zt+o{c41dp4}t9~UD-pTt;b5U4bF6$Rf$q+WWoA&;aGM_!@<9`~vtROob<)=O*ZEF;kkq>&iTppee zl1!59Kd!pX9dX|6HvIxI8pE7 zuJO8TZt^@1!Fe4ZX|p|buv~g=u5bSGBIf+uvHx^GzwvCmd^p&8{_MOOcD$Ec?QQGD z$#|>7p;?{#!{GdJv?+D=SyZ@nY|X=-o`Q9;3I2JM!s3;zVb$!<@cO`%W%02sZyc(9 z=KgX@;*`PZ!m90ZH!`ulZr-TzdL7YKZ96u2p|e`F>1AoO_ZY;#U9$AC752x_%#+FU zV6XE4YX8#f{aRm|r_0>?(6Y$;b1dt5^k~3g#2w$S;~926@>r(s)-u- zIQx9ax+rx+Tf50Zs?mT&*mx>ZgL^aXG;AspOS2=)5-N@=C7USD>=<0`-fr8E)4!!> z4RQ4rYK-J&rmHe=)&IzRdvWy@lEcj%lHmEggWq#~)jhN&^PE2~F!TI*d%IJkv%1dN zc^txPZDjb$_2z=$8BXRj4A~o zv4P2JGuu|TdRj$t2_<331-|>`?H!#*#5Ay?(7uF{n5!3Di%zzbb$XR51Dj@gI@wZK zC2^#IG=I+uTa4=ujEx%a)~9X*L$9DyV}|)ttZ+BykBjlMi@v}Tvl6vZ38ARc-P<~g z%n5omax^d!D0?8nypEAs5iWJJjm!-LW?M$~sXVOTPL`JV^EfP$$Yp^~I(dS4)EXJE zxi#jNY)c6BK@(IR$At=GxkH$R^d;jZ0tX2KL)QYr@~8O5kN9cyJ&TX=G$t!b>K3ur z-;qC;6I%40CnNfwB^ZOT*O4>D7lZKG z7=m-_nie138TKP_%2yS;z~`&LXL2URC(;D7T2+p_jCv&w(w2hTs8fyt zap!4SU9fJfu{#T-8TE`8fa>}g`ZDy)|R+k^ED!4=N3w4_f%frSN{tCf3;l+JXHH0Ke(xG+;m&6 zE0c;*$b_+LbIH>7Qr05t!IVK`h*s*>?Io0GdAC*D?WR!L7u2hZEG0_1MJkmj>b1Qr z?|;rQWBzl_%$ym$@xhsMX3p>X|Nhqh|D5yx8(w!i5PsKXX@2w}Uv7a##k=*~nKfPu zH|4mjkD2*;_^~%t9v?QjCoEOJl3h1LcTn;U{o`j&%^a3+-*U(JWqkVs%yCcd>_fAR zyH_s0!|`6I!`g6n3V*_%2UeXu_vXa?YR_BMBg(U{XO;w2GvEJ!#_ftfXBhgvJcrqQ zfQM$A`;M=lKhbt*0=IJBDz80@YM(wY4eDIg{nOf(rv67~)HJ%ka6}JzzuA1e_(Bccx1#^Jvs zI3oBTKlQSow&0ET<7o#~8cIhl;S2VF|JmLvv>5pN+~zNKy7<=d&1p-3kN}_6OTPnW z&wL8L=Y4Mv1}$-4Ie0^Se|uxBCb32Q-GQQOHjcx$3&(ymx9vUVsrShWoJsKVOv~$S zSI^EJMPE@Ru$|~p)LMycNO*Z>;NQ*u1u&I3{QB%<@@ErkO2b|YcdI{R&dRN`N&XVf zn;9&6Qt+`Ay4c4~E(iwaI-XLXuPivevi5HS4Z$1X{{}xsIL@8UUT}{caz0BHqb}NK z%Eijc$;nB_lkA@Me70>-{NVxEc46k3OLv4=5G1!TA!jPsU31g#I(FPsr+dXV0?rgi9iu5$ zY(odR&v3%MXwEeg%~9(e2kS3xRN1neU6!9a zl-{k<-aooNd$sZYUC!EZfn^R1c5ULOIaMEO+x}e^KH(3u&07Du=91j8mN`V7oqcas z+pX4N$&Bw=KFXJ_+?cWF*{~m*OB^+Weoe95?)A#vBD0D$bm$<9wVkRVw)uJ^gI2m6 zttyQCK7o_*irL}%>dJtoFX_%x>XUsPgLOCGF74XYzDV!4UwG*^x(@1lZ)IhaoM&e2 z3ESQl+HW3bNJZDYaORWPk`cF}ou2desA!j+Q1-p>VT(}>d+sAo&qKq{ZI3u)_GH76 zajogzPb@vM9piX2H#V~Zh6(UdOfM;|**`!~oz}R?&WEem)=7RD@B5^EC1}i8kiXX%01uE7g3rO5)dBm}VqeeY;DJPp| zkm#`>%hGNsKE*62L@~KzAi1eh<;DbuqufARiWHxecPu34l{7JY4i}B!M546ayK|eK z!71QrW$FmxB9w!<_w<2MXcWpJr}7Xqf*%+h#pQDNAl`&VaKggDuowt768)ZV5zOy} zP6#$!Av6hUeMiGX!4O>x9}STXhsZJ*ocR+P3Qkgz(;+_`Je^1+2P<~OrYsf@ih~S7 z)gl2JUy`aOfC5^g#Z#*aAGDN+Mxv^Exe0a6h?ok3F$>6cgH)vrmU>eF4Lqo7YGKM| ziQtp2GQxte*=QtNP}2y7V2OfY!4`x~M?X(VX|167MZVZta!aAsGJGk=gKEl z(E`vGYU(}th4!%`bb{p1(36X5v5K^>C87g_rL=EYunEZ&tr_83)tM<5DSap+*$@Gz z<&rBbA@BezqkE2}p$+G9*r!3MvJZ@7`UnMMRpn@Xc^WJf4CN(!#7LwzfXhr7fmS?A zE`Z>L8vGyN$(=SY8rW<+K}p|#zX3i{T8FB18tHT3>m^I6;n^DZoxVO*rdYaRHpl!! z7j#}!Cn87?q}}Bv3c({y2MMQp6KfkwQo5IsmBg(Ob`l#fpf9ls5vF@VRuVdiI8LZG zf}KQSflpmY!|=+!4Lqnug5k-@9_+*Zb_%fvg&2|%!agh~eK5ZFis2L2E=(VcPX>SD zLtcWSjHph2o1_qb^tvj&Mfjt+OpSYBEF*y>?xr1z0T3t16n0Zn6H6I4sg>jmiTSEa*kr4z~-~5FN;Uw9mD0_5=6ng!JWw<m#VGECJ$h^f36L1o-OQ@#OCvR4u~ zO^m@0j!|L~?e)b|uic(|1B?OSS1?{pjBHFxl^-K9zCn2lt+e6uUO1Z0adv#5S|yTd zN2bXS2QopDYVit5s9O9vae_99DV4w}UA0D8eZ&crN18(16rR2va7jb8I|Hhk-T*G6 zNL)7d6(fmD5qAb|g1a-}CN$i+^y>7(3jod+q_7il_8V|1$(^w);$XsdYZ+I>du_9` ze?ou?xH7Pg#Ai^+x5g=Ao7g?uU=9N_FkPlJA0pD0_~7VxVc5aO3Y8ww!HG|6PwVm< zR=mM0V~XC#5G;o@x!1`ARLH2G4)ZjKFZfOCN|5G_9RJ4tWxG-_DF$)tc2i2x$Am;i7xvJ(ale%3|Qx|m}mgX=@uhM!LnWuuWw zAkCZXu*fI#X%%{mz&HMh0yH$HVuU6_N#+EQ3vL6HWKR85LB)kQfge;KToIp$dH??a DKm{Lj From 9c3137ae3c4589d131271e90bd5a1ddd9c32149b Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 25 May 2026 19:29:07 -0700 Subject: [PATCH 022/255] chore: Add release information for Apache Hudi 1.2.0 (#18831) (cherry picked from commit 62ad88a3dc938d5362082fed4135af8cfe528bdb) --- doap_HUDI.rdf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf index e76272529d32e..921decaeec726 100644 --- a/doap_HUDI.rdf +++ b/doap_HUDI.rdf @@ -229,6 +229,13 @@ 1.1.1 + + + Apache Hudi 1.2.0 + 2026-05-23 + 1.2.0 + + From c7be772580dab386e569836933c9096b8cd605c0 Mon Sep 17 00:00:00 2001 From: fhan Date: Tue, 26 May 2026 11:18:46 +0800 Subject: [PATCH 023/255] fix(spark): Add options for archive procedure (#18437) * fix(spark): Add options for archive procedure * set 'enable_metadata' default value to true * fix args in SparkMain * fix options in ArchiveCommitsProcedure * fix(spark): set named parameters with higher priority and improve extractOptions() * optimize entire impl and add UTs for HoodieCLIUtils * optimize ArchiveCommitsProcedure. * optimize ArchiveExecutorUtils and HoodieCLIUtils according to hudi-agent review results. --------- Co-authored-by: fhan (cherry picked from commit b5c5801bf7d5e0b53c60ef5d46cea1e2d22cc6d0) --- .../apache/hudi/cli/commands/SparkMain.java | 2 +- .../org/apache/hudi/HoodieCLIUtils.scala | 60 ++++++- .../org/apache/hudi/TestHoodieCLIUtils.scala | 104 +++++++++++++ .../apache/hudi/cli/ArchiveExecutorUtils.java | 17 +- .../procedures/ArchiveCommitsProcedure.scala | 99 ++++++++++-- .../TestArchiveCommitsProcedure.scala | 147 +++++++++++++----- 6 files changed, 372 insertions(+), 57 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java index 097ba984dc923..a7955032497ea 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java @@ -599,7 +599,7 @@ private static HoodieWriteConfig getWriteConfig(String basePath, Boolean rollbac private static int archive(JavaSparkContext jsc, int minCommits, int maxCommits, int commitsRetained, boolean enableMetadata, String basePath) { try { - return ArchiveExecutorUtils.archive(jsc, minCommits, maxCommits, commitsRetained, enableMetadata, basePath); + return ArchiveExecutorUtils.archive(jsc, minCommits, maxCommits, commitsRetained, enableMetadata, basePath, new HashMap<>()); } catch (IOException ex) { return -1; } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala index 1b40b0fe57015..c6e37f5bb6dd9 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala @@ -107,11 +107,63 @@ object HoodieCLIUtils extends Logging { } } + /** + * Parse a comma-separated string of key=value pairs into a Map. + * + * Notes: + * - Whitespace surrounding keys/values is trimmed; empty tokens (e.g. from a + * trailing comma or `", ,"`) are silently ignored. + * - The delimiter is the first `=` in a token, so values may themselves + * contain `=` (e.g. `k=a=b` parses to `k -> "a=b"`). + * - Values cannot contain literal commas; the parser does not support + * escaping. Configs that need commas should be set via Spark conf instead. + * - If the same key appears more than once, a WARN is logged and the last + * occurrence wins (consistent with `toMap`'s last-write-wins semantics). + * + * @throws IllegalArgumentException if a non-empty token does not contain `=` + * or has an empty key. + */ def extractOptions(s: String): Map[String, String] = { - StringUtils.split(s, ",").asScala - .map(split => StringUtils.split(split, "=")) - .map(pair => pair.get(0) -> pair.get(1)) - .toMap + if (s == null) { + Map.empty + } else { + // Single pass: build the result Map and collect duplicate keys at the + // same time, avoiding an intermediate Seq + groupBy + toMap chain. + val (result, duplicates) = StringUtils.split(s, ",").asScala + .map(_.trim) + .filter(_.nonEmpty) + .map(parseOptionToken) + .foldLeft((Map.empty[String, String], Set.empty[String])) { + case ((acc, dups), (key, value)) => + val newDups = if (acc.contains(key)) dups + key else dups + (acc + (key -> value), newDups) + } + + if (duplicates.nonEmpty) { + logWarning(s"Duplicate option keys detected: ${duplicates.mkString(", ")}. " + + "The last occurrence will take effect.") + } + result + } + } + + private def parseOptionToken(token: String): (String, String) = { + val delimiterIndex = token.indexOf('=') + if (delimiterIndex <= 0) { + throw new IllegalArgumentException( + s"Invalid options format: '$token'. Expected 'key=value' pairs separated by commas, " + + "for example: 'k1=v1,k2=v2'.") + } + + val key = token.substring(0, delimiterIndex).trim + if (key.isEmpty) { + throw new IllegalArgumentException( + s"Invalid options format: '$token'. Option key must not be empty and options should " + + "follow 'key=value' format.") + } + + val value = token.substring(delimiterIndex + 1).trim + key -> value } def getLockOptions(tablePath: String, schema: String, lockConfig: TypedProperties): Map[String, String] = { diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala new file mode 100644 index 0000000000000..ca4869286c26b --- /dev/null +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi + +import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue} +import org.junit.jupiter.api.Test + +class TestHoodieCLIUtils { + + @Test + def testExtractOptionsBasic(): Unit = { + val parsed = HoodieCLIUtils.extractOptions("k1=v1,k2=v2") + assertEquals(2, parsed.size) + assertEquals("v1", parsed("k1")) + assertEquals("v2", parsed("k2")) + } + + @Test + def testExtractOptionsTrimsWhitespace(): Unit = { + val parsed = HoodieCLIUtils.extractOptions(" k1 = v1 , k2= v 2 ") + assertEquals("v1", parsed("k1")) + // internal whitespace inside value is preserved, only edges are trimmed + assertEquals("v 2", parsed("k2")) + } + + @Test + def testExtractOptionsIgnoresEmptyTokens(): Unit = { + // trailing comma, consecutive commas, leading comma — all silently ignored + val parsed = HoodieCLIUtils.extractOptions(",k1=v1,, ,k2=v2,") + assertEquals(2, parsed.size) + assertEquals("v1", parsed("k1")) + assertEquals("v2", parsed("k2")) + } + + @Test + def testExtractOptionsValueContainsEquals(): Unit = { + // only the first `=` should be treated as a delimiter + val parsed = HoodieCLIUtils.extractOptions("k=a=b=c") + assertEquals(1, parsed.size) + assertEquals("a=b=c", parsed("k")) + } + + @Test + def testExtractOptionsAllowsEmptyValue(): Unit = { + val parsed = HoodieCLIUtils.extractOptions("k=") + assertEquals(1, parsed.size) + assertEquals("", parsed("k")) + } + + @Test + def testExtractOptionsDuplicateKeyLastWins(): Unit = { + val parsed = HoodieCLIUtils.extractOptions("k=v1,k=v2,k=v3") + assertEquals(1, parsed.size) + assertEquals("v3", parsed("k")) + } + + @Test + def testExtractOptionsNullAndEmpty(): Unit = { + assertTrue(HoodieCLIUtils.extractOptions(null).isEmpty) + assertTrue(HoodieCLIUtils.extractOptions("").isEmpty) + assertTrue(HoodieCLIUtils.extractOptions(" ").isEmpty) + assertTrue(HoodieCLIUtils.extractOptions(",,, ").isEmpty) + } + + @Test + def testExtractOptionsThrowsOnMissingDelimiter(): Unit = { + val ex = assertThrows( + classOf[IllegalArgumentException], + () => HoodieCLIUtils.extractOptions("k1=v1,invalid")) + assertTrue(ex.getMessage.contains("invalid")) + } + + @Test + def testExtractOptionsThrowsOnEmptyKey(): Unit = { + val ex = assertThrows( + classOf[IllegalArgumentException], + () => HoodieCLIUtils.extractOptions("=v")) + assertTrue(ex.getMessage.contains("key=value") || ex.getMessage.contains("Option key")) + } + + @Test + def testExtractOptionsThrowsOnWhitespaceKey(): Unit = { + assertThrows( + classOf[IllegalArgumentException], + () => HoodieCLIUtils.extractOptions(" =v")) + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java index 772450903e5fb..b52d1965e0407 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java @@ -40,6 +40,7 @@ import org.apache.spark.api.java.JavaSparkContext; import java.io.IOException; +import java.util.Map; /** * Archive Utils. @@ -53,12 +54,26 @@ public static int archive(JavaSparkContext jsc, int maxCommits, int commitsRetained, boolean enableMetadata, - String basePath) throws IOException { + String basePath, + Map options) throws IOException { + // NOTE on builder ordering: + // `withArchivalConfig`/`withCleanConfig`/`withMetadataConfig` each call + // `putAll(subConfig.getProps())` onto `writeConfig.getProps()`, which + // includes every key filled in by `setDefaults` during the sub-config's + // `build()`. If `withProps(conf)` ran BEFORE them, those defaults would + // overwrite the user's options (e.g. `hoodie.keep.min.commits`). + // + // Therefore `withProps(conf)` is intentionally placed LAST so user-supplied + // options reliably win over sub-config defaults. Named procedure params + // (min/max/retain/enableMetadata) are forwarded via the dedicated builders + // below; if the caller wants those to win over a same-name key in `conf`, + // the procedure layer is responsible for not putting that key into `conf`. HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath) .withArchivalConfig(HoodieArchivalConfig.newBuilder().archiveCommitsWith(minCommits, maxCommits).build()) .withCleanConfig(HoodieCleanConfig.newBuilder().retainCommits(commitsRetained).build()) .withEmbeddedTimelineServerEnabled(false) .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(enableMetadata).build()) + .withProps(options) .build(); HoodieEngineContext context = new HoodieSparkEngineContext(jsc); HoodieSparkTable table = HoodieSparkTable.create(config, context); diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala index efc5a0cc5c2a3..6dd578c91cf61 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala @@ -17,8 +17,10 @@ package org.apache.spark.sql.hudi.command.procedures -import org.apache.hudi.SparkAdapterSupport +import org.apache.hudi.{HoodieCLIUtils, SparkAdapterSupport} import org.apache.hudi.cli.ArchiveExecutorUtils +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig} import org.apache.spark.internal.Logging import org.apache.spark.sql.Row @@ -26,17 +28,36 @@ import org.apache.spark.sql.types._ import java.util.function.Supplier +import scala.collection.JavaConverters._ + class ArchiveCommitsProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging { + // NOTE: min_commits / max_commits / retain_commits / enable_metadata are + // intentionally declared WITHOUT default values. Whether a caller actually + // passed them is determined by `isArgDefined`; their effective values fall + // back to the corresponding ConfigProperty defaults (see `call`). private val PARAMETERS = Array[ProcedureParameter]( ProcedureParameter.optional(0, "table", DataTypes.StringType), ProcedureParameter.optional(1, "path", DataTypes.StringType), - ProcedureParameter.optional(2, "min_commits", DataTypes.IntegerType, 20), - ProcedureParameter.optional(3, "max_commits", DataTypes.IntegerType, 30), - ProcedureParameter.optional(4, "retain_commits", DataTypes.IntegerType, 10), - ProcedureParameter.optional(5, "enable_metadata", DataTypes.BooleanType, true) + ProcedureParameter.optional(2, "min_commits", DataTypes.IntegerType), + ProcedureParameter.optional(3, "max_commits", DataTypes.IntegerType), + ProcedureParameter.optional(4, "retain_commits", DataTypes.IntegerType), + ProcedureParameter.optional(5, "enable_metadata", DataTypes.BooleanType), + // free-form hoodie.* config overrides; format: 'k1=v1,k2=v2' + ProcedureParameter.optional(6, "options", DataTypes.StringType) + ) + + // Mapping of (named parameter -> hoodie.* config key) used both to merge + // named-parameter overrides on top of `options` and to back-fill scalar + // values fed to ArchiveExecutorUtils. Listed once to keep the named-param + // <-> ConfigProperty wiring in a single place. + private val NAMED_PARAM_TO_CONFIG_KEY: Seq[(ProcedureParameter, String)] = Seq( + PARAMETERS(2) -> HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.key(), + PARAMETERS(3) -> HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.key(), + PARAMETERS(4) -> HoodieCleanConfig.CLEANER_COMMITS_RETAINED.key(), + PARAMETERS(5) -> HoodieMetadataConfig.ENABLE.key() ) private val OUTPUT_TYPE = new StructType(Array[StructField]( @@ -52,20 +73,74 @@ class ArchiveCommitsProcedure extends BaseProcedure val tableName = getArgValueOrDefault(args, PARAMETERS(0)) val tablePath = getArgValueOrDefault(args, PARAMETERS(1)) - - val minCommits = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Int] - val maxCommits = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Int] - val retainCommits = getArgValueOrDefault(args, PARAMETERS(4)).get.asInstanceOf[Int] - val enableMetadata = getArgValueOrDefault(args, PARAMETERS(5)).get.asInstanceOf[Boolean] + val confs = getArchiveConfigs(args) + + val minCommits = parseInt(confs, + HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.key(), + HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.defaultValue()) + val maxCommits = parseInt(confs, + HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.key(), + HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.defaultValue()) + val retainCommits = parseInt(confs, + HoodieCleanConfig.CLEANER_COMMITS_RETAINED.key(), + HoodieCleanConfig.CLEANER_COMMITS_RETAINED.defaultValue()) + val enableMetadata = parseBoolean(confs, + HoodieMetadataConfig.ENABLE.key(), + HoodieMetadataConfig.ENABLE.defaultValue().toString) val basePath = getBasePath(tableName, tablePath) - Seq(Row(ArchiveExecutorUtils.archive(jsc, minCommits, maxCommits, retainCommits, enableMetadata, - basePath))) + basePath, + confs.asJava))) + } + + /** + * Build the effective hoodie.* config map by overlaying named parameters + * (only those the caller explicitly passed) on top of the user `options` + * string. Whether a parameter was explicitly passed is decided by + * `isArgDefined` rather than by checking the parameter's default, so the + * precedence semantics stay correct even if future maintainers add defaults + * back to the named parameters. + */ + private def getArchiveConfigs(args: ProcedureArgs): Map[String, String] = { + val optionConfs = getArgValueOrDefault(args, PARAMETERS(6)) + .map(p => HoodieCLIUtils.extractOptions(p.toString)) + .getOrElse(Map.empty[String, String]) + + NAMED_PARAM_TO_CONFIG_KEY.foldLeft(optionConfs) { + case (confs, (parameter, configKey)) => + if (isArgDefined(args, parameter)) { + confs + (configKey -> getArgValueOrDefault(args, parameter).get.toString) + } else { + confs + } + } + } + + private def parseInt(confs: Map[String, String], key: String, default: String): Int = { + val raw = confs.getOrElse(key, default) + try { + raw.toInt + } catch { + case _: NumberFormatException => + throw new IllegalArgumentException( + s"Invalid integer value for '$key': '$raw'. Expected a base-10 integer.") + } + } + + private def parseBoolean(confs: Map[String, String], key: String, default: String): Boolean = { + val raw = confs.getOrElse(key, default).trim.toLowerCase + raw match { + case "true" => true + case "false" => false + case _ => + throw new IllegalArgumentException( + s"Invalid boolean value for '$key': '$raw'. Expected 'true' or 'false'.") + } } override def build = new ArchiveCommitsProcedure() diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala index c81ffcfb59d6f..e5be572eb65f1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala @@ -21,52 +21,121 @@ package org.apache.spark.sql.hudi.procedure class TestArchiveCommitsProcedure extends HoodieSparkProcedureTestBase { - test("Test Call archive_commits Procedure by Table") { + /** + * Helper: create a fresh COW table at the given location with `numCommits` + * insert commits already written. Returns the table name. + */ + private def createTableWithCommits(location: String, numCommits: Int): String = { + val tableName = generateTableName + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + | ) using hudi + | location '$location' + | tblproperties ( + | primaryKey = 'id', + | type = 'cow', + | orderingFields = 'ts', + | hoodie.metadata.enable = "false" + | ) + |""".stripMargin) + + (1 to numCommits).foreach { i => + spark.sql(s"insert into $tableName values($i, 'a$i', ${i * 10}, ${i * 1000})") + } + tableName + } + + test("Test Call archive_commits Procedure with named parameters") { withTempDir { tmp => - val tableName = generateTableName - spark.sql( - s""" - |create table $tableName ( - | id int, - | name string, - | price double, - | ts long - | ) using hudi - | location '${tmp.getCanonicalPath}' - | tblproperties ( - | primaryKey = 'id', - | type = 'cow', - | orderingFields = 'ts', - | hoodie.metadata.enable = "false" - | ) - |""".stripMargin) - - spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)") - spark.sql(s"insert into $tableName values(2, 'a2', 20, 2000)") - spark.sql(s"insert into $tableName values(3, 'a3', 30, 3000)") - spark.sql(s"insert into $tableName values(4, 'a4', 40, 4000)") - spark.sql(s"insert into $tableName values(5, 'a5', 50, 5000)") - spark.sql(s"insert into $tableName values(6, 'a6', 60, 6000)") - - val result1 = spark.sql(s"call archive_commits(table => '$tableName'" + - s", min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false)") + val tableName = createTableWithCommits(tmp.getCanonicalPath, 6) + + val result = spark.sql( + s"call archive_commits(table => '$tableName'," + + " min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false)") .collect() .map(row => Seq(row.getInt(0))) - assertResult(1)(result1.length) - assertResult(0)(result1(0).head) + assertResult(1)(result.length) + assertResult(0)(result(0).head) - // collect active commits for table - val commits = spark.sql(s"""call show_commits(table => '$tableName', limit => 10)""").collect() - assertResult(2) { - commits.length - } + val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect() + assertResult(2)(commits.length) + + val endTs = commits(0).get(0).toString + val archived = spark.sql( + s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect() + assertResult(4)(archived.length) + } + } + + test("Test Call archive_commits Procedure driven only by options") { + withTempDir { tmp => + val tableName = createTableWithCommits(tmp.getCanonicalPath, 6) + + // No min/max named params — archival behavior must come from `options` alone. + // This used to fail (Expected 2, but got 6) because withArchivalConfig#putAll + // would overwrite hoodie.keep.min.commits/hoodie.keep.max.commits from + // user props with the procedure's named-default min=20/max=30. + val result = spark.sql( + s"call archive_commits(table => '$tableName'," + + " retain_commits => 1," + + " options => 'hoodie.keep.min.commits=2,hoodie.keep.max.commits=3," + + "hoodie.commits.archival.batch=1,hoodie.metadata.enable=false')") + .collect() + .map(row => Seq(row.getInt(0))) + assertResult(1)(result.length) + assertResult(0)(result(0).head) + + val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect() + assertResult(2)(commits.length) - // collect archived commits for table val endTs = commits(0).get(0).toString - val archivedCommits = spark.sql(s"""call show_archived_commits(table => '$tableName', end_ts => '$endTs')""").collect() - assertResult(4) { - archivedCommits.length + val archived = spark.sql( + s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect() + assertResult(4)(archived.length) + } + } + + test("Test Call archive_commits Procedure: named parameters override options") { + withTempDir { tmp => + val tableName = createTableWithCommits(tmp.getCanonicalPath, 6) + + // options requests min=10/max=20 (would archive nothing for 6 commits), + // but named min_commits=2/max_commits=3 must take precedence. + val result = spark.sql( + s"call archive_commits(table => '$tableName'," + + " min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false," + + " options => 'hoodie.keep.min.commits=10,hoodie.keep.max.commits=20')") + .collect() + .map(row => Seq(row.getInt(0))) + assertResult(1)(result.length) + assertResult(0)(result(0).head) + + val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect() + // named params won → archival happened, only 2 active commits left + assertResult(2)(commits.length) + + val endTs = commits(0).get(0).toString + val archived = spark.sql( + s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect() + assertResult(4)(archived.length) + } + } + + test("Test Call archive_commits Procedure: invalid options string fails fast") { + withTempDir { tmp => + val tableName = createTableWithCommits(tmp.getCanonicalPath, 2) + + val ex = intercept[IllegalArgumentException] { + spark.sql( + s"call archive_commits(table => '$tableName', options => 'invalid_token')") + .collect() } + assert(ex.getMessage.contains("Invalid options format")) } } } From 4d06094008a7a4d347e80aa88b3f8120850d3407 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Tue, 26 May 2026 11:27:06 +0530 Subject: [PATCH 024/255] fix: RLI bootstrap fails due to NPE with cleaner table service (#18836) Co-authored-by: Lokesh Jain (cherry picked from commit 652d95253648f1d7b5437e803a405cfa9d1f6f86) --- .../HoodieBackedTableMetadataWriter.java | 24 +++++++++- .../TestHoodieBackedTableMetadataWriter.java | 45 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 5d09762909be7..7035c40c9b42b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -58,6 +58,7 @@ import org.apache.hudi.common.schema.HoodieSchemaCache; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; @@ -934,6 +935,27 @@ private long countRecordsInHFiles(List baseFilePaths) { .sum(); } + /** + * Resolves the data schema (with metadata fields added) for use during record index bootstrap. + * When the write config does not carry a schema (e.g. table-service operations such as clean), + * falls back to resolving the schema from the table's commit history / data files. + */ + static HoodieSchema resolveDataSchemaForRLIBootstrap(HoodieTableMetaClient metaClient, HoodieWriteConfig dataWriteConfig) { + String writeSchemaStr = dataWriteConfig.getWriteSchema(); + HoodieSchema rawSchema; + if (writeSchemaStr != null) { + rawSchema = HoodieSchema.parse(writeSchemaStr); + } else { + try { + rawSchema = new TableSchemaResolver(metaClient).getTableSchema(false); + } catch (Exception e) { + throw new HoodieException( + String.format("Could not resolve schema for table %s for record index bootstrap", metaClient.getBasePath()), e); + } + } + return HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(rawSchema, dataWriteConfig.allowOperationMetadataField())); + } + /** * Fetch record locations from FileSlice snapshot. * @@ -971,7 +993,7 @@ private static HoodieData readRecordKeysFromFileSliceSnapshot( final FileSlice fileSlice = partitionAndFileSlice.getValue(); final String fileId = fileSlice.getFileId(); HoodieReaderContext readerContext = readerContextFactory.getContext(); - HoodieSchema dataSchema = HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(HoodieSchema.parse(dataWriteConfig.getWriteSchema()), dataWriteConfig.allowOperationMetadataField())); + HoodieSchema dataSchema = resolveDataSchemaForRLIBootstrap(metaClient, dataWriteConfig); HoodieSchema requestedSchema = metaClient.getTableConfig().populateMetaFields() ? getRecordKeySchema() : HoodieSchemaUtils.projectSchema(dataSchema, Arrays.asList(metaClient.getTableConfig().getRecordKeyFields().orElse(new String[0]))); Option internalSchemaOption = SerDeHelper.fromJson(dataWriteConfig.getInternalSchema()); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java index 4065f113d518f..7fcc2ce9a5d85 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java @@ -25,7 +25,9 @@ import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -54,6 +56,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -64,6 +67,7 @@ import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -377,6 +381,47 @@ void testValidateRollbackForMDT() throws Exception { assertDoesNotThrow(() -> validateRollbackMethod.invoke(writer, instantToRollback)); } + // ---- resolveDataSchemaForRLIBootstrap tests ---- + + private static final String SIMPLE_SCHEMA_JSON = + "{\"type\":\"record\",\"name\":\"Test\",\"namespace\":\"test\"," + + "\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"; + + @Test + void resolveDataSchemaForRLIBootstrap_usesConfigSchemaWhenPresent() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(writeConfig.getWriteSchema()).thenReturn(SIMPLE_SCHEMA_JSON); + when(writeConfig.allowOperationMetadataField()).thenReturn(false); + + HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig); + + assertNotNull(result); + // metadata fields (_hoodie_*) should have been prepended + assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_"))); + } + + @Test + void resolveDataSchemaForRLIBootstrap_fallsBackToTableSchemaResolverWhenNull() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(writeConfig.getWriteSchema()).thenReturn(null); + when(writeConfig.allowOperationMetadataField()).thenReturn(false); + + HoodieSchema tableSchema = HoodieSchema.parse(SIMPLE_SCHEMA_JSON); + try (org.mockito.MockedConstruction mockedResolver = + mockConstruction(TableSchemaResolver.class, + (resolver, ctx) -> when(resolver.getTableSchema(false)).thenReturn(tableSchema))) { + + HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig); + + assertNotNull(result); + assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_"))); + // exactly one TableSchemaResolver was constructed (with metaClient) + assertEquals(1, mockedResolver.constructed().size()); + } + } + @SuppressWarnings("deprecation") private HoodieActiveTimeline createMockTimeline(List instants) { ActiveTimelineV2 timeline = new ActiveTimelineV2(); From 3e3d0b8af1ee106cdbf06c28d45e2730b280bb9a Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Tue, 26 May 2026 19:51:11 +0800 Subject: [PATCH 025/255] chore: Fix Flink CI Maven profile arguments (#18845) (cherry picked from commit 76ddebcc522876fe557647fa12631fa698ad6263) --- .github/workflows/bot.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 06468d0c4fa23..fb7dd760c5058 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -580,14 +580,14 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client + mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false + mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false - name: Generate merged coverage report if: always() run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1074,7 +1074,7 @@ jobs: mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources - mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" + mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" fi - name: IT - Bundle Validation - OpenJDK 11 env: @@ -1316,7 +1316,7 @@ jobs: FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }} FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }} run: - mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS + mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS - name: Quickstart Test env: SCALA_PROFILE: ${{ matrix.scalaProfile }} From d24332afdce4d0ad2c4648cfbe80d516e01c034d Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Tue, 26 May 2026 20:48:15 +0530 Subject: [PATCH 026/255] [MINOR] Handle cancellation error with HoodieMetadataTableValidator (#18371) --------- Co-authored-by: Lokesh Jain (cherry picked from commit 516d9e2093a5a10fa98d9e446675e66039db6647) --- .../apache/hudi/exception/ExceptionUtil.java | 51 +++++++++++++++++ .../hudi/exception/TestExceptionUtil.java | 57 +++++++++++++++++++ .../HoodieMetadataTableValidator.java | 3 + 3 files changed, 111 insertions(+) create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java new file mode 100644 index 0000000000000..92e2e3cc53564 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.exception; + +import org.apache.hudi.common.util.StringUtils; + +import javax.annotation.Nonnull; + +/** + * Util class for exception analysis. + */ +public final class ExceptionUtil { + private ExceptionUtil() { + } + + /** + * Returns true if error message is contained in any nested exception of provided {@link Throwable}. + */ + public static boolean validateErrorMsg(@Nonnull Throwable t, String errorMsg) { + if (StringUtils.isNullOrEmpty(errorMsg)) { + return false; + } + + Throwable cause = t; + while (cause != null) { + if (cause.getMessage() != null && cause.getMessage().contains(errorMsg)) { + return true; + } + cause = cause.getCause(); + } + + return false; + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java new file mode 100644 index 0000000000000..dba850b1b907a --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.exception; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.apache.hudi.exception.ExceptionUtil.validateErrorMsg; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestExceptionUtil { + + @Test + void testValidateErrorMsgInNestedCause() { + IOException rootException = new IOException("File not found: data.parquet"); + IllegalArgumentException middleException = new IllegalArgumentException("Invalid argument", rootException); + RuntimeException topException = new RuntimeException("Operation failed", middleException); + + // Check top level exception msg + assertTrue(validateErrorMsg(topException, "File not found")); + assertTrue(validateErrorMsg(topException, "data.parquet")); + // Check nested exception msg + assertTrue(validateErrorMsg(topException, "Invalid argument")); + assertTrue(validateErrorMsg(topException, "Operation failed")); + // Validate no exception matches + assertFalse(validateErrorMsg(topException, "Connection refused")); + } + + @Test + void testValidateErrorMsgWithEmptyMessage() { + RuntimeException exceptionWithMessage = new RuntimeException("Some error"); + assertFalse(validateErrorMsg(exceptionWithMessage, "")); + + RuntimeException exceptionWithoutMessage = new RuntimeException(); + // Empty string should not be found in any message (including null) + assertFalse(validateErrorMsg(exceptionWithoutMessage, "")); + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java index 778cd0090bc7e..daf9370c35509 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java @@ -75,6 +75,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.data.HoodieJavaRDD; import org.apache.hudi.data.HoodieSparkRDDUtils; +import org.apache.hudi.exception.ExceptionUtil; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieValidationException; @@ -690,6 +691,8 @@ public boolean doMetadataTableValidation() { } catch (SparkException sparkException) { if (sparkException.getCause() instanceof HoodieValidationException) { throw (HoodieValidationException) sparkException.getCause(); + } else if (ExceptionUtil.validateErrorMsg(sparkException, "cancelled because SparkContext was shut down")) { + throw new HoodieException(sparkException); } else { throw new HoodieValidationException("Unexpected spark failure", sparkException); } From 6834874bedfa600a56f6f8ab6b1a8bbdb891365d Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 27 May 2026 11:05:44 +0800 Subject: [PATCH 027/255] docs: Update DOAP file to include 0.15.1 (#18838) (cherry picked from commit ce29423932ec2a4088e08c8fae3a73e1f6d9dcb8) --- doap_HUDI.rdf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf index 921decaeec726..9f56441f3f45f 100644 --- a/doap_HUDI.rdf +++ b/doap_HUDI.rdf @@ -223,12 +223,21 @@ 2025-11-17 1.1.0 + + Apache Hudi 1.1.1 2025-12-18 1.1.1 + + + Apache Hudi 0.15.1 + 2026-05-19 + 0.15.1 + + Apache Hudi 1.2.0 From fbb5c1f1d90e7411a3d51e1e9355bbd79ed686fa Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Tue, 26 May 2026 20:12:18 -0700 Subject: [PATCH 028/255] =?UTF-8?q?feat(trino):=20[RFC-105]=20Trino=20Hudi?= =?UTF-8?q?=20Connector=20=E2=80=94=20Shim/Bundle=20Refactor=20(#18782)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit e5ae1115458c99696399b7df62d48f6c00764426) --- rfc/rfc-105/rfc-105.md | 225 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 rfc/rfc-105/rfc-105.md diff --git a/rfc/rfc-105/rfc-105.md b/rfc/rfc-105/rfc-105.md new file mode 100644 index 0000000000000..51c7aef2fb902 --- /dev/null +++ b/rfc/rfc-105/rfc-105.md @@ -0,0 +1,225 @@ + +# RFC-105: Trino Hudi Connector — Shim/Bundle Refactor + +## Proposers + +- @yihua +- @voonhous + +## Approvers + +- @codope +- @vinothchandar + +## Status + +Issue: [HUDI-18780](https://github.com/apache/hudi/issues/18780) + +> Please keep the status updated in `rfc/README.md`. + +## Motivation + +The Trino-Hudi connector currently lives in `trinodb/trino` at `plugin/trino-hudi`. Maintaining and evolving the connector through the Trino-OSS-only path has stalled in practice, and the cost falls on Hudi users: + +- **Hudi-side improvement PRs to the Trino Hudi connector are not landing.** Four stacked PRs targeting the Trino Hudi connector were closed by Trino's stale-bot for lack of review: + - [trinodb/trino#28518](https://github.com/trinodb/trino/pull/28518) + - [trinodb/trino#28533](https://github.com/trinodb/trino/pull/28533) + - [trinodb/trino#28644](https://github.com/trinodb/trino/pull/28644) + - [trinodb/trino#28645](https://github.com/trinodb/trino/pull/28645) +- **Significant Hudi-side work for the Trino connector is ready but cannot land** through the current path: metadata-table-driven partition listing, eight `HudiIndexSupport` strategies (column stats, partition stats, record-level, secondary, expression, bloom, bucket, partition bloom), MOR snapshot-isolation fixes (worker-side use of the latest commit time from the table handle), and file-system caching integration. +- **The current arrangement does not scale.** Connector evolution must go through Trino-side review for every change, while the expertise and the source-of-truth for Hudi internals live in this project. Hudi releases cannot directly deliver improvements to Hudi users querying via Trino. + +Following alignment between the Hudi and Trino communities, the agreed direction is to split the connector into a thin Trino-side shim plus a Hudi-published artifact carrying the connector logic. This lets the Hudi project ship Trino-Hudi improvements with each Hudi release, while Trino picks them up via a one-line dependency-version bump. + +The single requirement carried over from the Trino side is that a comprehensive test suite for the connector continues to be maintained on the Trino side. This RFC documents the agreed approach and the implementation plan. + +## Abstract + +We split the Trino-Hudi connector into two Maven artifacts: + +1. **`io.trino:trino-hudi`** stays in Trino OSS (`plugin/trino-hudi`) as a thin shim — a `HudiPlugin` class that registers the `io.trino.spi.Plugin` SPI entry point — plus the test harness (smoke tests, query runners, MinIO-backed integration tests). This module mostly does not change once landed. +2. **`org.apache.hudi:hudi-trino`** is a new Hudi-published Maven artifact (regular, non-shaded JAR) containing the actual connector logic at `io.trino.plugin.hudi.*` — `HudiConnectorFactory`, `HudiConnector`, `HudiMetadata`, `HudiSplitManager`, `HudiPageSourceProvider`, all index-support strategies, the `HoodieStorage`/`HoodieIOFactory` bridges to Trino's filesystem, etc. The artifact is built against the latest Trino release's SPI; it declares `hudi-common`, `hudi-io`, etc. as transitive dependencies and Trino's `trino-spi`, `trino-filesystem`, etc. as `provided`. + +The first publication ships in **Hudi 1.3.0**. The Trino-side shim PR pins `org.apache.hudi:hudi-trino:1.3.0`. Going forward, all Trino-Hudi connector evolution happens in Hudi OSS; Trino picks up changes by bumping the dependency version. To support this integration model, **Hudi will increase its release cadence**. + +## Background + +### State of the Trino-side connector today + +`plugin/trino-hudi` in `trinodb/trino` is the baseline: it implements the standard Trino SPI (`Plugin`, `ConnectorFactory`, `Connector`, `ConnectorMetadata`, `ConnectorSplitManager`, `ConnectorPageSourceProvider`, etc.), depends on `hudi-common` and `hudi-io`, and uses Hudi's `HoodieStorage` abstraction (RFC-74) over Trino's `TrinoFileSystem`. No direct Hadoop imports. + +### State of the Hudi-side `hudi-trino-plugin` work + +A more advanced version of the connector exists in Hudi-side branches under `hudi-trino-plugin/` (same `io.trino.plugin.hudi.*` package, built against a recent Trino release). On top of the Trino-OSS baseline it adds: + +- Eight `HudiIndexSupport` strategies (column stats, partition stats, record-level, secondary, expression, bloom, bucket, partition bloom) for file- and partition-level pruning via metadata tables. +- Metadata-table-driven partition discovery (async, resumable). +- MOR record-level merging via `HoodieFileGroupReader` (`HudiTrinoReaderContext`). +- Lazy commit-time on `HudiTableHandle` for snapshot-isolated MOR reads across workers. +- Background, weighted split generation; size-based split weighting; multi-reader routing (`HudiPageSource` for MOR, `HudiBaseFileOnlyPageSource` for COW/RO). +- File-system cache integration. +- HoodieStorage / HoodieIOFactory bridges over `TrinoFileSystem` (`HudiTrinoStorage`, `HudiTrinoInlineStorage`, `HudiTrinoIOFactory`). + +This is the body of code that will move into the `hudi-trino` Maven module on the Hudi side. + +### Why a "shim + Hudi-published artifact" pattern + +This pattern decouples Trino-Hudi connector evolution from the Trino-side release cycle: + +- The Hudi project can publish Trino-Hudi improvements with each Hudi release, without waiting for Trino-side reviews of every change. +- The Trino-side surface shrinks to a stable plugin-registration shim, so Trino-side review burden is minimal — typically a one-line version bump per Hudi release. +- All Hudi-Trino integration code (`io.trino.plugin.hudi.*`) is co-located with the Hudi core libraries it depends on. Changes that cross the Hudi-internal / connector boundary can land atomically. +- The artifact is **purpose-built for Trino** and implements Trino's SPI directly, so no intermediate adapter layer is needed between the published artifact and the Trino plugin. + +Trino's `trino-spi` is governed by `revapi-maven-plugin` (see `core/trino-spi/pom.xml`) which enforces backward compatibility on the SPI surface. This is what makes a single `hudi-trino` artifact targeting the latest Trino release viable across multiple subsequent Trino releases. + +Trino loads each plugin in an isolated `URLClassLoader`. Transitive dependencies of `hudi-trino` (Avro, Parquet, etc.) are isolated to the plugin's classloader and cannot conflict with other plugins. + +## Implementation + +### Architecture + +``` +trinodb/trino : plugin/trino-hudi (packaging = trino-plugin) + HudiPlugin.java ← thin shim: trivial Plugin SPI registration + META-INF/services/io.trino.spi.Plugin + src/test/java/... ← full Trino-side test suite + pom.xml ← depends on org.apache.hudi:hudi-trino:1.3.0 + │ + │ Maven Central + ▼ +apache/hudi : hudi-trino-plugin/ (Maven profile -Phudi-trino, + excluded from default reactor, + JDK 25 required) + io.trino.plugin.hudi.* ← all connector logic: + HudiConnectorFactory, HudiConnector, HudiMetadata, + HudiSplitManager, HudiPageSourceProvider, + cache/, file/, io/, partition/, + query/ (incl. 8 index-support strategies), + reader/, split/, stats/, storage/, util/ + src/test/java/... ← full duplicated + expanded suite + Published as: org.apache.hudi:hudi-trino:1.3.0 +``` + +### What lives where + +#### Trino-side `plugin/trino-hudi` (the shim) + +| File | Purpose | +|---|---| +| `src/main/java/io/trino/plugin/hudi/HudiPlugin.java` | Implements `io.trino.spi.Plugin`. Single method returning `new HudiConnectorFactory()` (from the `hudi-trino` artifact). ~10 lines. | +| `src/main/resources/META-INF/services/io.trino.spi.Plugin` | Service-loader pointer to `io.trino.plugin.hudi.HudiPlugin`. | +| `pom.xml` | `trino-plugin`; pins `org.apache.hudi:hudi-trino:`; SPI deps as `provided`. | +| `src/test/java/...` | All current Trino-side tests stay: `HudiQueryRunner`, `TestHudiSmokeTest`, `TestHudiMinioConnectorSmokeTest`, `TestHudiConnectorTest`, `TestHudiSharedMetastore`, `TestHudiSystemTables`, `TestHudiPlugin`, `TestHudiConfig`, plus data initializers. Required by the Trino-side test-coverage commitment. | + +#### Hudi-side `hudi-trino-plugin/` (the engine) + +Everything else from the current `hudi-trino-plugin/` work, organized exactly as it is today: + +| Subpackage | Responsibility | +|---|---| +| `io.trino.plugin.hudi` | `HudiConnectorFactory`, `HudiConnector`, `HudiMetadata`, `HudiSplitManager`, `HudiPageSourceProvider`, `HudiSplit`, `HudiTableHandle`, `HudiModule`, `HudiConfig`, `HudiSessionProperties`, `HudiTableProperties`, `HudiTransactionManager`, `HudiMetadataFactory`. | +| `.cache` | `HudiCacheKeyProvider` for file-system cache integration. | +| `.file` | `HudiBaseFile`, `HudiLogFile`, file metadata abstractions. | +| `.io` | `HudiTrinoIOFactory` (extends `HoodieIOFactory`), `HudiTrinoFileReaderFactory`, `TrinoSeekableDataInputStream`. | +| `.partition` | `HudiPartitionInfo`, `HiveHudiPartitionInfo`, `HudiPartitionInfoLoader` (async resumable task). | +| `.query` | `HudiDirectoryLister`, `HudiReadOptimizedDirectoryLister`, `HudiSnapshotDirectoryLister`; `query.index` package with 8 `HudiIndexSupport` strategies. | +| `.reader` | `HudiTrinoReaderContext extends HoodieReaderContext` for MOR record merging. | +| `.split` | `HudiSplitFactory`, `HudiBackgroundSplitLoader`, `HudiSplitSource`, `HudiSplitWeightProvider`, `SizeBasedSplitWeightProvider`. | +| `.stats` | `HudiTableStatistics`, `TableStatisticsReader`. | +| `.storage` | `HudiTrinoStorage` (extends `HoodieStorage`), `HudiTrinoInlineStorage`, `TrinoStorageConfiguration`. | +| `.util` | Serialization helpers, column synthesis, tuple-domain conversion, table-type utilities. | + +### API boundary + +The boundary between the shim and the published artifact is **Trino's SPI itself** — no intermediate API layer is introduced. + +- **Shim → artifact:** `HudiPlugin.getConnectorFactories()` returns `new HudiConnectorFactory()` defined in the artifact. Trino's runtime then calls `factory.create(catalogName, config, context)`. The `ConnectorContext` argument carries everything the artifact needs — `TypeManager`, `NodeManager`, `MetadataProvider`, `PageSorter`, `PageIndexerFactory`, `OpenTelemetry`, `Tracer`, `CatalogHandle` — without the artifact importing implementation classes. +- **Artifact → Trino:** the artifact's `HudiConnector` exposes the standard SPI providers (`ConnectorMetadata`, `ConnectorSplitManager`, `ConnectorPageSourceProvider`, etc.). Trino calls these. Classloader context is handled by the standard `ClassLoaderSafe*` wrappers (`io.trino.plugin.base.classloader.*`) — already used today. + +### Maven dependencies for `hudi-trino` + +- **`compile`:** Hudi libs (`hudi-common`, `hudi-io`, `hudi-hive-sync`, `hudi-sync-common`) and Trino libs (`trino-filesystem`, `trino-hive`, `trino-metastore`, `trino-parquet`, `trino-cache`), Guice, Airlift, Caffeine. +- **`provided`:** `trino-spi`, `slice`, Jackson, OpenTelemetry API, JOL (supplied by Trino at runtime). +- **`runtime`:** log-manager, Dropwizard metrics, OpenTelemetry SDK, `trino-hive-formats`. +- **`test`:** Trino testing libs (`trino-testing`, `trino-main`, `trino-testing-containers`, `trino-hdfs`), AssertJ, JUnit 5, Hudi test JARs. + +**Version alignment policy.** Trino versions are authoritative for shared libraries (Avro, Parquet, Jackson, Airlift). The `hudi-trino` POM pins these via `` to whatever the targeted Trino release uses. If Hudi internals need a newer version, the fix is on the Hudi side or via a Trino-version bump — never by shipping divergent classpath versions. + +### Build target on Hudi side + +Trino requires Java 25, while the rest of Hudi targets a lower Java floor. `hudi-trino-plugin` therefore lives behind a Maven profile (`-Phudi-trino`) and is **excluded from the default `mvn install` reactor**: + +```xml + + hudi-trino + + hudi-trino-plugin + + +``` + +Default build (`mvn install`) skips it; Trino-targeted build (`mvn install -Phudi-trino`) requires JDK 25. + +### CI + +Two new GitHub Actions on the Hudi side, required for any change touching `hudi-trino-plugin/**`: + +1. **`hudi-trino-ci.yml`** — runs the full test suite via `mvn verify -Phudi-trino` on JDK 25. Catches regressions before they ship in a Hudi release. +2. **`hudi-trino-compat.yml`** — nightly: pulls latest `trinodb/trino` master and latest `apache/hudi` master, builds Trino's relevant modules, then compiles `hudi-trino-plugin` against them and runs the `hudi-trino-plugin` test suite. Catches both SPI drift and behavioral incompatibilities before the next Trino release. + +On the Trino side, existing CI continues to build and test `plugin/trino-hudi`, exercising the published `hudi-trino` artifact end-to-end on every Trino PR. + +### Test strategy + +**Full test duplication.** The Trino-side smoke tests (`TestHudiSmokeTest`, `TestHudiMinioConnectorSmokeTest`, `TestHudiConnectorTest`, etc.) are mirrored on the Hudi side and additionally extended. + +- **Trino side runs them** on every Trino PR — fulfilling the Trino-side test-coverage commitment. +- **Hudi side runs them** on every Hudi PR touching `hudi-trino-plugin` — so Hudi contributors catch regressions before they ship in a Hudi release. The Hudi-side suite is also **expanded** with more granular unit tests covering split generation edge cases, all eight index-support strategies, the MOR record-merging path, lazy-commit-time snapshot isolation, and the cache-key provider. + +This duplication has a known cost — two places to update when adding tests — but is the right trade-off given: +- The Trino-side suite must remain comprehensive as agreed with the Trino community. +- Hudi-side contributors need fast feedback without waiting for a Trino-side PR cycle. + +### Risks & caveats + +- **Trino SPI drift.** A future Trino SPI change could break the pre-built `hudi-trino` artifact at runtime. Mitigation: the nightly compat CI compiles and runs the `hudi-trino-plugin` test suite against Trino master, flagging both compile-time and behavioral incompatibilities before a Trino release ships. +- **Avro / Parquet / Jackson version skew.** Resolved by policy: Trino's versions are authoritative, pinned via `` in the `hudi-trino` POM. Hudi-side fixes or Trino-version bumps adjust to it. +- **Test-infrastructure coupling.** `hudi-trino-plugin`'s test scope depends on `trino-testing`, `trino-main`, etc., coupling the Hudi build to Trino artifacts on Maven Central. Acceptable cost. +- **Release coordination.** A critical fix in `hudi-trino` ships only via a Hudi release. Mitigation: keep the Trino-side shim trivial so virtually all fixes can land in `hudi-trino`, and increase Hudi release cadence. +- **License / ASF process.** Cross-project releases between two ASF projects; covered by standard PMC announcements at first release. + +## Rollout/Adoption Plan + +**Step 1 — Hudi 1.3.0 publishes `hudi-trino`.** Land the `hudi-trino-plugin` work in `apache/hudi` master behind the `-Phudi-trino` profile, land the two CI workflows, then publish `org.apache.hudi:hudi-trino:1.3.0` to Maven Central as part of the 1.3.0 release. Hudi commits to a more frequent release cadence going forward: to start, a major release roughly every month, with more frequent minor releases to stabilize this module. A `hudi-trino` artifact is cut whenever there are enough improvements to ship to Trino users. Cadence is owned by the Hudi release cycle; Trino releases are not blocked on it. + +**Step 2 — Trino-side shim PR.** A small PR against `trinodb/trino` that replaces the contents of `plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/` with a single `HudiPlugin.java`, keeps `META-INF/services/io.trino.spi.Plugin` and all current tests, and adds `org.apache.hudi:hudi-trino:1.3.0` as a `compile` dependency. The PR is small by design — deletes connector code, points at the published artifact — so Trino-maintainer review burden is minimal. + +**Step 3 — Steady state.** Trino-Hudi feature work and bug fixes happen on the Hudi side. Each Hudi release publishes a new `hudi-trino` artifact. Trino picks up changes by bumping the pinned version — a one-line PR per release. Because Trino pins `hudi-trino` as a compile-time dependency, the `hudi-trino` ↔ Trino release mapping is **one-to-one**: each `hudi-trino` version is the version that ships embedded in a specific Trino release. This one-to-one mapping makes feature support and bug-fix tracking easy to reason about for both users and maintainers. Hudi publishes and maintains this version-compatibility table on the Hudi website (hudi.apache.org), and the Trino-side `plugin/trino-hudi` README links to it as the single source of truth. + +**Impact on existing users.** No behavioral change: same `HudiPlugin` registration, same catalog config. The first Trino release picking up `hudi-trino:1.3.0` gains the features that the previously-stalled PRs covered (metadata-table partition listing, index support, MOR snapshot-isolation correctness, file-system caching, advanced split generation). No migration tools needed. + +## Test Plan + +The RFC is validated when: + +- [ ] `hudi-trino-plugin` builds and its full test suite passes on the Hudi side via `mvn verify -Phudi-trino` (JDK 25), covering smoke tests, MinIO/Alluxio caching tests, MOR/COW tests, page source tests, split-factory tests, index support tests, system-table tests, and plugin/config tests. +- [ ] The `hudi-trino-compat.yml` workflow compiles `hudi-trino-plugin` against `trinodb/trino` master successfully. +- [ ] `org.apache.hudi:hudi-trino:1.3.0` is published to Maven Central. +- [ ] The Trino-side shim PR is green: Trino's CI for `plugin/trino-hudi` passes against the published `1.3.0` artifact, including MinIO/S3 integration and plugin-loading tests. +- [ ] At least one subsequent Hudi-side patch release exercises the "bump version in Trino" steady-state flow end-to-end. From aa5344634ff2e3c3c71d03227a2826ae68842457 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Wed, 27 May 2026 13:04:29 +0800 Subject: [PATCH 029/255] feat(flink): add lance format for Flink append only table (#18741) (cherry picked from commit 1bf6b44fcd1f0b682d2c246b620a0413f2ef432a) --- .../HoodieBloomFilterRowDataWriteSupport.java | 38 +++ .../row/HoodieFlinkLanceArrowUtils.java | 305 ++++++++++++++++++ .../row/HoodieRowDataCreateHandle.java | 2 +- .../row/HoodieRowDataFileWriterFactory.java | 55 ++++ .../storage/row/HoodieRowDataLanceWriter.java | 167 ++++++++++ .../row/HoodieRowDataParquetWriteSupport.java | 12 - .../row/TestHoodieFlinkLanceArrowUtils.java | 95 ++++++ .../apache/hudi/table/HoodieTableFactory.java | 46 ++- .../format/FlinkRowDataReaderContext.java | 21 +- .../HoodieRowDataFileReaderFactory.java | 6 + .../format/HoodieRowDataLanceReader.java | 301 +++++++++++++++++ .../format/cow/CopyOnWriteInputFormat.java | 76 +++-- .../org/apache/hudi/util/StreamerUtil.java | 19 ++ .../hudi/table/ITTestHoodieDataSource.java | 36 +-- .../TestHoodieFileGroupReaderOnFlink.java | 8 + .../hudi/table/TestHoodieTableFactory.java | 62 +++- .../hudi/table/catalog/TestHoodieCatalog.java | 30 ++ .../format/TestFlinkRowDataReaderContext.java | 10 - 18 files changed, 1191 insertions(+), 98 deletions(-) create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieBloomFilterRowDataWriteSupport.java create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkLanceArrowUtils.java create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataLanceWriter.java create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieFlinkLanceArrowUtils.java create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieBloomFilterRowDataWriteSupport.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieBloomFilterRowDataWriteSupport.java new file mode 100644 index 0000000000000..36d9e28be3e23 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieBloomFilterRowDataWriteSupport.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage.row; + +import org.apache.hudi.avro.HoodieBloomFilterWriteSupport; +import org.apache.hudi.common.bloom.BloomFilter; +import org.apache.hudi.common.util.StringUtils; + +/** + * Bloom-filter footer support for Flink RowData base-file writers. + */ +class HoodieBloomFilterRowDataWriteSupport extends HoodieBloomFilterWriteSupport { + + HoodieBloomFilterRowDataWriteSupport(BloomFilter bloomFilter) { + super(bloomFilter); + } + + @Override + protected byte[] getUTF8Bytes(String key) { + return StringUtils.getUTF8Bytes(key); + } +} diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkLanceArrowUtils.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkLanceArrowUtils.java new file mode 100644 index 0000000000000..07171f615b46a --- /dev/null +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkLanceArrowUtils.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage.row; + +import org.apache.hudi.exception.HoodieNotSupportedException; + +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.DateType; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.DoubleType; +import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.SmallIntType; +import org.apache.flink.table.types.logical.TimeType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarBinaryType; +import org.apache.flink.table.types.logical.VarCharType; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getPrecision; + +/** + * Primitive RowData/Arrow conversion helpers for Flink Lance base files. + */ +public final class HoodieFlinkLanceArrowUtils { + + private HoodieFlinkLanceArrowUtils() { + } + + public static Schema toArrowSchema(RowType rowType) { + List fields = new ArrayList<>(rowType.getFieldCount()); + for (RowType.RowField field : rowType.getFields()) { + fields.add(toArrowField(field.getName(), field.getType())); + } + return new Schema(fields); + } + + public static RowType toRowType(Schema schema) { + List fields = new ArrayList<>(schema.getFields().size()); + for (Field field : schema.getFields()) { + fields.add(new RowType.RowField(field.getName(), toLogicalType(field.getType()))); + } + return new RowType(fields); + } + + public static RowData toRowData(RowType rowType, List vectors, int rowId) { + GenericRowData rowData = new GenericRowData(vectors.size()); + for (int i = 0; i < vectors.size(); i++) { + FieldVector vector = vectors.get(i); + if (vector.isNull(rowId)) { + rowData.setField(i, null); + } else { + rowData.setField(i, readValue(rowType.getTypeAt(i), vector, rowId)); + } + } + return rowData; + } + + public static void writeValue(LogicalType type, FieldVector vector, int rowId, RowData rowData, int ordinal) { + writeValue(type, vector, rowId, rowData, ordinal, true); + } + + public static void writeValue(LogicalType type, FieldVector vector, int rowId, RowData rowData, int ordinal, boolean utcTimestamp) { + if (rowData.isNullAt(ordinal)) { + vector.setNull(rowId); + return; + } + switch (type.getTypeRoot()) { + case BOOLEAN: + ((BitVector) vector).setSafe(rowId, rowData.getBoolean(ordinal) ? 1 : 0); + return; + case TINYINT: + ((TinyIntVector) vector).setSafe(rowId, rowData.getByte(ordinal)); + return; + case SMALLINT: + ((SmallIntVector) vector).setSafe(rowId, rowData.getShort(ordinal)); + return; + case INTEGER: + ((IntVector) vector).setSafe(rowId, rowData.getInt(ordinal)); + return; + case DATE: + ((DateDayVector) vector).setSafe(rowId, rowData.getInt(ordinal)); + return; + case TIME_WITHOUT_TIME_ZONE: + ((TimeMilliVector) vector).setSafe(rowId, rowData.getInt(ordinal)); + return; + case BIGINT: + ((BigIntVector) vector).setSafe(rowId, rowData.getLong(ordinal)); + return; + case FLOAT: + ((Float4Vector) vector).setSafe(rowId, rowData.getFloat(ordinal)); + return; + case DOUBLE: + ((Float8Vector) vector).setSafe(rowId, rowData.getDouble(ordinal)); + return; + case CHAR: + case VARCHAR: + ((VarCharVector) vector).setSafe(rowId, rowData.getString(ordinal).toBytes()); + return; + case BINARY: + case VARBINARY: + ((VarBinaryVector) vector).setSafe(rowId, rowData.getBinary(ordinal)); + return; + case DECIMAL: + DecimalType decimalType = (DecimalType) type; + DecimalData decimal = rowData.getDecimal(ordinal, decimalType.getPrecision(), decimalType.getScale()); + ((DecimalVector) vector).setSafe(rowId, decimal.toBigDecimal()); + return; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + TimestampData timestamp = rowData.getTimestamp(ordinal, getPrecision(type)); + long micros = timestampToMicros(timestamp, getPrecision(type), utcTimestamp); + ((TimeStampMicroVector) vector).setSafe(rowId, micros); + return; + default: + throw unsupported(type); + } + } + + private static Object readValue(LogicalType type, ValueVector vector, int rowId) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return ((BitVector) vector).get(rowId) == 1; + case TINYINT: + return ((TinyIntVector) vector).get(rowId); + case SMALLINT: + return ((SmallIntVector) vector).get(rowId); + case INTEGER: + return ((IntVector) vector).get(rowId); + case DATE: + return ((DateDayVector) vector).get(rowId); + case TIME_WITHOUT_TIME_ZONE: + return ((TimeMilliVector) vector).get(rowId); + case BIGINT: + return ((BigIntVector) vector).get(rowId); + case FLOAT: + return ((Float4Vector) vector).get(rowId); + case DOUBLE: + return ((Float8Vector) vector).get(rowId); + case CHAR: + case VARCHAR: + return StringData.fromBytes(((VarCharVector) vector).get(rowId)); + case BINARY: + case VARBINARY: + return ((VarBinaryVector) vector).get(rowId); + case DECIMAL: + DecimalType decimalType = (DecimalType) type; + BigDecimal decimal = ((DecimalVector) vector).getObject(rowId); + return DecimalData.fromBigDecimal(decimal, decimalType.getPrecision(), decimalType.getScale()); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + long micros = ((TimeStampMicroVector) vector).get(rowId); + return TimestampData.fromEpochMillis(Math.floorDiv(micros, 1000L), (int) Math.floorMod(micros, 1000L) * 1000); + default: + throw unsupported(type); + } + } + + private static Field toArrowField(String name, LogicalType type) { + return new Field(name, FieldType.nullable(toArrowType(type)), Collections.emptyList()); + } + + private static ArrowType toArrowType(LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return ArrowType.Bool.INSTANCE; + case TINYINT: + return new ArrowType.Int(8, true); + case SMALLINT: + return new ArrowType.Int(16, true); + case INTEGER: + return new ArrowType.Int(32, true); + case BIGINT: + return new ArrowType.Int(64, true); + case FLOAT: + return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + case DOUBLE: + return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + case CHAR: + case VARCHAR: + return ArrowType.Utf8.INSTANCE; + case BINARY: + case VARBINARY: + return ArrowType.Binary.INSTANCE; + case DATE: + return new ArrowType.Date(DateUnit.DAY); + case TIME_WITHOUT_TIME_ZONE: + return new ArrowType.Time(TimeUnit.MILLISECOND, 32); + case DECIMAL: + DecimalType decimalType = (DecimalType) type; + return new ArrowType.Decimal(decimalType.getPrecision(), decimalType.getScale(), 128); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); + default: + throw unsupported(type); + } + } + + private static LogicalType toLogicalType(ArrowType arrowType) { + if (arrowType instanceof ArrowType.Bool) { + return new BooleanType(); + } else if (arrowType instanceof ArrowType.Int) { + ArrowType.Int intType = (ArrowType.Int) arrowType; + switch (intType.getBitWidth()) { + case 8: + return new TinyIntType(); + case 16: + return new SmallIntType(); + case 32: + return new IntType(); + case 64: + return new BigIntType(); + default: + throw new HoodieNotSupportedException("Unsupported Arrow int width for Lance Flink reader: " + intType.getBitWidth()); + } + } else if (arrowType instanceof ArrowType.FloatingPoint) { + ArrowType.FloatingPoint fp = (ArrowType.FloatingPoint) arrowType; + return fp.getPrecision() == FloatingPointPrecision.SINGLE + ? new FloatType() + : new DoubleType(); + } else if (arrowType instanceof ArrowType.Utf8) { + return new VarCharType(); + } else if (arrowType instanceof ArrowType.Binary) { + return new VarBinaryType(); + } else if (arrowType instanceof ArrowType.Date) { + return new DateType(); + } else if (arrowType instanceof ArrowType.Time) { + return new TimeType(); + } else if (arrowType instanceof ArrowType.Decimal) { + ArrowType.Decimal decimal = (ArrowType.Decimal) arrowType; + return new DecimalType(decimal.getPrecision(), decimal.getScale()); + } else if (arrowType instanceof ArrowType.Timestamp) { + ArrowType.Timestamp timestamp = (ArrowType.Timestamp) arrowType; + return timestamp.getTimezone() == null + ? new TimestampType(6) + : new LocalZonedTimestampType(6); + } + throw new HoodieNotSupportedException("Unsupported Arrow type for Lance Flink reader: " + arrowType); + } + + private static long timestampToMicros(TimestampData timestampData, int precision, boolean utcTimestamp) { + long millis = utcTimestamp ? timestampData.getMillisecond() : timestampData.toTimestamp().getTime(); + return precision > 3 && utcTimestamp + ? millis * 1000L + timestampData.getNanoOfMillisecond() / 1000L + : millis * 1000L; + } + + private static HoodieNotSupportedException unsupported(LogicalType type) { + return new HoodieNotSupportedException("Flink Lance base-file support currently supports primitive append-only columns; unsupported type: " + type); + } +} diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java index 45333cf4b5dbe..dbccd3975012d 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java @@ -294,6 +294,6 @@ protected HoodieRowDataFileWriter createNewFileWriter( throws IOException { StoragePath storagePath = new StoragePath(path.toUri()); return (HoodieRowDataFileWriter) new HoodieRowDataFileWriterFactory(hoodieTable.getStorage()) - .newParquetFileWriter(instantTime, storagePath, config, rowType, hoodieTable.getTaskContextSupplier()); + .getFileWriter(instantTime, storagePath, config, rowType, hoodieTable.getTaskContextSupplier()); } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java index be3242164a40e..49671d37f1084 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java @@ -23,8 +23,10 @@ import org.apache.hudi.common.config.HoodieParquetConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; @@ -35,6 +37,7 @@ import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; +import org.apache.hudi.util.HoodieSchemaConverter; import org.apache.hudi.util.RowDataQueryContexts; import org.apache.flink.table.types.logical.RowType; @@ -44,6 +47,10 @@ import java.io.IOException; import java.io.OutputStream; +import static org.apache.hudi.common.model.HoodieFileFormat.HFILE; +import static org.apache.hudi.common.model.HoodieFileFormat.LANCE; +import static org.apache.hudi.common.model.HoodieFileFormat.ORC; +import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; import static org.apache.hudi.common.util.ParquetUtils.getCompressionCodecName; /** @@ -55,6 +62,30 @@ public HoodieRowDataFileWriterFactory(HoodieStorage storage) { super(storage); } + public HoodieFileWriter getFileWriter(String instantTime, StoragePath storagePath, HoodieWriteConfig config, RowType rowType, + TaskContextSupplier taskContextSupplier) throws IOException { + final String extension = FSUtils.getFileExtension(storagePath.getName()); + return getFileWriterByFormat(extension, instantTime, storagePath, config, rowType, taskContextSupplier); + } + + private HoodieFileWriter getFileWriterByFormat( + String extension, String instantTime, StoragePath path, HoodieConfig config, RowType rowType, + TaskContextSupplier taskContextSupplier) throws IOException { + if (PARQUET.getFileExtension().equals(extension)) { + return newParquetFileWriter(instantTime, path, config, rowType, taskContextSupplier); + } + if (HFILE.getFileExtension().equals(extension)) { + return newHFileFileWriter(instantTime, path, config, HoodieSchemaConverter.convertToSchema(rowType), taskContextSupplier); + } + if (ORC.getFileExtension().equals(extension)) { + return newOrcFileWriter(instantTime, path, config, HoodieSchemaConverter.convertToSchema(rowType), taskContextSupplier); + } + if (LANCE.getFileExtension().equals(extension)) { + return newLanceFileWriter(instantTime, path, config, rowType, taskContextSupplier); + } + throw new UnsupportedOperationException(extension + " format not supported yet."); + } + /** * Create a parquet writer on a given OutputStream. * @@ -136,6 +167,30 @@ public HoodieFileWriter newParquetFileWriter( instantTime, taskContextSupplier, populateMetaFields, withOperation); } + public HoodieFileWriter newLanceFileWriter( + String instantTime, + StoragePath path, + HoodieConfig config, + RowType rowType, + TaskContextSupplier taskContextSupplier) { + boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); + boolean withOperation = config.getBooleanOrDefault(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD); + Option bloomFilter = enableBloomFilter(populateMetaFields, config) + ? Option.of(createBloomFilter(config)) : Option.empty(); + return new HoodieRowDataLanceWriter( + path, + rowType, + instantTime, + taskContextSupplier, + bloomFilter, + config.getLongOrDefault(HoodieStorageConfig.LANCE_MAX_FILE_SIZE), + config.getLongOrDefault(HoodieStorageConfig.LANCE_WRITE_ALLOCATOR_SIZE_BYTES), + config.getLongOrDefault(HoodieStorageConfig.LANCE_WRITE_FLUSH_BYTE_WATERMARK), + config.getBooleanOrDefault(HoodieStorageConfig.WRITE_UTC_TIMEZONE), + populateMetaFields, + withOperation); + } + private static HoodieParquetConfig getParquetConfig( HoodieConfig config, HoodieRowDataParquetWriteSupport writeSupport) { return new HoodieParquetConfig<>( diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataLanceWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataLanceWriter.java new file mode 100644 index 0000000000000..f4388c23fc437 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataLanceWriter.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage.row; + +import org.apache.hudi.client.model.HoodieRowDataCreation; +import org.apache.hudi.common.bloom.BloomFilter; +import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.io.lance.HoodieBaseLanceWriter; +import org.apache.hudi.storage.StoragePath; + +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import java.io.IOException; +import java.util.function.Function; + +/** + * Lance writer for Flink {@link RowData} append-only base files. + */ +public class HoodieRowDataLanceWriter extends HoodieBaseLanceWriter + implements HoodieRowDataFileWriter { + + private static final long MIN_RECORDS_FOR_SIZE_CHECK = 100L; + private static final long MAX_RECORDS_FOR_SIZE_CHECK = 10000L; + + private final RowType rowType; + private final Schema arrowSchema; + private final String fileName; + private final String instantTime; + private final long maxFileSize; + private final boolean utcTimestamp; + private final boolean populateMetaFields; + private final boolean withOperation; + private final Function seqIdGenerator; + private long recordCountForNextSizeCheck = MIN_RECORDS_FOR_SIZE_CHECK; + + public HoodieRowDataLanceWriter( + StoragePath file, + RowType rowType, + String instantTime, + TaskContextSupplier taskContextSupplier, + Option bloomFilterOpt, + long maxFileSize, + long allocatorSize, + long flushByteWatermark, + boolean utcTimestamp, + boolean populateMetaFields, + boolean withOperation) { + super(file, DEFAULT_BATCH_SIZE, allocatorSize, flushByteWatermark, + bloomFilterOpt.map(HoodieBloomFilterRowDataWriteSupport::new)); + ValidationUtils.checkArgument(maxFileSize > 0, "maxFileSize must be a positive number"); + ValidationUtils.checkArgument(allocatorSize > 0, "allocatorSize must be a positive number"); + ValidationUtils.checkArgument(flushByteWatermark > 0, "flushByteWatermark must be a positive number"); + ValidationUtils.checkArgument(flushByteWatermark < allocatorSize, + "flushByteWatermark (" + flushByteWatermark + ") must be less than allocatorSize (" + + allocatorSize + ")"); + this.rowType = rowType; + this.arrowSchema = HoodieFlinkLanceArrowUtils.toArrowSchema(rowType); + this.fileName = file.getName(); + this.instantTime = instantTime; + this.maxFileSize = maxFileSize; + this.utcTimestamp = utcTimestamp; + this.populateMetaFields = populateMetaFields; + this.withOperation = withOperation; + this.seqIdGenerator = recordIndex -> { + Integer partitionId = taskContextSupplier.getPartitionIdSupplier().get(); + return HoodieRecord.generateSequenceId(instantTime, partitionId, recordIndex); + }; + } + + @Override + public boolean canWrite() { + long writtenCount = getWrittenRecordCount(); + if (writtenCount >= recordCountForNextSizeCheck) { + long dataSize = getDataSize(); + long avgRecordSize = Math.max(dataSize / writtenCount, 1); + if (dataSize > (maxFileSize - avgRecordSize * 2)) { + return false; + } + recordCountForNextSizeCheck = writtenCount + Math.min( + Math.max(MIN_RECORDS_FOR_SIZE_CHECK, (maxFileSize / avgRecordSize - writtenCount) / 2), + MAX_RECORDS_FOR_SIZE_CHECK); + } + return true; + } + + @Override + public void writeRow(String key, RowData row) throws IOException { + bloomFilterWriteSupportOpt.ifPresent(bloomFilterWriteSupport -> bloomFilterWriteSupport.addKey(key)); + super.write(row); + } + + @Override + public void writeRowWithMetaData(HoodieKey key, RowData row) throws IOException { + if (populateMetaFields) { + RowData rowWithMeta = updateRecordMetadata(row, key, getWrittenRecordCount()); + writeRow(key.getRecordKey(), rowWithMeta); + } else { + writeRow(key.getRecordKey(), row); + } + } + + @Override + protected ArrowWriter createArrowWriter(VectorSchemaRoot root) { + return new RowDataArrowWriter(root); + } + + @Override + protected Schema getArrowSchema() { + return arrowSchema; + } + + private class RowDataArrowWriter implements ArrowWriter { + private final VectorSchemaRoot root; + private int rowId; + + private RowDataArrowWriter(VectorSchemaRoot root) { + this.root = root; + } + + @Override + public void write(RowData row) { + for (int i = 0; i < rowType.getFieldCount(); i++) { + HoodieFlinkLanceArrowUtils.writeValue(rowType.getTypeAt(i), root.getVector(i), rowId, row, i, utcTimestamp); + } + rowId++; + } + + @Override + public void finishBatch() { + root.getFieldVectors().forEach(vector -> vector.setValueCount(rowId)); + root.setRowCount(rowId); + } + + @Override + public void reset() { + rowId = 0; + } + } + + private RowData updateRecordMetadata(RowData row, HoodieKey key, long recordCount) { + return HoodieRowDataCreation.create(instantTime, seqIdGenerator.apply(recordCount), + key.getRecordKey(), key.getPartitionPath(), fileName, row, withOperation, true); + } +} diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java index b2b1d9c058cf4..8dfb06872e9c3 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java @@ -21,7 +21,6 @@ import org.apache.hudi.avro.HoodieBloomFilterWriteSupport; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.StringUtils; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; @@ -61,15 +60,4 @@ public void add(String recordKey) { this.bloomFilterWriteSupportOpt.ifPresent(bloomFilterWriteSupport -> bloomFilterWriteSupport.addKey(recordKey)); } - - private static class HoodieBloomFilterRowDataWriteSupport extends HoodieBloomFilterWriteSupport { - public HoodieBloomFilterRowDataWriteSupport(BloomFilter bloomFilter) { - super(bloomFilter); - } - - @Override - protected byte[] getUTF8Bytes(String key) { - return StringUtils.getUTF8Bytes(key); - } - } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieFlinkLanceArrowUtils.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieFlinkLanceArrowUtils.java new file mode 100644 index 0000000000000..2a9ecc5b01450 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieFlinkLanceArrowUtils.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage.row; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * Tests for {@link HoodieFlinkLanceArrowUtils}. + */ +public class TestHoodieFlinkLanceArrowUtils { + + @Test + public void testTimestampSchemaRoundTripPreservesLocalTimezone() { + RowType rowType = RowType.of( + new LogicalType[] {new TimestampType(6), new LocalZonedTimestampType(6)}, + new String[] {"timestamp", "local_timestamp"}); + + RowType roundTripped = HoodieFlinkLanceArrowUtils.toRowType( + HoodieFlinkLanceArrowUtils.toArrowSchema(rowType)); + + assertInstanceOf(TimestampType.class, roundTripped.getTypeAt(0)); + assertInstanceOf(LocalZonedTimestampType.class, roundTripped.getTypeAt(1)); + } + + @Test + public void testTimestampWriteHonorsUtcTimestampFlag() { + TimestampData timestampData = TimestampData.fromEpochMillis(1234L, 567000); + GenericRowData rowData = GenericRowData.of(timestampData); + + try (BufferAllocator allocator = new RootAllocator(); + TimeStampMicroVector vector = new TimeStampMicroVector( + "ts", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + allocator)) { + HoodieFlinkLanceArrowUtils.writeValue(new TimestampType(6), vector, 0, rowData, 0, true); + assertEquals(1234567L, vector.get(0)); + + HoodieFlinkLanceArrowUtils.writeValue(new TimestampType(6), vector, 1, rowData, 0, false); + assertEquals(timestampData.toTimestamp().getTime() * 1000L, vector.get(1)); + } + } + + @Test + public void testTimestampReadNormalizesPreEpochMicros() { + try (BufferAllocator allocator = new RootAllocator(); + TimeStampMicroVector vector = new TimeStampMicroVector( + "ts", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + allocator)) { + vector.setSafe(0, -1_234_567L); + vector.setValueCount(1); + + RowData rowData = HoodieFlinkLanceArrowUtils.toRowData( + RowType.of(new LogicalType[] {new TimestampType(6)}, new String[] {"ts"}), + Collections.singletonList(vector), + 0); + + assertEquals(TimestampData.fromEpochMillis(-1235L, 433000), rowData.getTimestamp(0, 6)); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java index 2f2ada47731a0..ae8dbb1e963b8 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java @@ -18,6 +18,7 @@ package org.apache.hudi.table; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieTableType; @@ -86,9 +87,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { StoragePath path = new StoragePath(conf.getOptional(FlinkOptions.PATH).orElseThrow(() -> new ValidationException("Option [path] should not be empty."))); setupTableOptions(conf.get(FlinkOptions.PATH), conf); - checkBaseFileFormat(conf); ResolvedSchema schema = context.getCatalogTable().getResolvedSchema(); setupConfOptions(conf, context.getObjectIdentifier(), context.getCatalogTable(), schema); + checkBaseFileFormatForRead(conf, schema); return new HoodieTableSource( SerializableSchema.create(schema), path, @@ -116,11 +117,6 @@ public DynamicTableSink createDynamicTableSink(Context context) { private void setupTableOptions(String basePath, Configuration conf) { StreamerUtil.getTableConfig(basePath, HadoopConfigurations.getHadoopConf(conf)) .ifPresent(tableConfig -> { - // Guard: reject Lance from existing table config (hoodie.properties); checkBaseFileFormat() handles user-supplied config separately - if (tableConfig.contains(HoodieTableConfig.BASE_FILE_FORMAT) - && HoodieFileFormat.LANCE.name().equalsIgnoreCase(tableConfig.getString(HoodieTableConfig.BASE_FILE_FORMAT))) { - throw new HoodieValidationException(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG); - } if (tableConfig.contains(HoodieTableConfig.RECORDKEY_FIELDS) && !conf.contains(FlinkOptions.RECORD_KEY_FIELD)) { conf.set(FlinkOptions.RECORD_KEY_FIELD, tableConfig.getString(HoodieTableConfig.RECORDKEY_FIELDS)); @@ -177,8 +173,8 @@ public Set> optionalOptions() { * @param schema The table schema */ private void sanityCheck(Configuration conf, ResolvedSchema schema) { - checkBaseFileFormat(conf); checkTableType(conf); + checkBaseFileFormatForWrite(conf, schema); checkIndexType(conf); if (!OptionsResolver.isAppendMode(conf)) { @@ -220,15 +216,41 @@ private void checkIndexType(Configuration conf) { } /** - * Validate the base file format. Lance is only supported with the Spark engine. + * Validate the base file format. Flink Lance support is scoped to append-only COW tables. */ - private void checkBaseFileFormat(Configuration conf) { - String baseFileFormat = conf.getString(HoodieTableConfig.BASE_FILE_FORMAT.key(), null); - if (baseFileFormat != null && HoodieFileFormat.LANCE.name().equalsIgnoreCase(baseFileFormat)) { - throw new HoodieValidationException(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG); + private void checkBaseFileFormatForRead(Configuration conf, ResolvedSchema schema) { + checkLanceBaseFileFormat(conf, schema); + } + + private void checkBaseFileFormatForWrite(Configuration conf, ResolvedSchema schema) { + checkLanceBaseFileFormat(conf, schema); + if (isLanceBaseFileFormat(conf) && !OptionsResolver.isAppendMode(conf)) { + throw new HoodieValidationException("Flink Lance base-file writes require append-only INSERT mode. Set '" + + FlinkOptions.OPERATION.key() + "' = 'insert'."); + } + } + + private void checkLanceBaseFileFormat(Configuration conf, ResolvedSchema schema) { + if (!isLanceBaseFileFormat(conf)) { + return; + } + if (conf.containsKey(FlinkOptions.RECORD_KEY_FIELD.key()) || schema.getPrimaryKey().isPresent()) { + throw new HoodieValidationException("Flink Lance base-file support is only available for append-only tables without primary keys."); + } + if (OptionsResolver.isMorTable(conf)) { + throw new HoodieValidationException("Flink Lance base-file support is only available for COPY_ON_WRITE append-only tables."); + } + if (OptionsResolver.isSchemaEvolutionEnabled(conf)) { + throw new HoodieValidationException("Flink Lance base-file support does not support schema evolution. Set '" + + HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key() + "' = 'false'."); } } + private boolean isLanceBaseFileFormat(Configuration conf) { + String baseFileFormat = conf.getString(HoodieTableConfig.BASE_FILE_FORMAT.key(), null); + return baseFileFormat != null && HoodieFileFormat.LANCE.name().equalsIgnoreCase(baseFileFormat); + } + /** * Validate the table type. */ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java index 2015244e24ceb..130f94e59bf14 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java @@ -39,6 +39,7 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieValidationException; import org.apache.hudi.io.storage.HoodieIOFactory; import org.apache.hudi.source.ExpressionPredicates; @@ -98,18 +99,30 @@ public ClosableIterator getFileRecordIterator( HoodieSchema dataSchema, HoodieSchema requiredSchema, HoodieStorage storage) throws IOException { - if (filePath.toString().endsWith(HoodieFileFormat.LANCE.getFileExtension())) { - throw new UnsupportedOperationException(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG); - } boolean isLogFile = FSUtils.isLogFile(filePath); // disable schema evolution in fileReader if it's log file, since schema evolution for log file is handled in `FileGroupRecordBuffer` InternalSchemaManager schemaManager = isLogFile ? InternalSchemaManager.DISABLED : internalSchemaManager.get(); + if (filePath.getName().endsWith(HoodieFileFormat.LANCE.getFileExtension())) { + if (schemaManager != InternalSchemaManager.DISABLED) { + throw new HoodieValidationException("Flink Lance base-file support does not support schema evolution."); + } + HoodieRowDataLanceReader rowDataLanceReader = + (HoodieRowDataLanceReader) HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.FLINK) + .getFileReader(tableConfig, filePath, HoodieFileFormat.LANCE, Option.empty()); + try { + return rowDataLanceReader.getRowDataIterator(RowDataQueryContexts.fromSchema(requiredSchema).getRowType(), requiredSchema); + } catch (RuntimeException e) { + rowDataLanceReader.close(); + throw new HoodieException("Failed to get iterator from lance reader", e); + } + } + DataType rowType = RowDataQueryContexts.fromSchema(dataSchema).getRowType(); HoodieRowDataParquetReader rowDataParquetReader = (HoodieRowDataParquetReader) HoodieIOFactory.getIOFactory(storage) .getReaderFactory(HoodieRecord.HoodieRecordType.FLINK) .getFileReader(tableConfig, filePath, HoodieFileFormat.PARQUET, Option.empty()); - DataType rowType = RowDataQueryContexts.fromSchema(dataSchema).getRowType(); return rowDataParquetReader.getRowDataIterator(schemaManager, rowType, requiredSchema, getSafePredicates(requiredSchema)); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataFileReaderFactory.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataFileReaderFactory.java index 2e7401d4caa0d..3ad64bd3972a5 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataFileReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataFileReaderFactory.java @@ -18,6 +18,7 @@ package org.apache.hudi.table.format; +import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.io.storage.HoodieFileReader; import org.apache.hudi.io.storage.HoodieFileReaderFactory; import org.apache.hudi.storage.HoodieStorage; @@ -35,4 +36,9 @@ public HoodieRowDataFileReaderFactory(HoodieStorage storage) { protected HoodieFileReader newParquetFileReader(StoragePath path) { return new HoodieRowDataParquetReader(storage, path); } + + @Override + protected HoodieFileReader newLanceFileReader(HoodieConfig hoodieConfig, StoragePath path) { + return new HoodieRowDataLanceReader(path, hoodieConfig); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java new file mode 100644 index 0000000000000..8c7564af67303 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format; + +import org.apache.hudi.client.model.HoodieFlinkRecord; +import org.apache.hudi.common.bloom.BloomFilter; +import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter; +import org.apache.hudi.common.bloom.SimpleBloomFilter; +import org.apache.hudi.common.config.HoodieConfig; +import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaUtils; +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.CloseableMappingIterator; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.io.memory.HoodieArrowAllocator; +import org.apache.hudi.io.storage.HoodieFileReader; +import org.apache.hudi.io.storage.row.HoodieFlinkLanceArrowUtils; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.util.HoodieSchemaConverter; +import org.apache.hudi.util.RowDataQueryContexts; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; +import org.lance.file.LanceFileReader; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER; + +/** + * Lance reader for Flink RowData base files. + */ +public class HoodieRowDataLanceReader implements HoodieFileReader { + + private static final int DEFAULT_BATCH_SIZE = 512; + + private final StoragePath path; + private final long dataAllocatorSize; + private final BufferAllocator metadataAllocator; + private final LanceFileReader metadataReader; + private final Schema arrowSchema; + private boolean closed; + + public HoodieRowDataLanceReader(StoragePath path, HoodieConfig hoodieConfig) { + this.path = path; + this.dataAllocatorSize = hoodieConfig.getLongOrDefault(HoodieStorageConfig.LANCE_READ_ALLOCATOR_SIZE_BYTES); + this.metadataAllocator = HoodieArrowAllocator.newChildAllocator( + getClass().getSimpleName() + "-metadata-" + path.getName(), + hoodieConfig.getLongOrDefault(HoodieStorageConfig.LANCE_READ_METADATA_ALLOCATOR_SIZE_BYTES)); + try { + this.metadataReader = LanceFileReader.open(path.toString(), metadataAllocator); + this.arrowSchema = metadataReader.schema(); + } catch (Exception e) { + close(); + throw new HoodieException("Failed to create Lance reader for: " + path, e); + } + } + + @Override + public String[] readMinMaxRecordKeys() { + Map metadata = arrowSchema.getCustomMetadata(); + if (metadata != null) { + String minKey = metadata.get(HOODIE_MIN_RECORD_KEY_FOOTER); + String maxKey = metadata.get(HOODIE_MAX_RECORD_KEY_FOOTER); + if (minKey != null && maxKey != null) { + return new String[] {minKey, maxKey}; + } + } + throw new HoodieException("Could not read min/max record key out of Lance file: " + path); + } + + @Override + public BloomFilter readBloomFilter() { + Map metadata = arrowSchema.getCustomMetadata(); + if (metadata == null || !metadata.containsKey(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY)) { + return null; + } + String bloomSer = metadata.get(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY); + String filterType = metadata.get(HOODIE_BLOOM_FILTER_TYPE_CODE); + if (filterType != null && filterType.contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) { + return new HoodieDynamicBoundedBloomFilter(bloomSer); + } + return new SimpleBloomFilter(bloomSer); + } + + @Override + public Set> filterRowKeys(Set candidateRowKeys) { + throw new HoodieException("Filtering row keys from Lance files is not supported for Flink append-only tables without primary keys: " + path); + } + + @Override + public ClosableIterator> getRecordIterator(HoodieSchema readerSchema, HoodieSchema requestedSchema) throws IOException { + ClosableIterator rowDataItr = getRowDataIterator(RowDataQueryContexts.fromSchema(requestedSchema).getRowType(), requestedSchema); + return new CloseableMappingIterator<>(rowDataItr, HoodieFlinkRecord::new); + } + + @Override + public ClosableIterator getRecordKeyIterator() throws IOException { + HoodieSchema schema = HoodieSchemaUtils.getRecordKeySchema(); + ClosableIterator rowDataItr = getRowDataIterator(RowDataQueryContexts.fromSchema(schema).getRowType(), schema); + return new CloseableMappingIterator<>(rowDataItr, rowData -> rowData.getString(0).toString()); + } + + public ClosableIterator getRowDataIterator(DataType dataType, HoodieSchema requestedSchema) { + RowType rowType = (RowType) dataType.getLogicalType(); + List columnNames = new ArrayList<>(rowType.getFieldCount()); + for (RowType.RowField field : rowType.getFields()) { + columnNames.add(field.getName()); + } + BufferAllocator allocator = HoodieArrowAllocator.newChildAllocator( + getClass().getSimpleName() + "-data-" + path.getName(), dataAllocatorSize); + LanceFileReader lanceReader = null; + ArrowReader arrowReader = null; + try { + lanceReader = LanceFileReader.open(path.toString(), allocator); + arrowReader = lanceReader.readAll(columnNames, null, DEFAULT_BATCH_SIZE); + return new LanceRowDataIterator(allocator, lanceReader, arrowReader, rowType, this); + } catch (Exception e) { + if (arrowReader != null) { + try { + arrowReader.close(); + } catch (Exception closeException) { + e.addSuppressed(closeException); + } + } + if (lanceReader != null) { + try { + lanceReader.close(); + } catch (Exception closeException) { + e.addSuppressed(closeException); + } + } + allocator.close(); + throw new HoodieException("Failed to create Lance row iterator for: " + path, e); + } + } + + @Override + public HoodieSchema getSchema() { + RowType rowType = HoodieFlinkLanceArrowUtils.toRowType(arrowSchema); + return HoodieSchemaConverter.convertToSchema(rowType); + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (metadataReader != null) { + try { + metadataReader.close(); + } catch (Exception e) { + // ignore close failure; readers surface data-path exceptions earlier + } + } + if (metadataAllocator != null) { + metadataAllocator.close(); + } + } + + @Override + public long getTotalRecords() { + try { + return metadataReader.numRows(); + } catch (Exception e) { + throw new HoodieException("Failed to read row count from Lance file: " + path, e); + } + } + + private static class LanceRowDataIterator implements ClosableIterator { + private final BufferAllocator allocator; + private final LanceFileReader lanceReader; + private final ArrowReader arrowReader; + private final RowType rowType; + private final HoodieRowDataLanceReader reader; + private VectorSchemaRoot batch; + private List orderedVectors; + private int rowId; + private boolean hasNext; + private boolean closed; + + private LanceRowDataIterator( + BufferAllocator allocator, + LanceFileReader lanceReader, + ArrowReader arrowReader, + RowType rowType, + HoodieRowDataLanceReader reader) { + this.allocator = allocator; + this.lanceReader = lanceReader; + this.arrowReader = arrowReader; + this.rowType = rowType; + this.reader = reader; + loadNextBatch(); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public RowData next() { + RowData rowData = HoodieFlinkLanceArrowUtils.toRowData(rowType, orderedVectors, rowId++); + if (rowId >= batch.getRowCount()) { + loadNextBatch(); + } + return rowData; + } + + private void loadNextBatch() { + try { + do { + hasNext = arrowReader.loadNextBatch(); + if (hasNext) { + batch = arrowReader.getVectorSchemaRoot(); + orderedVectors = orderVectors(rowType, batch.getFieldVectors()); + rowId = 0; + } + } while (hasNext && batch.getRowCount() == 0); + } catch (IOException e) { + throw new HoodieIOException("Failed to read Lance batch", e); + } + } + + private static List orderVectors(RowType rowType, List vectors) { + Map vectorsByName = new HashMap<>(); + for (FieldVector vector : vectors) { + vectorsByName.put(vector.getName(), vector); + } + List orderedVectors = new ArrayList<>(rowType.getFieldCount()); + for (RowType.RowField field : rowType.getFields()) { + FieldVector vector = vectorsByName.get(field.getName()); + if (vector == null) { + throw new HoodieException("Missing Lance column in projected batch: " + field.getName()); + } + orderedVectors.add(vector); + } + return orderedVectors; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + try { + arrowReader.close(); + } catch (Exception e) { + throw new HoodieException("Failed to close Lance Arrow reader", e); + } finally { + try { + lanceReader.close(); + } catch (Exception e) { + throw new HoodieException("Failed to close Lance reader", e); + } finally { + try { + allocator.close(); + } finally { + reader.close(); + } + } + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java index 5a2e94b833baf..9e59e1ed33447 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java @@ -18,12 +18,19 @@ package org.apache.hudi.table.format.cow; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.source.ExpressionPredicates.Predicate; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.format.FilePathUtils; +import org.apache.hudi.table.format.HoodieRowDataLanceReader; import org.apache.hudi.table.format.InternalSchemaManager; import org.apache.hudi.table.format.RecordIterators; +import org.apache.hudi.util.HoodieSchemaConverter; +import org.apache.hudi.util.StreamerUtil; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.io.FileInputFormat; @@ -33,6 +40,7 @@ import org.apache.flink.core.fs.FileInputSplit; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.utils.SerializableConfiguration; +import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.DataType; import org.apache.hadoop.conf.Configuration; @@ -116,32 +124,50 @@ public CopyOnWriteInputFormat( @Override public void open(FileInputSplit fileSplit) throws IOException { - LinkedHashMap partObjects = FilePathUtils.generatePartitionSpecs( - fileSplit.getPath().getPath(), - Arrays.asList(fullFieldNames), - Arrays.asList(fullFieldTypes), - this.partDefaultName, - this.partPathField, - this.hiveStylePartitioning - ); - - this.itr = RecordIterators.getParquetRecordIterator( - internalSchemaManager, - utcTimestamp, - true, - conf.conf(), - fullFieldNames, - fullFieldTypes, - partObjects, - selectedFields, - 2048, - fileSplit.getPath(), - fileSplit.getStart(), - fileSplit.getLength(), - predicates); + if (fileSplit.getPath().getName().endsWith(HoodieFileFormat.LANCE.getFileExtension())) { + this.itr = getLanceRecordIterator(fileSplit.getPath()); + } else { + LinkedHashMap partObjects = FilePathUtils.generatePartitionSpecs( + fileSplit.getPath().getPath(), + Arrays.asList(fullFieldNames), + Arrays.asList(fullFieldTypes), + this.partDefaultName, + this.partPathField, + this.hiveStylePartitioning + ); + this.itr = RecordIterators.getParquetRecordIterator( + internalSchemaManager, + utcTimestamp, + true, + conf.conf(), + fullFieldNames, + fullFieldTypes, + partObjects, + selectedFields, + 2048, + fileSplit.getPath(), + fileSplit.getStart(), + fileSplit.getLength(), + predicates); + } this.currentReadCount = 0L; } + private ClosableIterator getLanceRecordIterator(Path path) { + DataType selectedDataType = DataTypes.ROW(Arrays.stream(selectedFields) + .mapToObj(i -> DataTypes.FIELD(fullFieldNames[i], fullFieldTypes[i])) + .toArray(DataTypes.Field[]::new)) + .bridgedTo(RowData.class); + HoodieSchema requestedSchema = HoodieSchemaConverter.convertToSchema(selectedDataType.getLogicalType()); + HoodieRowDataLanceReader reader = new HoodieRowDataLanceReader(new StoragePath(path.toString()), StreamerUtil.getLanceReadConfig(conf.conf())); + try { + return reader.getRowDataIterator(selectedDataType, requestedSchema); + } catch (RuntimeException e) { + reader.close(); + throw new HoodieException("Failed to get iterator from lance reader", e); + } + } + @Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { if (minNumSplits < 1) { @@ -379,6 +405,10 @@ private int getBlockIndexForPosition(BlockLocation[] blocks, long offset, long h } private boolean testForUnsplittable(FileStatus pathFile) { + if (pathFile.getPath().getName().endsWith(HoodieFileFormat.LANCE.getFileExtension())) { + unsplittable = true; + return true; + } if (getInflaterInputStreamFactory(pathFile.getPath()) != null) { unsplittable = true; return true; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index f88d1a76a3251..c17e0528e4586 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -25,7 +25,9 @@ import org.apache.hudi.client.model.PartialUpdateFlinkRecordMerger; import org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider; import org.apache.hudi.common.config.DFSPropertiesConfiguration; +import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.HoodieTimeGeneratorConfig; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.config.TypedProperties; @@ -275,6 +277,22 @@ public static TypedProperties flinkConf2TypedProperties(Configuration conf) { return properties; } + /** + * Builds a Lance read config from storage options carried in the Hadoop configuration. + */ + public static HoodieConfig getLanceReadConfig(org.apache.hadoop.conf.Configuration conf) { + HoodieConfig hoodieConfig = new HoodieConfig(); + String dataAllocatorSize = conf.get(HoodieStorageConfig.LANCE_READ_ALLOCATOR_SIZE_BYTES.key()); + if (dataAllocatorSize != null) { + hoodieConfig.setValue(HoodieStorageConfig.LANCE_READ_ALLOCATOR_SIZE_BYTES, dataAllocatorSize); + } + String metadataAllocatorSize = conf.get(HoodieStorageConfig.LANCE_READ_METADATA_ALLOCATOR_SIZE_BYTES.key()); + if (metadataAllocatorSize != null) { + hoodieConfig.setValue(HoodieStorageConfig.LANCE_READ_METADATA_ALLOCATOR_SIZE_BYTES, metadataAllocatorSize); + } + return hoodieConfig; + } + public static void initTableFromClientIfNecessary(Configuration conf) { // Since Flink 2.0, the adaptive execution for batch job will generate job graph incrementally // for multiple stages (FLIP-469). And the write coordinator is initialized along with write @@ -318,6 +336,7 @@ public static HoodieTableMetaClient initTableIfNotExists( .setTableName(conf.get(FlinkOptions.TABLE_NAME)) .setTableVersion(conf.get(FlinkOptions.WRITE_TABLE_VERSION)) .setTableFormat(conf.get(FlinkOptions.WRITE_TABLE_FORMAT)) + .setBaseFileFormat(conf.getString(HoodieTableConfig.BASE_FILE_FORMAT.key(), null)) .setRecordMergeMode(getMergeMode(conf)) .setRecordMergeStrategyId(getMergeStrategyId(conf)) .setPayloadClassName(getPayloadClass(conf)) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 5d6a5daa061e5..1cc5beaad4180 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -21,7 +21,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; -import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; @@ -65,7 +64,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.types.Row; import org.apache.flink.util.CollectionUtil; -import org.apache.flink.util.ExceptionUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -1360,31 +1358,27 @@ void testBatchReadEmptyTablePath() throws Exception { } @Test - void testLanceFormatRejectedByFlink() { - // Lance base file format is only supported with the Spark engine. - // Flink should reject it early with a clear error on both read and write paths. - String createLanceTable = sql("lance_t1") + void testLanceFormatAppendOnlyWriteAndRead() { + String createHoodieTable = sql("lance_t1") .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) - .options(getDefaultKeys()) + .option(FlinkOptions.OPERATION, "insert") .option("hoodie.table.base.file.format", "LANCE") .end(); + batchTableEnv.executeSql(createHoodieTable); - // Creating the table itself succeeds (DDL is just metadata registration), - // but any attempt to read or write should fail. - // Flink wraps our HoodieValidationException in its own ValidationException. - batchTableEnv.executeSql(createLanceTable); + execInsertSql(batchTableEnv, "insert into lance_t1 values " + + "('id1', 'Alice', 23, TIMESTAMP '1970-01-01 00:00:01', 'par1')," + + "('id2', 'Bob', 31, TIMESTAMP '1970-01-01 00:00:02', 'par2')"); - // Source (read) path should throw - ValidationException readEx = assertThrows(ValidationException.class, - () -> execSelectSql(batchTableEnv, "select * from lance_t1"), - "Lance format should be rejected when reading via Flink"); - assertTrue(ExceptionUtils.findThrowableWithMessage(readEx, HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG).isPresent()); + List rows = CollectionUtil.iteratorToList( + batchTableEnv.executeSql("select uuid, name, age, ts, `partition` from lance_t1").collect()); + assertRowsEquals(rows, + "[+I[id1, Alice, 23, 1970-01-01T00:00:01, par1], " + + "+I[id2, Bob, 31, 1970-01-01T00:00:02, par2]]"); - // Sink (write) path should throw - ValidationException writeEx = assertThrows(ValidationException.class, - () -> execInsertSql(batchTableEnv, "insert into lance_t1 values ('id1', 'Alice', 23, TIMESTAMP '1970-01-01 00:00:01', 'par1')"), - "Lance format should be rejected when writing via Flink"); - assertTrue(ExceptionUtils.findThrowableWithMessage(writeEx, HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG).isPresent()); + List projectedRows = CollectionUtil.iteratorToList( + batchTableEnv.executeSql("select name, uuid from lance_t1").collect()); + assertRowsEquals(projectedRows, "[+I[Alice, id1], +I[Bob, id2]]"); } @ParameterizedTest diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieFileGroupReaderOnFlink.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieFileGroupReaderOnFlink.java index 7a8ce8dcd80a4..84e1136bc2965 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieFileGroupReaderOnFlink.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieFileGroupReaderOnFlink.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.engine.HoodieReaderContext; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.schema.HoodieSchema; @@ -57,6 +58,7 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -85,6 +87,12 @@ public class TestHoodieFileGroupReaderOnFlink extends TestHoodieFileGroupReaderB private Configuration conf; private Option instantRangeOpt = Option.empty(); + @BeforeAll + public static void setUpClass() { + // add the lance format when composition type is supported + supportedFileFormats = Collections.singletonList(HoodieFileFormat.PARQUET); + } + @BeforeEach public void setup() { conf = new Configuration(); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java index f594e9575860c..6aa6f20581c3e 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java @@ -18,10 +18,10 @@ package org.apache.hudi.table; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.EventTimeAvroPayload; -import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.exception.HoodieValidationException; @@ -788,28 +788,60 @@ void testSetupWriteOptionsForSink() { } @Test - void testLanceFormatNotSupportedByFlink() { - // Lance base file format is only supported with the Spark engine. - // Both source and sink should reject it with a clear error message. - this.conf.setString("hoodie.table.base.file.format", "LANCE"); - ResolvedSchema schema = SchemaBuilder.instance() + void testLanceFormatSupportedForAppendOnlyTables() { + Configuration lanceConf = new Configuration(); + lanceConf.set(FlinkOptions.PATH, new File(tempFile, "lance").getAbsolutePath()); + lanceConf.set(FlinkOptions.TABLE_NAME, "lance_t1"); + lanceConf.set(FlinkOptions.OPERATION, "insert"); + lanceConf.setString("hoodie.table.base.file.format", "LANCE"); + ResolvedSchema appendOnlySchema = SchemaBuilder.instance() .field("f0", DataTypes.INT().notNull()) .field("f1", DataTypes.VARCHAR(20)) .field("f2", DataTypes.TIMESTAMP(3)) .field("ts", DataTypes.TIMESTAMP(3)) + .build(); + final MockContext appendOnlyContext = MockContext.getInstance(lanceConf, appendOnlySchema, "f2"); + + assertDoesNotThrow(() -> new HoodieTableFactory().createDynamicTableSink(appendOnlyContext)); + + Configuration morConf = new Configuration(lanceConf); + morConf.set(FlinkOptions.TABLE_TYPE, FlinkOptions.TABLE_TYPE_MERGE_ON_READ); + final MockContext morContext = MockContext.getInstance(morConf, appendOnlySchema, "f2"); + HoodieValidationException morEx = assertThrows(HoodieValidationException.class, + () -> new HoodieTableFactory().createDynamicTableSink(morContext)); + assertThat(morEx.getMessage(), is("Flink Lance base-file support is only available for COPY_ON_WRITE append-only tables.")); + + Configuration upsertConf = new Configuration(lanceConf); + upsertConf.set(FlinkOptions.OPERATION, "upsert"); + final MockContext upsertContext = MockContext.getInstance(upsertConf, appendOnlySchema, "f2"); + HoodieValidationException operationEx = assertThrows(HoodieValidationException.class, + () -> new HoodieTableFactory().createDynamicTableSink(upsertContext)); + assertThat(operationEx.getMessage(), is("Flink Lance base-file writes require append-only INSERT mode. Set '" + + FlinkOptions.OPERATION.key() + "' = 'insert'.")); + + Configuration schemaEvolutionConf = new Configuration(lanceConf); + schemaEvolutionConf.setString(HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key(), "true"); + final MockContext schemaEvolutionContext = MockContext.getInstance(schemaEvolutionConf, appendOnlySchema, "f2"); + HoodieValidationException schemaEvolutionEx = assertThrows(HoodieValidationException.class, + () -> new HoodieTableFactory().createDynamicTableSink(schemaEvolutionContext)); + assertThat(schemaEvolutionEx.getMessage(), is("Flink Lance base-file support does not support schema evolution. Set '" + + HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key() + "' = 'false'.")); + + ResolvedSchema primaryKeySchema = SchemaBuilder.instance() + .field("f0", DataTypes.INT().notNull()) + .field("f1", DataTypes.VARCHAR(20)) .primaryKey("f0") .build(); - final MockContext context = MockContext.getInstance(this.conf, schema, "f2"); - - // Source path should throw - HoodieValidationException sourceEx = assertThrows(HoodieValidationException.class, - () -> new HoodieTableFactory().createDynamicTableSource(context)); - assertThat(sourceEx.getMessage(), is(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG)); + final MockContext primaryKeyContext = MockContext.getInstance(lanceConf, primaryKeySchema, "f1"); + HoodieValidationException primaryKeyEx = assertThrows(HoodieValidationException.class, + () -> new HoodieTableFactory().createDynamicTableSink(primaryKeyContext)); + assertThat(primaryKeyEx.getMessage(), is("Flink Lance base-file support is only available for append-only tables without primary keys.")); - // Sink path should throw + lanceConf.set(FlinkOptions.RECORD_KEY_FIELD, "f0"); + final MockContext keyedContext = MockContext.getInstance(lanceConf, appendOnlySchema, "f2"); HoodieValidationException sinkEx = assertThrows(HoodieValidationException.class, - () -> new HoodieTableFactory().createDynamicTableSink(context)); - assertThat(sinkEx.getMessage(), is(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG)); + () -> new HoodieTableFactory().createDynamicTableSink(keyedContext)); + assertThat(sinkEx.getMessage(), is("Flink Lance base-file support is only available for append-only tables without primary keys.")); } // ------------------------------------------------------------------------- diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java index 9934d8a4e1605..379a769ec8153 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; import org.apache.hudi.common.model.PartitionBucketIndexHashingConfig; import org.apache.hudi.common.schema.HoodieSchema; @@ -380,6 +381,35 @@ public void testCreateNonAppendTableWithoutRecordKey() { assertEquals("Primary key definition is missing", exception.getMessage()); } + @Test + public void testCreateAppendOnlyLanceTableWithoutPrimaryKey() throws Exception { + ObjectPath tablePath = new ObjectPath(TEST_DEFAULT_DATABASE, "tb_lance_append_only"); + Map lanceOptions = new HashMap<>(EXPECTED_OPTIONS); + lanceOptions.put(FlinkOptions.TABLE_TYPE.key(), FlinkOptions.TABLE_TYPE_COPY_ON_WRITE); + lanceOptions.put(FlinkOptions.OPERATION.key(), "insert"); + lanceOptions.put(FlinkOptions.PRE_COMBINE.key(), "false"); + lanceOptions.put(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.LANCE.name()); + ResolvedSchema appendOnlySchema = new ResolvedSchema(CREATE_COLUMNS, Collections.emptyList(), null); + ResolvedCatalogTable lanceTable = new ResolvedCatalogTable( + CatalogUtils.createCatalogTable( + Schema.newBuilder().fromResolvedSchema(appendOnlySchema).build(), + Arrays.asList("partition"), + lanceOptions, + "test_lance_append_only"), + appendOnlySchema + ); + + catalog.createTable(tablePath, lanceTable, false); + + assertTrue(catalog.tableExists(tablePath)); + CatalogBaseTable actualTable = catalog.getTable(tablePath); + assertFalse(actualTable.getOptions().containsKey(TableOptionProperties.PK_COLUMNS)); + HoodieTableMetaClient metaClient = createMetaClient( + new HadoopStorageConfiguration(HadoopConfigurations.getHadoopConf(new Configuration())), + catalog.inferTablePath(catalogPathStr, tablePath)); + assertThat(metaClient.getTableConfig().getBaseFileFormat(), is(HoodieFileFormat.LANCE)); + } + @Test void testCreateTableWithPartitionBucketIndex() throws TableAlreadyExistException, DatabaseNotExistException, IOException { String rule = "regex"; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java index 189e297024fb4..751f63d6de75a 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java @@ -28,7 +28,6 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.source.ExpressionPredicates; import org.apache.hudi.storage.StorageConfiguration; -import org.apache.hudi.storage.StoragePath; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; @@ -44,7 +43,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -134,14 +132,6 @@ void testConstructEngineRecordWithNullUpdate() { assertTrue(result.getBoolean(2)); } - @Test - void testLanceFormatThrowsInGetFileRecordIterator() { - StoragePath lancePath = new StoragePath("/tmp/test-table/partition/file.lance"); - UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, - () -> readerContext.getFileRecordIterator(lancePath, 0, 100, SCHEMA, SCHEMA, null)); - assertEquals(HoodieFileFormat.LANCE_SPARK_ONLY_ERROR_MSG, ex.getMessage()); - } - private GenericRowData createBaseRow(int id, String name, boolean active) { return GenericRowData.of(id, StringData.fromString(name), active); } From ab60e0971f53a6378f85098899e67980fb5ffe6b Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Thu, 28 May 2026 21:30:11 +0800 Subject: [PATCH 030/255] refactor(flink): Refactor RowData writer factory to use HoodieSchema (#18873) (cherry picked from commit 588998fc719392d7fe428986eb4df56e856c8b4d) --- .../row/HoodieRowDataCreateHandle.java | 12 +++- .../row/HoodieRowDataFileWriterFactory.java | 64 ++----------------- .../row/HoodieRowDataParquetWriteSupport.java | 6 +- .../row/RowDataParquetWriteSupport.java | 6 +- ...estHoodieRowDataParquetConfigInjector.java | 11 +++- 5 files changed, 31 insertions(+), 68 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java index dbccd3975012d..e3a8445d6675a 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java @@ -35,11 +35,13 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieInsertException; +import org.apache.hudi.io.storage.HoodieFileWriterFactory; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.marker.WriteMarkers; import org.apache.hudi.table.marker.WriteMarkersFactory; +import org.apache.hudi.util.HoodieSchemaConverter; import lombok.extern.slf4j.Slf4j; import org.apache.flink.table.data.RowData; @@ -293,7 +295,13 @@ protected HoodieRowDataFileWriter createNewFileWriter( Path path, HoodieTable hoodieTable, HoodieWriteConfig config, RowType rowType, String instantTime) throws IOException { StoragePath storagePath = new StoragePath(path.toUri()); - return (HoodieRowDataFileWriter) new HoodieRowDataFileWriterFactory(hoodieTable.getStorage()) - .getFileWriter(instantTime, storagePath, config, rowType, hoodieTable.getTaskContextSupplier()); + return (HoodieRowDataFileWriter) HoodieFileWriterFactory.getFileWriter( + instantTime, + storagePath, + hoodieTable.getStorage(), + config, + HoodieSchemaConverter.convertToSchema(rowType).getNonNullType(), + hoodieTable.getTaskContextSupplier(), + HoodieRecord.HoodieRecordType.FLINK); } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java index 49671d37f1084..9fe3d6d5f3273 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java @@ -23,7 +23,6 @@ import org.apache.hudi.common.config.HoodieParquetConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.engine.TaskContextSupplier; -import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.util.Option; @@ -38,7 +37,6 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; import org.apache.hudi.util.HoodieSchemaConverter; -import org.apache.hudi.util.RowDataQueryContexts; import org.apache.flink.table.types.logical.RowType; import org.apache.hadoop.conf.Configuration; @@ -47,10 +45,6 @@ import java.io.IOException; import java.io.OutputStream; -import static org.apache.hudi.common.model.HoodieFileFormat.HFILE; -import static org.apache.hudi.common.model.HoodieFileFormat.LANCE; -import static org.apache.hudi.common.model.HoodieFileFormat.ORC; -import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; import static org.apache.hudi.common.util.ParquetUtils.getCompressionCodecName; /** @@ -62,30 +56,6 @@ public HoodieRowDataFileWriterFactory(HoodieStorage storage) { super(storage); } - public HoodieFileWriter getFileWriter(String instantTime, StoragePath storagePath, HoodieWriteConfig config, RowType rowType, - TaskContextSupplier taskContextSupplier) throws IOException { - final String extension = FSUtils.getFileExtension(storagePath.getName()); - return getFileWriterByFormat(extension, instantTime, storagePath, config, rowType, taskContextSupplier); - } - - private HoodieFileWriter getFileWriterByFormat( - String extension, String instantTime, StoragePath path, HoodieConfig config, RowType rowType, - TaskContextSupplier taskContextSupplier) throws IOException { - if (PARQUET.getFileExtension().equals(extension)) { - return newParquetFileWriter(instantTime, path, config, rowType, taskContextSupplier); - } - if (HFILE.getFileExtension().equals(extension)) { - return newHFileFileWriter(instantTime, path, config, HoodieSchemaConverter.convertToSchema(rowType), taskContextSupplier); - } - if (ORC.getFileExtension().equals(extension)) { - return newOrcFileWriter(instantTime, path, config, HoodieSchemaConverter.convertToSchema(rowType), taskContextSupplier); - } - if (LANCE.getFileExtension().equals(extension)) { - return newLanceFileWriter(instantTime, path, config, rowType, taskContextSupplier); - } - throw new UnsupportedOperationException(extension + " format not supported yet."); - } - /** * Create a parquet writer on a given OutputStream. * @@ -100,11 +70,9 @@ protected HoodieFileWriter newParquetFileWriter( OutputStream outputStream, HoodieConfig config, HoodieSchema schema) throws IOException { - //TODO boundary to revisit in follow up to use HoodieSchema directly - final RowType rowType = (RowType) RowDataQueryContexts.fromSchema(schema).getRowType().getLogicalType(); HoodieRowDataParquetWriteSupport writeSupport = new HoodieRowDataParquetWriteSupport( - storage.getConf().unwrapAs(Configuration.class), rowType, null); + storage.getConf().unwrapAs(Configuration.class), schema, null); return new HoodieRowDataParquetOutputStreamWriter( new FSDataOutputStream(outputStream, null), writeSupport, getParquetConfig(config, writeSupport)); } @@ -127,28 +95,6 @@ public HoodieFileWriter newParquetFileWriter( HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { - //TODO boundary to revisit in follow up to use HoodieSchema directly - final RowType rowType = (RowType) RowDataQueryContexts.fromSchema(schema).getRowType().getLogicalType(); - return newParquetFileWriter(instantTime, storagePath, config, rowType, taskContextSupplier); - } - - /** - * Create a parquet RowData writer on a given storage path. - * - * @param instantTime instant time to write - * @param storagePath file storage path - * @param config hoodie configuration - * @param rowType rowType of record - * @param taskContextSupplier task context supplier - * - * @return a RowData parquet writer - */ - public HoodieFileWriter newParquetFileWriter( - String instantTime, - StoragePath storagePath, - HoodieConfig config, - RowType rowType, - TaskContextSupplier taskContextSupplier) throws IOException { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); boolean withOperation = config.getBooleanOrDefault(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD); @@ -160,23 +106,25 @@ public HoodieFileWriter newParquetFileWriter( BloomFilter filter = createBloomFilter(hoodieConfig); HoodieRowDataParquetWriteSupport writeSupport = (HoodieRowDataParquetWriteSupport) ReflectionUtils.loadClass( hoodieConfig.getStringOrDefault(HoodieStorageConfig.HOODIE_PARQUET_FLINK_ROW_DATA_WRITE_SUPPORT_CLASS), - new Class[] {Configuration.class, RowType.class, BloomFilter.class}, - conf, rowType, filter); + new Class[] {Configuration.class, HoodieSchema.class, BloomFilter.class}, + conf, schema, filter); return new HoodieRowDataParquetWriter(storagePath, getParquetConfig(hoodieConfig, writeSupport), instantTime, taskContextSupplier, populateMetaFields, withOperation); } + @Override public HoodieFileWriter newLanceFileWriter( String instantTime, StoragePath path, HoodieConfig config, - RowType rowType, + HoodieSchema schema, TaskContextSupplier taskContextSupplier) { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); boolean withOperation = config.getBooleanOrDefault(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD); Option bloomFilter = enableBloomFilter(populateMetaFields, config) ? Option.of(createBloomFilter(config)) : Option.empty(); + RowType rowType = HoodieSchemaConverter.convertToRowType(schema); return new HoodieRowDataLanceWriter( path, rowType, diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java index 8dfb06872e9c3..14ed278b70a28 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java @@ -20,10 +20,10 @@ import org.apache.hudi.avro.HoodieBloomFilterWriteSupport; import org.apache.hudi.common.bloom.BloomFilter; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.Option; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.types.logical.RowType; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.hadoop.api.WriteSupport; @@ -37,8 +37,8 @@ public class HoodieRowDataParquetWriteSupport extends RowDataParquetWriteSupport private final Option> bloomFilterWriteSupportOpt; - public HoodieRowDataParquetWriteSupport(Configuration conf, RowType rowType, BloomFilter bloomFilter) { - super(rowType, conf); + public HoodieRowDataParquetWriteSupport(Configuration conf, HoodieSchema schema, BloomFilter bloomFilter) { + super(schema, conf); this.bloomFilterWriteSupportOpt = Option.ofNullable(bloomFilter) .map(HoodieBloomFilterRowDataWriteSupport::new); } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/RowDataParquetWriteSupport.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/RowDataParquetWriteSupport.java index 7315461db5006..01d2806e76335 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/RowDataParquetWriteSupport.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/RowDataParquetWriteSupport.java @@ -19,8 +19,10 @@ package org.apache.hudi.io.storage.row; import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.io.storage.row.parquet.ParquetRowDataWriter; import org.apache.hudi.io.storage.row.parquet.ParquetSchemaConverter; +import org.apache.hudi.util.HoodieSchemaConverter; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; @@ -41,9 +43,9 @@ public class RowDataParquetWriteSupport extends WriteSupport { private ParquetRowDataWriter writer; protected final Configuration hadoopConf; - public RowDataParquetWriteSupport(RowType rowType, Configuration config) { + public RowDataParquetWriteSupport(HoodieSchema hoodieSchema, Configuration config) { super(); - this.rowType = rowType; + this.rowType = HoodieSchemaConverter.convertToRowType(hoodieSchema); this.hadoopConf = new Configuration(config); this.schema = ParquetSchemaConverter.convertToParquetMessageType("flink_schema", rowType); } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataParquetConfigInjector.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataParquetConfigInjector.java index f65a2844649bd..67bf8e09cb6dc 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataParquetConfigInjector.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataParquetConfigInjector.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.engine.LocalTaskContextSupplier; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.testutils.DisableDictionaryInjector; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.collection.Pair; @@ -31,6 +32,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; +import org.apache.hudi.util.HoodieSchemaConverter; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.GenericRowData; @@ -111,6 +113,7 @@ public void testDisableDictionaryEncodingViaInjector() throws Exception { basePath + "/partition/path/test_dictionary_" + instantTime + ".parquet"); RowType rowType = getTestRowType(); + HoodieSchema schema = HoodieSchemaConverter.convertToSchema(rowType); // Create config with the custom injector HoodieConfig config = new HoodieConfig(); @@ -120,7 +123,7 @@ public void testDisableDictionaryEncodingViaInjector() throws Exception { // Create writer and write some data HoodieRowDataFileWriterFactory factory = new HoodieRowDataFileWriterFactory(storage); HoodieFileWriter writer = factory.newParquetFileWriter( - instantTime, parquetPath, config, rowType, new LocalTaskContextSupplier()); + instantTime, parquetPath, config, schema, new LocalTaskContextSupplier()); assertTrue(writer instanceof HoodieRowDataParquetWriter); @@ -169,6 +172,7 @@ public void testInvalidInjectorClassThrowsException() throws IOException { basePath + "/partition/path/test_invalid_" + instantTime + ".parquet"); RowType rowType = getTestRowType(); + HoodieSchema schema = HoodieSchemaConverter.convertToSchema(rowType); // Create config with an invalid/non-existent injector class HoodieConfig config = new HoodieConfig(); @@ -177,7 +181,7 @@ public void testInvalidInjectorClassThrowsException() throws IOException { // Should throw an exception when trying to create the writer HoodieRowDataFileWriterFactory factory = new HoodieRowDataFileWriterFactory(storage); assertThrows(Exception.class, () -> { - factory.newParquetFileWriter(instantTime, parquetPath, config, rowType, new LocalTaskContextSupplier()); + factory.newParquetFileWriter(instantTime, parquetPath, config, schema, new LocalTaskContextSupplier()); }); } @@ -189,6 +193,7 @@ public void testNoInjectorUsesDefaultConfig() throws Exception { basePath + "/partition/path/test_no_injector_" + instantTime + ".parquet"); RowType rowType = getTestRowType(); + HoodieSchema schema = HoodieSchemaConverter.convertToSchema(rowType); // Create config WITHOUT injector - should use default settings HoodieConfig config = new HoodieConfig(); @@ -197,7 +202,7 @@ public void testNoInjectorUsesDefaultConfig() throws Exception { // Create writer and write some data HoodieRowDataFileWriterFactory factory = new HoodieRowDataFileWriterFactory(storage); HoodieFileWriter writer = factory.newParquetFileWriter( - instantTime, parquetPath, config, rowType, new LocalTaskContextSupplier()); + instantTime, parquetPath, config, schema, new LocalTaskContextSupplier()); assertTrue(writer instanceof HoodieRowDataParquetWriter); From f690a22350352917bad5d1e9f724b2b50ac05453 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 28 May 2026 09:33:55 -0700 Subject: [PATCH 031/255] fix(cli): Fix the typo in show-inflight CLI command (#18868) (cherry picked from commit ff72186936eb516f5cf18448781e21969b8a9a44) --- .../java/org/apache/hudi/cli/commands/CommitsCommand.java | 2 +- .../org/apache/hudi/cli/commands/TestCommitsCommand.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java index fa177eca99527..f2089c9d38447 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java @@ -363,7 +363,7 @@ public String showCommitFiles( limit, headerOnly, rows, exportTableName); } - @ShellMethod(key = "commits show_infights", value = "Show inflight instants that are left longer than a certain duration") + @ShellMethod(key = "commits show_inflights", value = "Show inflight instants that are left longer than a certain duration") public String showInflightCommits( @ShellOption(value = {"--lookbackInMins"}, help = "Only show inflight commits that started before the specified lookback duration (in minutes).", defaultValue = "0") final Long durationInMins) { HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java index 7a0fd534309b3..9c80bbd2ec808 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java @@ -609,7 +609,7 @@ public void testInflightCommand() throws Exception { // Reload meta client to pick up new instants metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); - Object lookupBackInZeroMinsResult = shell.evaluate(() -> "commits show_infights --lookbackInMins 0"); + Object lookupBackInZeroMinsResult = shell.evaluate(() -> "commits show_inflights --lookbackInMins 0"); assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackInZeroMinsResult)); // All three instants should be shown when duration is 0 @@ -619,21 +619,21 @@ public void testInflightCommand() throws Exception { assertTrue(output.contains(oldInstantTime3)); // Only one instants should be shown when duration is 15 since 2nd commit is a completed commit. - Object lookupBackIn15MinsResult = shell.evaluate(() -> "commits show_infights --lookbackInMins 15"); + Object lookupBackIn15MinsResult = shell.evaluate(() -> "commits show_inflights --lookbackInMins 15"); assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn15MinsResult)); output = lookupBackIn15MinsResult.toString(); assertTrue(output.contains(oldInstantTime1)); assertFalse(output.contains(oldInstantTime2)); // Only one instant should be shown when duration is 50 - Object lookupBackIn50MinsResult = shell.evaluate(() -> "commits show_infights --lookbackInMins 50"); + Object lookupBackIn50MinsResult = shell.evaluate(() -> "commits show_inflights --lookbackInMins 50"); assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn50MinsResult)); output = lookupBackIn50MinsResult.toString(); assertTrue(output.contains(oldInstantTime1)); assertFalse(output.contains(oldInstantTime2)); // No instants should be shown when duration is > 60 - Object lookupBackIn70MinsResult = shell.evaluate(() -> "commits show_infights --lookbackInMins 70"); + Object lookupBackIn70MinsResult = shell.evaluate(() -> "commits show_inflights --lookbackInMins 70"); assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn70MinsResult)); output = lookupBackIn70MinsResult.toString(); assertTrue(output.contains(oldInstantTime1)); From 0655deb7e12c67e0315f633a23d846db8643420c Mon Sep 17 00:00:00 2001 From: Davis-Zhang-Onehouse <169106455+Davis-Zhang-Onehouse@users.noreply.github.com> Date: Thu, 28 May 2026 12:06:10 -0700 Subject: [PATCH 032/255] perf(streamer): fold validate() error-table WriteStatus sums into one pass (#18871) * perf(streamer): fold validate() error-table WriteStatus sums into one pass (cherry picked from commit 7af8cdf2c20e8ac5c57adaaad9fe4279974a5877) --- .../hudi/utilities/streamer/StreamSync.java | 40 ++++++++- .../TestStreamSyncWriteStatusValidation.java | 83 +++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncWriteStatusValidation.java diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 2b1406c7deab5..edeafb2755e36 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -1403,6 +1403,39 @@ public JavaRDD getWriteStatusRDD() { } } + /** + * Sums {@link WriteStatus#getTotalRecords()} and {@link WriteStatus#getTotalErrorRecords()} over the + * given RDD in a single Spark action, returned as a {@code (totalRecords, totalErroredRecords)} tuple. + * + *

    Folding both counters into one {@code aggregate} pass avoids re-deserializing every cached + * {@link WriteStatus} block a second time. Issuing two separate {@code mapToDouble(...).sum()} + * actions on the persisted error-table {@code WriteStatus} RDD doubles Kryo deserialization of + * cached partitions during commit validation and adds gratuitous heap pressure on memory-strained + * executors. + * + *

    {@code aggregate} is used (instead of {@code mapPartitions(...).reduce(...)}) so that a + * 0-partition RDD (e.g. {@code sc.emptyRDD()}, which {@link BaseErrorTableWriter#upsert} can + * return for an empty commit) returns {@code (0L, 0L)} rather than raising + * {@code UnsupportedOperationException} as {@code reduce} would. The mutable {@code long[2]} + * accumulator keeps per-record allocations at zero. + */ + @VisibleForTesting + static Tuple2 sumRecordAndErrorCounts(JavaRDD writeStatuses) { + long[] counts = writeStatuses.aggregate( + new long[]{0L, 0L}, + (acc, status) -> { + acc[0] += status.getTotalRecords(); + acc[1] += status.getTotalErrorRecords(); + return acc; + }, + (left, right) -> { + left[0] += right[0]; + left[1] += right[1]; + return left; + }); + return new Tuple2<>(counts[0], counts[1]); + } + /** * WriteStatus Validator for commits to hoodie streamer data table. * The writes to error table is taken care as well. @@ -1447,9 +1480,10 @@ public boolean validate(long tableTotalRecords, long tableTotalErroredRecords, O long totalRecords = tableTotalRecords; long totalErroredRecords = tableTotalErroredRecords; - if (isErrorTableWriteUnificationEnabled) { - totalRecords += errorTableWriteStatusRDDOpt.map(status -> status.mapToDouble(WriteStatus::getTotalRecords).sum().longValue()).orElse(0L); - totalErroredRecords += errorTableWriteStatusRDDOpt.map(status -> status.mapToDouble(WriteStatus::getTotalErrorRecords).sum().longValue()).orElse(0L); + if (isErrorTableWriteUnificationEnabled && errorTableWriteStatusRDDOpt.isPresent()) { + Tuple2 errorTableCounts = sumRecordAndErrorCounts(errorTableWriteStatusRDDOpt.get()); + totalRecords += errorTableCounts._1; + totalErroredRecords += errorTableCounts._2; } long totalSuccessfulRecords = totalRecords - totalErroredRecords; this.totalSuccessfulRecords.set(totalSuccessfulRecords); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncWriteStatusValidation.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncWriteStatusValidation.java new file mode 100644 index 0000000000000..793cfd5de8baa --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncWriteStatusValidation.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; + +import org.apache.spark.api.java.JavaRDD; +import org.junit.jupiter.api.Test; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for {@link StreamSync#sumRecordAndErrorCounts(JavaRDD)}, the single-pass fold that replaces two + * separate {@code mapToDouble(...).sum()} actions in error-table commit validation. + */ +class TestStreamSyncWriteStatusValidation extends SparkClientFunctionalTestHarness { + + @Test + void sumsTotalAndErroredRecordsAcrossPartitions() { + List writeStatuses = new ArrayList<>(); + writeStatuses.add(writeStatus(10L, 3L)); + writeStatuses.add(writeStatus(20L, 5L)); + writeStatuses.add(writeStatus(7L, 0L)); + // More partitions than elements forces an empty partition, exercising the aggregate fold. + JavaRDD writeStatusRDD = jsc().parallelize(writeStatuses, 4); + + Tuple2 counts = StreamSync.sumRecordAndErrorCounts(writeStatusRDD); + + assertEquals(37L, counts._1, "total records summed across all partitions"); + assertEquals(8L, counts._2, "total errored records summed across all partitions"); + } + + @Test + void sumsToZeroWhenNoWriteStatusesPresent() { + JavaRDD emptyRDD = jsc().parallelize(new ArrayList(), 2); + + Tuple2 counts = StreamSync.sumRecordAndErrorCounts(emptyRDD); + + assertEquals(0L, counts._1); + assertEquals(0L, counts._2); + } + + @Test + void sumsToZeroForZeroPartitionRDD() { + // BaseErrorTableWriter.upsert can return sc.emptyRDD() on an empty commit; JavaRDD.reduce throws + // UnsupportedOperationException on a 0-partition RDD, whereas aggregate returns the zero value. + JavaRDD emptyRDD = jsc().emptyRDD(); + + Tuple2 counts = StreamSync.sumRecordAndErrorCounts(emptyRDD); + + assertEquals(0L, counts._1); + assertEquals(0L, counts._2); + } + + private static WriteStatus writeStatus(long totalRecords, long totalErrorRecords) { + WriteStatus writeStatus = new WriteStatus(); + writeStatus.setTotalRecords(totalRecords); + writeStatus.setTotalErrorRecords(totalErrorRecords); + return writeStatus; + } +} From 1263c295236feaf9d5b61c92e49b310aae80ca45 Mon Sep 17 00:00:00 2001 From: fhan Date: Mon, 1 Jun 2026 06:38:55 +0800 Subject: [PATCH 033/255] fix(hudi-sync): Fix Hive test temp directory cleanup with JUnit TempDir (#18883) Co-authored-by: fhan (cherry picked from commit 6e40cffcc92b7b6b03410909eee16514c64296d3) --- .../apache/hudi/hive/TestHiveSyncTool.java | 16 +++++++++---- .../hudi/hive/testutils/HiveTestUtil.java | 23 +++++++++++++------ .../utilities/TestHiveIncrementalPuller.java | 2 +- .../hudi/utilities/TestHudiHiveSyncJob.java | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index ec9d4b0406de7..6d971bcc52263 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -67,6 +67,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; @@ -75,7 +76,6 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; -import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -181,6 +181,8 @@ private static Iterable syncModeAndTouchPartitionsEnabled() { private HiveSyncTool hiveSyncTool; private HoodieHiveSyncClient hiveClient; + @TempDir + java.nio.file.Path tempDir; @AfterAll public static void cleanUpClass() throws IOException { @@ -213,7 +215,7 @@ private static Iterable syncDataSourceTableParams() { @BeforeEach public void setUp() throws Exception { - HiveTestUtil.setUp(Option.empty(), true); + HiveTestUtil.setUp(Option.empty(), true, tempDir); } @AfterEach @@ -243,7 +245,7 @@ public void testUpdateBasePath(boolean useSchemaFromCommitMetadata, String syncM HiveTestUtil.fileSystem.delete(new Path(basePath), true); // create a new cow table and reSync - basePath = Files.createTempDirectory("hivesynctest" + Instant.now().toEpochMilli()).toUri().toString(); + basePath = createTempBasePath("hivesynctest"); hiveSyncProps.setProperty(META_SYNC_BASE_PATH.key(), basePath); HiveTestUtil.createCOWTable(instantTime, 1, useSchemaFromCommitMetadata); reInitHiveSyncClient(); @@ -864,7 +866,7 @@ public void testRecreateCOWTableOnBasePathChange(String syncMode, String enableP String commitTime2 = "105"; // let's update the basepath - basePath = Files.createTempDirectory("hivesynctest_new" + Instant.now().toEpochMilli()).toUri().toString(); + basePath = createTempBasePath("hivesynctest-new"); hiveSyncProps.setProperty(META_SYNC_BASE_PATH.key(), basePath); // let's create new table in new basepath @@ -1116,7 +1118,7 @@ public void testSyncMergeOnReadWithBasePathChange(boolean useSchemaFromCommitMet reSyncHiveTable(); // change the hoodie base path - basePath = Files.createTempDirectory("hivesynctest_new" + Instant.now().toEpochMilli()).toUri().toString(); + basePath = createTempBasePath("hivesynctest-new"); hiveSyncProps.setProperty(META_SYNC_BASE_PATH.key(), basePath); String instantTime2 = "102"; @@ -2268,6 +2270,10 @@ private void reInitHiveSyncClient() { hiveClient = (HoodieHiveSyncClient) hiveSyncTool.syncClient; } + private String createTempBasePath(String prefix) throws IOException { + return Files.createTempDirectory(tempDir, prefix).toUri().toString(); + } + private int getPartitionFieldSize() { return hiveSyncProps.getString(META_SYNC_PARTITION_FIELDS.key()).split(",").length; } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 4d75ca0e6b420..24e76b2d9f695 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -85,7 +85,6 @@ import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.file.Files; -import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -140,6 +139,11 @@ public class HiveTestUtil { private static Set createdTablesSet = new HashSet<>(); public static void setUp(Option hiveSyncProperties, boolean shouldClearBasePathAndTables) throws Exception { + setUp(hiveSyncProperties, shouldClearBasePathAndTables, null); + } + + public static void setUp(Option hiveSyncProperties, boolean shouldClearBasePathAndTables, + java.nio.file.Path tempDir) throws Exception { configuration = new Configuration(); if (zkServer == null) { zkService = new ZookeeperTestService(configuration); @@ -155,7 +159,10 @@ public static void setUp(Option hiveSyncProperties, boolean sho hiveSyncProps.setProperty(HIVE_URL.key(), hiveTestService.getJdbcHive2Url()); basePath = hiveSyncProps.getProperty(META_SYNC_BASE_PATH.key()); } else { - basePath = Files.createTempDirectory("hivesynctest" + Instant.now().toEpochMilli()).toUri().toString(); + java.nio.file.Path baseDir = tempDir == null + ? Files.createTempDirectory("hivesynctest") + : Files.createTempDirectory(Files.createDirectories(tempDir), "hivesynctest"); + basePath = baseDir.toUri().toString(); hiveSyncProps = new TypedProperties(); hiveSyncProps.setProperty(HIVE_URL.key(), hiveTestService.getJdbcHive2Url()); @@ -183,15 +190,17 @@ public static void setUp(Option hiveSyncProperties, boolean sho if (shouldClearBasePathAndTables) { clear(); } + if (!hiveSyncProperties.isPresent()) { + HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE) + .setTableName(TABLE_NAME) + .setPayloadClass(HoodieAvroPayload.class) + .initTable(HadoopFSUtils.getStorageConfWithCopy(configuration), basePath); + } } public static void clear() throws IOException, HiveException, MetaException { fileSystem.delete(new Path(basePath), true); - HoodieTableMetaClient.newTableBuilder() - .setTableType(HoodieTableType.COPY_ON_WRITE) - .setTableName(TABLE_NAME) - .setPayloadClass(HoodieAvroPayload.class) - .initTable(HadoopFSUtils.getStorageConfWithCopy(configuration), basePath); if (ddlExecutor != null) { for (String tableName : createdTablesSet) { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java index 42e0ee8748638..5bfb6933fdc6a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java @@ -67,7 +67,7 @@ public static void cleanUpClass() throws Exception { @BeforeEach public void setUp() throws Exception { - HiveTestUtil.setUp(Option.empty(), true); + HiveTestUtil.setUp(Option.empty(), true, tempDir); } @AfterEach diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHudiHiveSyncJob.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHudiHiveSyncJob.java index 64abaf68a8db8..0099eaebbf390 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHudiHiveSyncJob.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHudiHiveSyncJob.java @@ -67,7 +67,7 @@ public class TestHudiHiveSyncJob { @BeforeEach void setUp() throws Exception { - HiveTestUtil.setUp(Option.empty(), true); + HiveTestUtil.setUp(Option.empty(), true, tempDir); } @AfterEach From 8568c472312a9355eacad652516ab14df6cad8f4 Mon Sep 17 00:00:00 2001 From: fhan Date: Mon, 1 Jun 2026 10:51:31 +0800 Subject: [PATCH 034/255] fix(flink): set canonical base path in Hive sync config (#18884) Co-authored-by: fhan (cherry picked from commit da189952aacabc867ce45701ab2812fd23bd22be) --- .../apache/hudi/sink/utils/HiveSyncContext.java | 2 ++ .../hudi/sink/utils/TestHiveSyncContext.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java index 05b8878c01579..349084e128d99 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java @@ -36,6 +36,7 @@ import java.util.Properties; +import static org.apache.hudi.common.config.HoodieCommonConfig.BASE_PATH; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_PASS; @@ -101,6 +102,7 @@ public static HiveSyncContext create(Configuration conf, StorageConfiguration Date: Mon, 1 Jun 2026 17:13:07 +0800 Subject: [PATCH 035/255] fix(flink): fix data loss in stream read from earliest (#18848) * fix(flink): fix data loss in stream read from earliest * fix(flink): optimize UTs and refine de-duplicate full-table-scan timeline comment --------- Co-authored-by: fhan (cherry picked from commit 4e5034d354628bab348a55d694171632a0c7c2a0) --- .../hudi/source/IncrementalInputSplits.java | 35 ++- .../hudi/table/ITTestHoodieDataSource.java | 281 ++++++++++++++++++ 2 files changed, 313 insertions(+), 3 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java index bc6ebd29da21a..344b24bc10c4b 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java @@ -184,7 +184,12 @@ public Result inputSplits( return Result.EMPTY; } fileInfoList = fileIndex.getFilesInPartitions(); - List allFileSlices = getFileSlices(metaClient, commitTimeline, readPartitions, fileInfoList, analyzingResult.getMaxCompletionTime(), false); + // Use the full commits-and-compaction timeline rather than the (possibly compaction-filtered) + // activeTimeline carried by the QueryContext. Otherwise, on a MOR table with + // 'read.streaming.skip_compaction = true', file slice boundaries would be wrongly + // computed and log files could be missed, causing data loss. + List allFileSlices = getFileSlices(metaClient, getFullCommitsTimeline(metaClient), + readPartitions, fileInfoList, analyzingResult.getMaxCompletionTime(), false); fileSlices = fileIndex.filterFileSlices(allFileSlices); } else { if (cdcEnabled) { @@ -217,7 +222,11 @@ public Result inputSplits( return Result.EMPTY; } fileInfoList = fileIndex.getFilesInPartitions(); - List allFileSlices = getFileSlices(metaClient, commitTimeline, readPartitions, fileInfoList, analyzingResult.getMaxCompletionTime(), false); + // Same reason as the full-table-scan branch above: build the FileSystemView with the + // complete commits-and-compaction timeline to avoid losing data when 'skip_compaction' + // is enabled. + List allFileSlices = getFileSlices(metaClient, getFullCommitsTimeline(metaClient), + readPartitions, fileInfoList, analyzingResult.getMaxCompletionTime(), false); fileSlices = fileIndex.filterFileSlices(allFileSlices); } else { fileSlices = getFileSlices(metaClient, commitTimeline, readPartitions, files, analyzingResult.getMaxCompletionTime(), false); @@ -295,7 +304,10 @@ public Result inputSplits( log.warn("No files found for reading under path: {}", path); return Result.EMPTY; } - List allFileSlices = getFileSlices(metaClient, commitTimeline, readPartitions, pathInfoList, offsetToIssue, false); + // Same reason as the batch full-table-scan branch: + // see getFullCommitsTimeline() for why a compaction-filtered timeline must not be used here. + List allFileSlices = getFileSlices(metaClient, getFullCommitsTimeline(metaClient), + readPartitions, pathInfoList, offsetToIssue, false); List fileSlices = fileIndex.filterFileSlices(allFileSlices); List inputSplits = getInputSplits(fileSlices, metaClient, endInstant, null); @@ -391,6 +403,23 @@ private List getIncInputSplits( return getInputSplits(fileSlices, metaClient, endInstant, instantRange); } + /** + * Returns the full commit timeline (including completed compaction instants) for building + * a {@link HoodieTableFileSystemView} during full table scan. + * + *

    NOTE: when streaming/batch read enables {@code skip_compaction}, the {@code activeTimeline} + * carried by {@link IncrementalQueryAnalyzer.QueryContext} has already filtered out the + * compaction instants. Using such a partial timeline to construct a {@link HoodieTableFileSystemView} + * would mis-classify the file slice boundaries on a MOR table (since file slice boundaries + * are derived from compaction instants), leading to data loss when reading from the earliest + * or after start commit got archived. For full table scan we should always rely on the + * complete commits-and-compaction timeline; the {@code skip_compaction} semantics is preserved + * by the instant range filtering applied later on the generated input splits. + */ + private static HoodieTimeline getFullCommitsTimeline(HoodieTableMetaClient metaClient) { + return metaClient.getCommitsAndCompactionTimeline().filterCompletedAndCompactionInstants(); + } + private List getFileSlices( HoodieTableMetaClient metaClient, HoodieTimeline commitTimeline, diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 1cc5beaad4180..4af3598c0a979 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -28,16 +28,21 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode; import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.TimelineUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.configuration.HadoopConfigurations; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bucket.partition.PartitionBucketIndexUtils; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.sink.buffer.BufferMemoryType; import org.apache.hudi.sink.buffer.BufferType; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; import org.apache.hudi.table.catalog.HoodieCatalogTestUtils; import org.apache.hudi.table.catalog.HoodieHiveCatalog; import org.apache.hudi.util.StreamerUtil; @@ -89,6 +94,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -712,6 +719,280 @@ void testStreamReadMorTableWithCompactionPlan(boolean useSourceV2) throws Except assertRowsEquals(rows, TestData.DATA_SET_SOURCE_INSERT); } + /** + * Regression test for HUDI: data loss in stream read from earliest when + * {@code read.streaming.skip_compaction = true} on a MOR table with completed + * compaction commits. Covers the streaming earliest full table scan branch in + * {@link org.apache.hudi.source.IncrementalInputSplits#inputSplits(HoodieTableMetaClient, String, boolean)}. + * + *

    Triggering condition: + *

    + * + *

    Construction: + *

      + *
    1. Offline write {@code DATA_SET_INSERT} (8 records, ids 1..8) and then + * {@code DATA_SET_UPDATE_INSERT} (8 records, where ids 1..5 update existing keys + * and ids 9..11 are new) via {@link TestData#writeDataAsBatch}, which deterministically + * triggers an inline compaction once {@code COMPACTION_DELTA_COMMITS = 1} + + * {@code COMPACTION_ASYNC_ENABLED = true} are set. After this step the table has + * both a base file (from compaction) and log files written by the UPDATE batch.
      2. + *
    2. Streaming read from earliest with {@code skip_compaction = true} and wait until + * the expected number of merged rows are received. Without the fix, the FS view used + * in the earliest full-table-scan branch is built from a compaction-filtered + * timeline, file slice boundaries are wrongly computed, log files are missed + * and the read will never reach the expected row count (the test would time out).
      3. + *
    + */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testStreamReadMorTableWithCompactionFromEarliest(boolean useSourceV2) throws Exception { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.TABLE_NAME, "t1"); + conf.set(FlinkOptions.TABLE_TYPE, MERGE_ON_READ.name()); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()); + conf.set(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2); + // mandatory for writeDataAsBatch#inlineCompaction to actually run a compaction + conf.set(FlinkOptions.COMPACTION_ASYNC_ENABLED, true); + conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1); + + // Step 1: offline-write two batches with deterministic inline compaction in between. + TestData.writeDataAsBatch(TestData.DATA_SET_INSERT, conf); + TestData.writeDataAsBatch(TestData.DATA_SET_UPDATE_INSERT, conf); + + // Step 2: streaming read from earliest with skip_compaction = true. + String hoodieTableDDL = sql("t1") + .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) + .options(getDefaultKeys()) + .option(FlinkOptions.TABLE_TYPE, MERGE_ON_READ) + .option(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()) + .option(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2) + .option(FlinkOptions.READ_AS_STREAMING, true) + .option(FlinkOptions.READ_START_COMMIT, FlinkOptions.START_COMMIT_EARLIEST) + .option(FlinkOptions.READ_STREAMING_CHECK_INTERVAL, 2) + .option(FlinkOptions.READ_SOURCE_V2_ENABLED, useSourceV2) + // skip compaction instant -> active timeline drops compaction commit + .option(FlinkOptions.READ_STREAMING_SKIP_COMPACT, true) + .end(); + streamTableEnv.executeSql(hoodieTableDDL); + + // After the UPDATE batch, the merged result must contain all up-to-date records: + // - 5 updated records (id1..id5 from DATA_SET_UPDATE_INSERT) + // - 3 carried-over records (id6, id7, id8 from DATA_SET_INSERT, not touched by UPDATE) + // - 3 newly inserted records (id9, id10, id11 from DATA_SET_UPDATE_INSERT) + // i.e. 11 records in total. Without the fix the streaming read would never reach + // expectedNum = 11 and the test would time out via the CollectSink. + final int expectedNum = 11; + List rows = execSelectSqlWithExpectedNum(streamTableEnv, "select * from t1", expectedNum); + assertEquals(expectedNum, rows.size(), + "Expect 11 up-to-date records to be visible after earliest streaming read" + + " with skip_compaction on a MOR table that has a completed compaction commit" + + ", actual rows: " + rows); + } + + /** + * Regression test for HUDI: data loss in batch read from earliest when + * {@code read.streaming.skip_compaction = true} on a MOR table with completed + * compaction commits. Covers the batch full-table-scan branch in + * {@link org.apache.hudi.source.IncrementalInputSplits#inputSplits(HoodieTableMetaClient, boolean)}. + * + *

    This complements {@link #testStreamReadMorTableWithCompactionFromEarliest(boolean)} + * which only exercises the streaming code path. Without the fix, building the + * {@link org.apache.hudi.common.table.view.HoodieTableFileSystemView} with a + * compaction-filtered timeline would mis-classify file slice boundaries and + * lose log files. + */ + @Test + void testBatchReadMorTableWithCompactionFromEarliest() throws Exception { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.TABLE_NAME, "t1"); + conf.set(FlinkOptions.TABLE_TYPE, MERGE_ON_READ.name()); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()); + conf.set(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2); + // mandatory for writeDataAsBatch#inlineCompaction to actually run a compaction + conf.set(FlinkOptions.COMPACTION_ASYNC_ENABLED, true); + conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1); + + // Offline-write two batches against overlapping record keys, the 2nd write triggers + // an inline compaction that merges existing log files into a new base file - exactly + // the scenario that exposes the buggy file-slice classification when skip_compaction + // is enabled. + TestData.writeDataAsBatch(TestData.DATA_SET_INSERT, conf); + TestData.writeDataAsBatch(TestData.DATA_SET_UPDATE_INSERT, conf); + + String hoodieTableDDL = sql("t1") + .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) + .options(getDefaultKeys()) + .option(FlinkOptions.TABLE_TYPE, MERGE_ON_READ) + .option(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()) + .option(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2) + .option(FlinkOptions.READ_START_COMMIT, FlinkOptions.START_COMMIT_EARLIEST) + // skip compaction instant -> active timeline drops compaction commit + .option(FlinkOptions.READ_STREAMING_SKIP_COMPACT, true) + .end(); + batchTableEnv.executeSql(hoodieTableDDL); + + List result = CollectionUtil.iteratorToList( + batchTableEnv.executeSql("select * from t1").collect()); + // After update, the merged result must contain all up-to-date records: + // - 5 updated records (id1..id5 from DATA_SET_UPDATE_INSERT) + // - 3 carried-over records (id6, id7, id8 from DATA_SET_INSERT, not touched by UPDATE) + // - 3 newly inserted records (id9, id10, id11 from DATA_SET_UPDATE_INSERT) + // i.e. 11 records in total. Without the fix, log files belonging to the file slice prior + // to the inline compaction would be silently dropped by the file system view because + // the active timeline filtered out the compaction commit, and the result size would be + // smaller than 11. + assertEquals(11, result.size(), + "Expect all up-to-date records to be visible after earliest + skip_compaction batch read" + + ", actual rows: " + result); + } + + /** + * Regression test for HUDI: data loss when the start commit has been archived + * and {@code read.streaming.skip_compaction = true} on a MOR table. + * Covers the batch "fallback to full table scan" branch in + * {@link org.apache.hudi.source.IncrementalInputSplits#inputSplits(HoodieTableMetaClient, boolean)} + * which is reached when {@code hasArchivedInstants == true}. + * + *

    Construction: + *

      + *
    1. Write 10 delta-commit batches of {@code (id1,id2), (id3,id4), ...} on a MOR table + * so that each batch only inserts new keys (clear, predictable per-commit semantics).
      2. + *
    2. Trigger one completed compaction commit by issuing an extra UPDATE batch on + * {@code id1..id4} with {@code COMPACTION_DELTA_COMMITS = 1} via + * {@link TestData#writeDataAsBatch} (which explicitly calls {@code inlineCompaction()}). + * This creates exactly the file-slice boundary that the buggy FS view would + * mis-classify.
      3. + *
    3. Pick the LAST archived delta-commit as {@code read.start-commit} (filtered by + * {@code action = deltacommit} to exclude any archived compaction {@code commit} + * instants). This is deterministic regardless of how many delta commits were + * archived by the cleaner+archiver and routes the reader through the + * "archived start commit -> fullTableScan" branch.
      4. + *
    4. Read with {@code skip_compaction = true} and assert on the SET of record-keys + * in the result (not just on count). The expected key set is derived dynamically + * from the timeline: every delta_commit whose completion time is >= the chosen + * start_commit contributes its written ids, plus id1..id4 from the UPDATE batch + * are always present because the UPDATE is the latest write. Without the fix, + * log files of the file slice that straddles the compaction commit are silently + * dropped, so some of these ids would be missing.
      5. + *
    + */ + @Test + void testBatchReadMorTableWithCompactionStartCommitArchived() throws Exception { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.TABLE_NAME, "t1"); + conf.set(FlinkOptions.RECORD_KEY_FIELD, "uuid"); + conf.set(FlinkOptions.ORDERING_FIELDS, "ts"); + conf.set(FlinkOptions.TABLE_TYPE, MERGE_ON_READ.name()); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()); + conf.set(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2); + // aggressive archival to force older instants out of the active timeline + conf.set(FlinkOptions.ARCHIVE_MIN_COMMITS, 4); + conf.set(FlinkOptions.ARCHIVE_MAX_COMMITS, 5); + conf.set(FlinkOptions.CLEAN_RETAIN_COMMITS, 3); + conf.setString("hoodie.commits.archival.batch", "1"); + + // Step 1: write 10 batches of 2 new records each -> 10 delta_commit instants, 20 distinct keys. + for (int i = 0; i < 20; i += 2) { + List dataset = TestData.dataSetInsert(i + 1, i + 2); + TestData.writeData(dataset, conf); + } + + // Step 2: trigger at least one completed compaction commit by issuing one more delta_commit + // that UPDATES the very first record keys (id1..id4) and enabling COMPACTION_DELTA_COMMITS=1. + // The update writes new log files for the file group that contains id1..id4, and the inline + // compaction merges them into a new base file -> a real compaction file-slice boundary. + // NOTE: use writeDataAsBatch (which explicitly calls inlineCompaction()), since the plain + // writeData helper does not run the compaction even with COMPACTION_DELTA_COMMITS=1. + conf.set(FlinkOptions.COMPACTION_ASYNC_ENABLED, true); + conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1); + TestData.writeDataAsBatch(TestData.dataSetInsert(1, 2, 3, 4), conf); + + // Step 3: list the full timeline in one shot to map start_commit -> expected id set. + // Delta_commit instants are strictly monotonically increasing, so the sorted list of all + // delta_commits across active + archived timelines gives a 1:1 mapping to the 10 batches + // written in Step 1: the k-th delta_commit wrote id_{2k+1} and id_{2k+2}. + HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient( + new HadoopStorageConfiguration(HadoopConfigurations.getHadoopConf(new Configuration())), + tempFile.getAbsolutePath()); + // Use the merged (archived + active) timeline to capture all delta_commits, + // even those that may have been archived by the aggressive archival settings. + List batchInstantTimes = TimelineUtils.getTimeline(metaClient, true) + .getCommitsTimeline().filterCompletedInstants() + .filter(instant -> HoodieTimeline.DELTA_COMMIT_ACTION.equals(instant.getAction())) + .getInstantsAsStream().map(HoodieInstant::requestedTime).collect(Collectors.toList()); + // Step 1 produced exactly 10 delta_commits; the 11th (if present) is from Step 2 UPDATE. + // Keep only the first 10 to build the batch-index -> id mapping. + assertTrue(batchInstantTimes.size() >= 10, + "Expected at least 10 delta_commits from Step 1, got " + batchInstantTimes.size()); + batchInstantTimes = batchInstantTimes.subList(0, 10); + + // Step 4: pick the LAST archived delta_commit that belongs to Step 1's batches as + // start commit. This avoids any drift caused by archival ordering or by compaction + // `commit` instants being interleaved with delta_commits in the archived timeline, + // and also ignores the Step 2 UPDATE batch in case it also got archived. + Set step1InstantTimeSet = new TreeSet<>(batchInstantTimes); + List archivedDeltaCommits = metaClient.getArchivedTimeline().getCommitsTimeline() + .filterCompletedInstants() + .filter(instant -> HoodieTimeline.DELTA_COMMIT_ACTION.equals(instant.getAction())) + .filter(instant -> step1InstantTimeSet.contains(instant.requestedTime())) + .getInstants(); + // make sure archival actually happened on Step 1's batches, otherwise the test premise + // (the reader hits the archived start commit + fullTableScan branch) does not hold. + assertTrue(!archivedDeltaCommits.isEmpty(), + "archival did not happen as expected on Step 1's batches, archived delta commits = " + + archivedDeltaCommits + ", Step 1 batch instant times = " + batchInstantTimes); + HoodieInstant startInstant = archivedDeltaCommits.get(archivedDeltaCommits.size() - 1); + String archivedStartInstant = startInstant.requestedTime(); + + // The expected key set: every Step 1 batch whose instant time is >= start_commit contributes + // its 2 ids; plus id1..id4 from the Step 2 UPDATE batch (always the latest write, never + // excluded since its completion time is the largest). + int firstIncludedBatchIdx = batchInstantTimes.indexOf(archivedStartInstant); + assertTrue(firstIncludedBatchIdx >= 0, + "chosen start_commit " + archivedStartInstant + " is not one of the Step 1 batch instant times " + batchInstantTimes); + Set expectedIds = new TreeSet<>(); + for (int i = firstIncludedBatchIdx; i < batchInstantTimes.size(); i++) { + expectedIds.add("id" + (2 * i + 1)); + expectedIds.add("id" + (2 * i + 2)); + } + // UPDATE batch ids — always present in the merged view because the UPDATE is the latest write. + expectedIds.add("id1"); + expectedIds.add("id2"); + expectedIds.add("id3"); + expectedIds.add("id4"); + + String hoodieTableDDL = sql("t1") + .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) + .options(getDefaultKeys()) + .option(FlinkOptions.TABLE_TYPE, MERGE_ON_READ) + .option(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()) + .option(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, 2) + .option(FlinkOptions.READ_START_COMMIT, archivedStartInstant) + .option(FlinkOptions.READ_STREAMING_SKIP_COMPACT, true) + .end(); + batchTableEnv.executeSql(hoodieTableDDL); + + List result = CollectionUtil.iteratorToList( + batchTableEnv.executeSql("select uuid from t1").collect()); + Set actualIds = new TreeSet<>(); + for (Row r : result) { + actualIds.add(r.getField(0).toString()); + } + // Without the fix, the FS view used to construct file slices for the fallback full-table-scan + // branch is built from a compaction-filtered timeline, so log files of the file slice that + // straddles the compaction commit are silently dropped and some ids would be missing from + // {@code actualIds}. With the fix, every expected id must be present. + assertEquals(expectedIds, actualIds, + "Expected id set " + expectedIds + " but got " + actualIds + + " when reading from archived start commit " + archivedStartInstant + + " with skip_compaction = true on a MOR table that has a completed compaction commit"); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void testStreamReadMorTableWithBucketIndex(boolean partitioned) throws Exception { From 9c94936e6827a8a03dacccc9daac76f67513526a Mon Sep 17 00:00:00 2001 From: fhan Date: Mon, 1 Jun 2026 17:14:44 +0800 Subject: [PATCH 036/255] fix(spark): fix MOR bulk insert commit operation error (#18878) * fix(spark): fix mor bulk insert commit type error --------- Co-authored-by: fhan Co-authored-by: Y Ethan Guo (cherry picked from commit 4fdac3d32931f04f64e99b368218bd304677f130) --- .../apache/hudi/HoodieSparkSqlWriter.scala | 3 ++ .../hudi/TestHoodieSparkSqlWriter.scala | 35 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index aaa79929256f6..07058b885b620 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -1099,6 +1099,9 @@ class HoodieSparkSqlWriterInternal { } } val mergedParams = mutable.Map.empty ++ HoodieWriterUtils.parametersWithWriteDefaults(translatedOptsWithMappedTableConfig.toMap) + if (!mergedParams.contains(HoodieTableConfig.TYPE.key) && mergedParams.contains(TABLE_TYPE.key)) { + mergedParams(HoodieTableConfig.TYPE.key) = mergedParams(TABLE_TYPE.key) + } if (mergedParams.contains(KEYGENERATOR_CLASS_NAME.key) && !mergedParams.contains(HoodieTableConfig.KEY_GENERATOR_TYPE.key)) { mergedParams(HoodieTableConfig.KEY_GENERATOR_TYPE.key) = KeyGeneratorType.fromClassName(mergedParams(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key)).name } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala index 15f0b5d571b12..8968caaf348f0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala @@ -23,7 +23,7 @@ import org.apache.hudi.common.config.{HoodieConfig, HoodieMetadataConfig, Record import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieFileFormat, HoodieRecord, HoodieRecordPayload, HoodieReplaceCommitMetadata, HoodieTableType, WriteOperationType} import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} -import org.apache.hudi.common.table.timeline.TimelineUtils +import org.apache.hudi.common.table.timeline.{HoodieTimeline, TimelineUtils} import org.apache.hudi.common.testutils.HoodieTestDataGenerator import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.exception.{HoodieException, SchemaCompatibilityException} @@ -389,6 +389,39 @@ def testBulkInsertForDropPartitionColumn(): Unit = { } } + /** + * Regression test for MOR row-writer bulk_insert commit action. + * + * The writer receives table type through the datasource option + * hoodie.datasource.write.table.type. mergeParamsAndGetHoodieConfig must also + * propagate it to hoodie.table.type, because HoodieWriteConfig#getTableType + * reads the table-config key when row-writer bulk_insert chooses the commit action. + */ + @Test + def testMorRowWriterBulkInsertUsesDeltaCommitAction(): Unit = { + val fooTableModifier = commonTableModifier + .updated("hoodie.bulkinsert.shuffle.parallelism", "4") + .updated(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) + .updated(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL) + .updated(DataSourceWriteOptions.ENABLE_ROW_WRITER.key, "true") + // Keep the timeline focused on the write instant; otherwise compaction instants can obscure + // the commit action selected by the row-writer bulk_insert path. + .updated(DataSourceWriteOptions.ASYNC_COMPACT_ENABLE.key, "true") + + val schema = DataSourceTestUtils.getStructTypeExampleSchema + val structType = HoodieSchemaConversionUtils.convertHoodieSchemaToStructType(schema) + val records = DataSourceTestUtils.generateRandomRows(100) + val recordsSeq = convertRowListToSeq(records) + val df = spark.createDataFrame(sc.parallelize(recordsSeq), structType) + + HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, fooTableModifier, df) + + val metaClient = createMetaClient(spark, tempBasePath) + assertEquals(HoodieTableType.MERGE_ON_READ, metaClient.getTableConfig.getTableType) + val lastCompletedWrite = metaClient.getActiveTimeline.getCommitsTimeline.filterCompletedInstants().lastInstant().get() + assertEquals(HoodieTimeline.DELTA_COMMIT_ACTION, lastCompletedWrite.getAction) + } + /** * Test case for insert dataset without ordering fields. */ From d1b9e3f7dc4f653b653d88edd93cde4cd5b80a5d Mon Sep 17 00:00:00 2001 From: Xinli Shang Date: Mon, 1 Jun 2026 10:16:49 -0500 Subject: [PATCH 037/255] feat(utilities): migrate HoodieStreamerWriteStatusValidator into pre-commit validator framework (#18765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(utilities): migrate HoodieStreamerWriteStatusValidator into pre-commit validator framework (#18750) Completes the migration tracked in issue #18750 by deleting HoodieStreamerWriteStatusValidator (HSWSV) and replacing it with explicit pre-commit orchestration in StreamSync. Implements all 4 phases of the plan in one change. What changed - StreamSync.writeToSinkAndDoMetaSync() now orchestrates explicitly: 1. run user-configured pre-commit validators 2. count records (SuccessfulRecordCounter) 3. commit error table with strategy handling (ErrorTableCommitter) 4. apply the write-error gate (preserves commitOnErrors semantics) 5. call writeClient.commit() WITHOUT a WriteStatusValidator callback - HoodieStreamerWriteStatusValidator inner class removed (~100 LOC). - HSWSV's three concerns extracted into named helpers: - SuccessfulRecordCounter — pure counting, supports error-table unification - ErrorTableCommitter — error-table commit, returns success/failure - WriteErrorReporter — top-N errored-status logging - SparkWriteErrorValidator added as an opt-in BasePreCommitValidator that applies the same write-error check using the framework's failure.policy. - HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES doc references the new validator. What did NOT change - WriteStatusValidator interface and the writeClient.commit() hook are preserved — DataSourceUtils.SparkDataSourceWriteStatusValidator is another active caller in the Spark datasource path. - Default behavior for users who do not configure hoodie.precommit.validators is unchanged: the inline error gate in StreamSync preserves HSWSV semantics (commitOnErrors=false fails on any write error; commitOnErrors=true logs a warning and proceeds). - Error-table failure strategies (ROLLBACK_COMMIT, LOG_ERROR) preserved. Note: because the orchestration now runs before writeClient.commit(), ROLLBACK_COMMIT no longer needs to roll back — the commit simply doesn't happen. Verification - mvn -pl hudi-utilities -am test-compile: BUILD SUCCESS - mvn -pl hudi-utilities test -Dtest=TestSparkKafkaOffsetValidator,TestSparkValidationContext, TestSparkStreamerValidatorUtils,TestSparkWriteErrorValidator, TestSuccessfulRecordCounter -> 49/49 pass - mvn -pl hudi-utilities test -Dtest=TestStreamSync,TestHoodieStreamerUtils -> 51/51 pass - mvn -pl hudi-utilities,hudi-client/hudi-client-common checkstyle:check -> 0 violations - mvn -pl hudi-utilities,hudi-client/hudi-client-common apache-rat:check -> 0 unapproved Diff: 700 insertions, 135 deletions across 8 files (6 new, 2 modified). * fix(utilities): address self-review and Gemini-review findings on #18750 Follow-up to the previous commit on this branch. Addresses two batches of issues found in self-review and one external Gemini review. StreamSync orchestration ------------------------ - Reorder: commit error table BEFORE running validators. Previously, if SparkWriteErrorValidator or another validator failed first, the error-table records for the batch were lost. Now error records survive validator-driven aborts. - Split path: only collect writeStatuses to the driver when validators are configured. The default no-validator path now uses a distributed Spark aggregation (SuccessfulRecordCounter.computeFromRdd), restoring pre-#18750 no-overhead behavior. - Reuse already-collected list for WriteErrorReporter when available, to avoid a redundant Spark action. - Document the latent error-table / data-table inconsistency that predates #18750 (preserved, not regressed). - Document why SparkWriteErrorValidator is intentionally redundant with the inline Step 4 gate. SparkWriteErrorValidator ------------------------ - Update stale Javadoc: HSWSV is deleted in this branch, not "running alongside". - Fix user-facing error message: previously referenced a non-existent config key hoodie.streamer.commit.on.errors=true. Now references the real CLI flag --commit-on-errors. - Reject invalid failure.policy values at construction with a clear error listing allowed values, instead of a raw IllegalArgumentException with a confusing message. SuccessfulRecordCounter ----------------------- - Add computeFromRdd entry point so callers without a pre-collected list can compute counts via distributed Spark aggregation. - Switch from mapToDouble(...).sum().longValue() to a long fold to avoid silent precision loss above 2^53 record counts. - Add null guards on public entry points. ErrorTableCommitter ------------------- - Reword "side-effect-only" Javadoc which read as the opposite of intent. - Add null guards on public entry point. WriteErrorReporter ------------------ - Add List overload so callers that already have the collected list avoid an extra Spark action. - Demote the "Printing out the top N errors" header from ERROR to INFO (the header is not itself an error). - Standardize on Lombok @Slf4j for logger setup (was manual LoggerFactory), matching the convention used by other classes in the package. Tests ----- - TestErrorTableCommitter: 9 new tests covering unification on/off paths, success/failure propagation, RDD-no-op contract, and null safety. - TestWriteErrorReporter: 6 new tests covering null and empty inputs, max-cap enforcement, and the List overload. - TestSuccessfulRecordCounter: 5 new tests covering the unification path and computeFromRdd via a real local Spark context, plus null safety. - TestSparkWriteErrorValidator: 3 new tests for invalid policy parsing and regression test that the error message references --commit-on-errors. Verification ------------ - mvn -pl hudi-utilities test (helper + validator tests): 73/73 pass - mvn -pl hudi-utilities test -Dtest=TestStreamSync,TestHoodieStreamerUtils: 51/51 pass - mvn -pl hudi-utilities,hudi-client/hudi-client-common checkstyle:check: 0 violations - mvn -pl hudi-utilities,hudi-client/hudi-client-common apache-rat:check: 0 unapproved * ci: re-trigger CI run The previous run hit a Microsoft Container Registry block on the Azurite image pull, failing the integration-tests job for reasons unrelated to this change. Empty commit to trigger a fresh CI run. The underlying flake is fixed in #18772 (gracefully skips when MCR is blocked); once that merges, this PR can rebase and the failure mode will no longer block CI. * fix(utilities): address review feedback on #18750 - Always cache+collect writeStatuses; drop the dual-path (validators vs. no-validators) collect branching and the null sentinel (@danny0405). - Use the 6-arg writeClient.commit() overload — the trailing Option.empty() WriteStatusValidator slot was redundant (@danny0405). - Re-add writeClient.rollback(instantTime) before throwing on error-table ROLLBACK_COMMIT and the write-error gate so the inflight data-table instant doesn't leak under LAZY failed-writes cleanup policy (preserves HSWSV behavior). - Widen the latent-quirk comment on Step 1 to call out that a Step 2 validator failure (including the offset validator) has the same error-table-vs-data-table divergence as the Step 4 gate. - SparkWriteErrorValidator: import java.util.Arrays and drop the inline FQN. - Delete dead helpers exposed only by the removed no-validators path: SuccessfulRecordCounter.computeFromRdd and WriteErrorReporter.logTopErrors(JavaRDD), plus their tests. Verified: TestSparkKafkaOffsetValidator, TestSparkValidationContext, TestSparkStreamerValidatorUtils, TestSparkWriteErrorValidator, TestSuccessfulRecordCounter, TestWriteErrorReporter 61/61. TestStreamSync, TestHoodieStreamerUtils 51/51. * fix(utilities): address hudi-agent review feedback on #18750 - StreamSync Step 2: roll back the inflight data-table instant when a validator throws HoodieValidationException, so it doesn't leak under LAZY failed-writes cleanup. Same argument as Step 1 ROLLBACK_COMMIT and the Step 4 gate. Error-table records committed in Step 1 are preserved by design. - SparkWriteErrorValidator Javadoc: call out that under hoodie.errortable.write.unification.enabled=true the validator is strictly weaker than HSWSV because ValidationContext exposes only data-table stats. Users who need unified error counts should keep the inline commitOnErrors gate enabled. - SuccessfulRecordCounter: replace WriteStatusLongExtractor enum with a Function parameter; call sites use method references WriteStatus::getTotalRecords / ::getTotalErrorRecords. - TestSuccessfulRecordCounter: replace inline org.apache.spark.* FQNs with imports for SparkConf and JavaSparkContext. Verified: TestSuccessfulRecordCounter, TestSparkWriteErrorValidator, TestErrorTableCommitter, TestWriteErrorReporter, TestSparkKafkaOffsetValidator, TestSparkValidationContext, TestSparkStreamerValidatorUtils 70/70. TestStreamSync, TestHoodieStreamerUtils 51/51. Checkstyle / apache-rat: 0 violations on hudi-utilities and hudi-client-common. * fix(utilities): wrap and log validator failure on #18750 Address Danny's review comment on StreamSync.java:936. The Step 2 catch block now logs a contextual error and wraps the HoodieValidationException in a HoodieStreamerWriteException with the instant time, matching the rollback+wrap pattern used by Step 1 (error-table ROLLBACK_COMMIT) and Step 4 (write-error gate). The original validation exception is preserved as the cause. * fix(utilities): address hudi-agent review nits on #18750 - SuccessfulRecordCounter.Counts: rename getTotalErroredRecords() / totalErroredRecords to getTotalErrorRecords() / totalErrorRecords so the terminology matches WriteStatus.getTotalErrorRecords() (no past participle). Propagate the rename through StreamSync, SparkWriteErrorValidator Javadoc, and TestSuccessfulRecordCounter. - WriteErrorReporter and SparkWriteErrorValidator: replace Lombok @Slf4j with explicit `private static final Logger LOG = LoggerFactory.getLogger(...)` to match the streamer package style (StreamSync etc.). No behavior change. Tests: TestSuccessfulRecordCounter, TestSparkWriteErrorValidator, TestSparkValidationContext, TestSparkKafkaOffsetValidator, TestSparkStreamerValidatorUtils all pass (56/56). * fix(utilities): one-pass RDD aggregate in SuccessfulRecordCounter; clarify WARN_LOG+commitOnErrors error message * fix(utilities): restore StreamSync.sumRecordAndErrorCounts referenced by upstream TestStreamSyncWriteStatusValidation The merge resolution dropped the in-StreamSync helper added in upstream master because the migrated SuccessfulRecordCounter already has the equivalent one-pass aggregate. However the upstream merge also brought in TestStreamSyncWriteStatusValidation which calls StreamSync.sumRecordAndErrorCounts directly, breaking test compilation. Restore the helper as a @VisibleForTesting static method so the upstream test compiles. Production code paths use SuccessfulRecordCounter and are unaffected. --------- Co-authored-by: Xinli Shang (cherry picked from commit 8490c968d9dc71b44d4ca907aa1a50ba3111407c) --- .../HoodiePreCommitValidatorConfig.java | 4 +- .../streamer/ErrorTableCommitter.java | 86 ++++++ .../hudi/utilities/streamer/StreamSync.java | 244 +++++++----------- .../streamer/SuccessfulRecordCounter.java | 106 ++++++++ .../streamer/WriteErrorReporter.java | 69 +++++ .../validator/SparkWriteErrorValidator.java | 136 ++++++++++ .../streamer/TestErrorTableCommitter.java | 135 ++++++++++ .../streamer/TestSuccessfulRecordCounter.java | 200 ++++++++++++++ .../streamer/TestWriteErrorReporter.java | 99 +++++++ .../TestSparkWriteErrorValidator.java | 198 ++++++++++++++ 10 files changed, 1132 insertions(+), 145 deletions(-) create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableCommitter.java create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SuccessfulRecordCounter.java create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableCommitter.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestSuccessfulRecordCounter.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestWriteErrorReporter.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkWriteErrorValidator.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java index 169494b7244ac..f4999bc39e166 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java @@ -46,7 +46,9 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig { .withDocumentation("Comma separated list of class names that can be invoked to validate commit. " + "Available streaming offset validators: " + "org.apache.hudi.sink.validator.FlinkKafkaOffsetValidator (Flink Kafka), " - + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator (Spark/HoodieStreamer Kafka)"); + + "org.apache.hudi.utilities.streamer.validator.SparkKafkaOffsetValidator (Spark/HoodieStreamer Kafka). " + + "Available write-error validators: " + + "org.apache.hudi.utilities.streamer.validator.SparkWriteErrorValidator (Spark/HoodieStreamer write errors)."); public static final String VALIDATOR_TABLE_VARIABLE = ""; public static final ConfigProperty EQUALITY_SQL_QUERIES = ConfigProperty diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableCommitter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableCommitter.java new file mode 100644 index 0000000000000..1986e911a06b3 --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableCommitter.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.util.Option; + +import org.apache.spark.api.java.JavaRDD; + +import java.util.Objects; + +/** + * Commits the error-table side of a HoodieStreamer commit. + * + *

    Two paths exist, mirroring the original {@code HoodieStreamerWriteStatusValidator} behavior:

    + *
      + *
    • Unified write path ({@code isErrorTableWriteUnificationEnabled=true}): commit the + * error-table write statuses produced alongside the data-table write.
      • + *
    • Legacy path: invoke {@link BaseErrorTableWriter#upsertAndCommit(String, Option)} + * which performs both the upsert and the commit internally.
      • + *
    + * + *

    This helper performs the commit and reports success/failure. It deliberately does not + * handle the {@code ROLLBACK_COMMIT} / {@code LOG_ERROR} failure strategies — that policy decision + * lives in {@code StreamSync.writeToSinkAndDoMetaSync()}, which understands the surrounding + * orchestration. Extracted from {@code HoodieStreamerWriteStatusValidator} as part of #18750.

    + */ +public final class ErrorTableCommitter { + + private ErrorTableCommitter() { + } + + /** + * Commit the error-table writes for the given instant. + * + * @param errorTableWriter The configured error-table writer. Must not be null. + * @param errorTableWriteStatusRDDOpt Optional error-table write status RDD, populated when + * unification is enabled. Must not be null + * ({@link Option#empty()} if no RDD). + * @param isErrorTableWriteUnificationEnabled Whether unified-write mode is enabled. + * @param instantTime Instant being committed. + * @param latestCommittedInstant Optional latest completed instant, passed to legacy + * {@code upsertAndCommit}. Must not be null + * ({@link Option#empty()} if none). + * @return {@code true} if the error-table commit succeeded (or was a no-op); + * {@code false} if it failed and the caller must apply a failure-policy action. + */ + public static boolean commit(BaseErrorTableWriter errorTableWriter, + Option> errorTableWriteStatusRDDOpt, + boolean isErrorTableWriteUnificationEnabled, + String instantTime, + Option latestCommittedInstant) { + Objects.requireNonNull(errorTableWriter, "errorTableWriter"); + Objects.requireNonNull(errorTableWriteStatusRDDOpt, "errorTableWriteStatusRDDOpt"); + Objects.requireNonNull(instantTime, "instantTime"); + Objects.requireNonNull(latestCommittedInstant, "latestCommittedInstant"); + + if (isErrorTableWriteUnificationEnabled) { + // In unification mode the error-table writes were produced upstream by the unified write + // path. Commit them here; nothing to do when the optional RDD is absent (true no-op). + if (errorTableWriteStatusRDDOpt.isPresent()) { + return errorTableWriter.commit(errorTableWriteStatusRDDOpt.get()); + } + return true; + } + // Legacy path: writer performs both upsert and commit internally. + return errorTableWriter.upsertAndCommit(instantTime, latestCommittedInstant); + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index edeafb2755e36..9811fe9b01784 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -26,7 +26,6 @@ import org.apache.hudi.HoodieSchemaUtils; import org.apache.hudi.HoodieSparkSqlWriter; import org.apache.hudi.HoodieSparkUtils; -import org.apache.hudi.callback.common.WriteStatusValidator; import org.apache.hudi.client.HoodieWriteResult; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.WriteStatus; @@ -40,7 +39,6 @@ import org.apache.hudi.common.config.HoodieTimeGeneratorConfig; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.config.TypedProperties; -import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -77,7 +75,6 @@ import org.apache.hudi.config.HoodiePreCommitValidatorConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; -import org.apache.hudi.data.HoodieJavaRDD; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieMetaSyncException; @@ -872,38 +869,113 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood Map checkpointCommitMetadata = extractCheckpointMetadata(inputBatch, props, writeClient.getConfig().getWriteVersion().versionCode(), cfg); AtomicLong totalSuccessfulRecords = new AtomicLong(0); Option latestCommittedInstant = getLatestCommittedInstant(); - WriteStatusValidator writeStatusValidator = new HoodieStreamerWriteStatusValidator(cfg.commitOnErrors, instantTime, - cfg, errorTableWriter, errorTableWriteStatusRDDOpt, errorWriteFailureStrategy, isErrorTableWriteUnificationEnabled, writeClient, latestCommittedInstant, - totalSuccessfulRecords); String commitActionType = CommitUtils.getCommitActionType(cfg.operation, HoodieTableType.valueOf(cfg.tableType)); - // Cache the RDD only when pre-commit validators are configured. Validators collect the RDD - // before commit, so without caching the same DAG would re-evaluate inside writeClient.commit(). - // When no validators are configured, commit consumes the RDD once and caching adds no value. - // shouldUnpersist is true only when we created the cache here (validators present and storage - // level was NONE), so the finally block knows to release it. - boolean validatorsConfigured = !StringUtils.isNullOrEmpty(props.getString( - HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), - HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.defaultValue())); - boolean shouldUnpersist = validatorsConfigured && writeStatusRDD.getStorageLevel().equals(StorageLevel.NONE()); + // Pre-commit orchestration (issue #18750): the legacy HoodieStreamerWriteStatusValidator + // ran inside writeClient.commit() via the WriteStatusValidator callback and combined three + // concerns — count records, commit the error table, and gate on write errors. Each is now + // an explicit step here before writeClient.commit(), so the writer no longer receives a + // callback. Step order is deliberate (see comments below). + // + // The RDD is cached once and the write statuses are collected once on the driver. Both the + // count/error-logging steps and writeClient.commit() consume the materialized partitions + // rather than re-evaluating the upstream DAG. shouldUnpersist tracks whether we engaged the + // cache here so the finally block knows to release it. + boolean shouldUnpersist = writeStatusRDD.getStorageLevel().equals(StorageLevel.NONE()); if (shouldUnpersist) { writeStatusRDD.cache(); } boolean success; try { + List writeStatuses = writeStatusRDD.collect(); + boolean validatorsConfigured = !StringUtils.isNullOrEmpty(props.getString( + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.key(), + HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES.defaultValue())); + + // Step 1: Commit the error table BEFORE running validators or the write-error gate. + // Error records captured here are a genuine artifact of the write attempt and should + // survive even when a validator later blocks the data-table commit (otherwise the + // operator loses the captured errors and the next run has nothing to triage against). + // Latent design quirk (preserved from HSWSV): if error-table commit succeeds and any + // subsequent step fails (Step 2 validator including the offset validator, Step 4 gate, + // or writeClient.commit), the error table will have a committed instant for a data-table + // instant that never lands. Downstream consumers of the error table should tolerate this + // divergence. + if (errorTableWriter.isPresent()) { + boolean errorTableSuccess = ErrorTableCommitter.commit(errorTableWriter.get(), + errorTableWriteStatusRDDOpt, isErrorTableWriteUnificationEnabled, instantTime, + latestCommittedInstant); + if (!errorTableSuccess) { + switch (errorWriteFailureStrategy) { + case ROLLBACK_COMMIT: + // Roll back the inflight data-table instant so it doesn't leak under LAZY + // failed-writes cleanup policy (preserves HSWSV behavior). + writeClient.rollback(instantTime); + throw new HoodieStreamerWriteException("Error table commit failed for instant " + instantTime); + case LOG_ERROR: + LOG.error("Error table write failed for instant {}", instantTime); + break; + default: + throw new HoodieStreamerWriteException("Write failure strategy not implemented for " + errorWriteFailureStrategy); + } + } + } + + // Step 2: Run user-configured pre-commit validators (offset, custom, and the opt-in + // SparkWriteErrorValidator). Validators are intentionally stronger than commitOnErrors + // — a failure here aborts the data-table commit regardless of the gate in Step 4. + // Roll back the inflight data-table instant on validation failure so it doesn't leak + // under LAZY failed-writes cleanup policy (consistent with Step 1 ROLLBACK_COMMIT and + // the Step 4 gate below). Error-table records already committed in Step 1 are preserved + // by design — see Step 1's latent-quirk note. if (validatorsConfigured) { - List writeStatuses = writeStatusRDD.collect(); - - // Run pre-commit streaming offset validators (if configured). - // Placement before writeClient.commit() is intentional: offset validation is a stronger - // guard than commitOnErrors — if offset deviation indicates potential data loss, the commit - // must be prevented regardless of the commitOnErrors policy. - SparkStreamerValidatorUtils.runValidators(props, instantTime, writeStatuses, - checkpointCommitMetadata, metaClient); + try { + SparkStreamerValidatorUtils.runValidators(props, instantTime, writeStatuses, + checkpointCommitMetadata, metaClient); + } catch (HoodieValidationException e) { + LOG.error("Pre-commit validators failed for instant {}", instantTime, e); + writeClient.rollback(instantTime); + throw new HoodieStreamerWriteException("Pre-commit validators failed for instant " + instantTime, e); + } + } + + // Step 3: Count records. Drives the runMetaSync() decision below the try/finally. + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + writeStatuses, errorTableWriteStatusRDDOpt, isErrorTableWriteUnificationEnabled); + totalSuccessfulRecords.set(counts.getTotalSuccessfulRecords()); + LOG.info("instantTime={}, totalRecords={}, totalErrorRecords={}, totalSuccessfulRecords={}", + instantTime, counts.getTotalRecords(), counts.getTotalErrorRecords(), + counts.getTotalSuccessfulRecords()); + if (counts.getTotalRecords() == 0) { + LOG.info("No new data, perform empty commit."); + } + + // Step 4: Apply the legacy HSWSV write-error gate. + // commitOnErrors=false (default): any error -> log top N + fail. + // commitOnErrors=true: log a warning, proceed to commit. + // This gate is redundant with SparkWriteErrorValidator when that validator is configured + // with failure.policy=FAIL — both will reject the same commits. The redundancy is + // intentional: the gate preserves HSWSV's default behavior for users who do not configure + // any validators, while the validator gives users running multiple validators a unified + // failure-policy story. + if (counts.hasErrors()) { + if (cfg.commitOnErrors) { + LOG.warn("Some records failed to be merged but forcing commit since commitOnErrors set. Errors/Total={}/{}", + counts.getTotalErrorRecords(), counts.getTotalRecords()); + } else { + LOG.error("Delta Sync found errors when writing. Errors/Total={}/{}", + counts.getTotalErrorRecords(), counts.getTotalRecords()); + WriteErrorReporter.logTopErrors(writeStatuses); + // Roll back the inflight data-table instant so it doesn't leak under LAZY + // failed-writes cleanup policy (preserves HSWSV behavior). + writeClient.rollback(instantTime); + throw new HoodieStreamerWriteException("Commit " + instantTime + " has write errors and commitOnErrors=false"); + } } - success = writeClient.commit(instantTime, writeStatusRDD, Option.of(checkpointCommitMetadata), commitActionType, partitionToReplacedFileIds, Option.empty(), - Option.of(writeStatusValidator)); + // Step 5: Commit. No WriteStatusValidator callback — all checks are above. + success = writeClient.commit(instantTime, writeStatusRDD, Option.of(checkpointCommitMetadata), + commitActionType, partitionToReplacedFileIds, Option.empty()); } finally { if (shouldUnpersist) { writeStatusRDD.unpersist(); @@ -1407,17 +1479,9 @@ public JavaRDD getWriteStatusRDD() { * Sums {@link WriteStatus#getTotalRecords()} and {@link WriteStatus#getTotalErrorRecords()} over the * given RDD in a single Spark action, returned as a {@code (totalRecords, totalErroredRecords)} tuple. * - *

    Folding both counters into one {@code aggregate} pass avoids re-deserializing every cached - * {@link WriteStatus} block a second time. Issuing two separate {@code mapToDouble(...).sum()} - * actions on the persisted error-table {@code WriteStatus} RDD doubles Kryo deserialization of - * cached partitions during commit validation and adds gratuitous heap pressure on memory-strained - * executors. - * - *

    {@code aggregate} is used (instead of {@code mapPartitions(...).reduce(...)}) so that a - * 0-partition RDD (e.g. {@code sc.emptyRDD()}, which {@link BaseErrorTableWriter#upsert} can - * return for an empty commit) returns {@code (0L, 0L)} rather than raising - * {@code UnsupportedOperationException} as {@code reduce} would. The mutable {@code long[2]} - * accumulator keeps per-record allocations at zero. + *

    {@code aggregate} (not {@code reduce}) is used so a 0-partition RDD returns {@code (0L, 0L)} + * instead of raising {@code UnsupportedOperationException}; the mutable {@code long[2]} accumulator + * avoids per-record allocations. */ @VisibleForTesting static Tuple2 sumRecordAndErrorCounts(JavaRDD writeStatuses) { @@ -1435,112 +1499,4 @@ static Tuple2 sumRecordAndErrorCounts(JavaRDD writeStat }); return new Tuple2<>(counts[0], counts[1]); } - - /** - * WriteStatus Validator for commits to hoodie streamer data table. - * The writes to error table is taken care as well. - */ - static class HoodieStreamerWriteStatusValidator implements WriteStatusValidator { - - private final boolean commitOnErrors; - private final String instantTime; - private final HoodieStreamer.Config cfg; - private final Option errorTableWriter; - private final Option> errorTableWriteStatusRDDOpt; - private final HoodieErrorTableConfig.ErrorWriteFailureStrategy errorWriteFailureStrategy; - private final boolean isErrorTableWriteUnificationEnabled; - private final SparkRDDWriteClient writeClient; - private final Option latestCommittedInstant; - private final AtomicLong totalSuccessfulRecords; - - HoodieStreamerWriteStatusValidator(boolean commitOnErrors, - String instantTime, - HoodieStreamer.Config cfg, - Option errorTableWriter, - Option> errorTableWriteStatusRDDOpt, - HoodieErrorTableConfig.ErrorWriteFailureStrategy errorWriteFailureStrategy, - boolean isErrorTableWriteUnificationEnabled, - SparkRDDWriteClient writeClient, - Option latestCommittedInstant, - AtomicLong totalSuccessfulRecords) { - this.commitOnErrors = commitOnErrors; - this.instantTime = instantTime; - this.cfg = cfg; - this.errorTableWriter = errorTableWriter; - this.errorTableWriteStatusRDDOpt = errorTableWriteStatusRDDOpt; - this.errorWriteFailureStrategy = errorWriteFailureStrategy; - this.isErrorTableWriteUnificationEnabled = isErrorTableWriteUnificationEnabled; - this.writeClient = writeClient; - this.latestCommittedInstant = latestCommittedInstant; - this.totalSuccessfulRecords = totalSuccessfulRecords; - } - - @Override - public boolean validate(long tableTotalRecords, long tableTotalErroredRecords, Option> writeStatusesOpt) { - - long totalRecords = tableTotalRecords; - long totalErroredRecords = tableTotalErroredRecords; - if (isErrorTableWriteUnificationEnabled && errorTableWriteStatusRDDOpt.isPresent()) { - Tuple2 errorTableCounts = sumRecordAndErrorCounts(errorTableWriteStatusRDDOpt.get()); - totalRecords += errorTableCounts._1; - totalErroredRecords += errorTableCounts._2; - } - long totalSuccessfulRecords = totalRecords - totalErroredRecords; - this.totalSuccessfulRecords.set(totalSuccessfulRecords); - LOG.info("instantTime={}, totalRecords={}, totalErrorRecords={}, totalSuccessfulRecords={}", - instantTime, totalRecords, totalErroredRecords, totalSuccessfulRecords); - if (totalRecords == 0) { - LOG.info("No new data, perform empty commit."); - } - boolean hasErrorRecords = totalErroredRecords > 0; - if (!hasErrorRecords || commitOnErrors) { - if (hasErrorRecords) { - LOG.warn("Some records failed to be merged but forcing commit since commitOnErrors set. Errors/Total={}/{}", - totalErroredRecords, totalRecords); - } - } - - if (errorTableWriter.isPresent()) { - boolean errorTableSuccess = true; - // Commit the error events triggered so far to the error table - if (isErrorTableWriteUnificationEnabled && errorTableWriteStatusRDDOpt.isPresent()) { - errorTableSuccess = errorTableWriter.get().commit(errorTableWriteStatusRDDOpt.get()); - } else if (!isErrorTableWriteUnificationEnabled) { - errorTableSuccess = errorTableWriter.get().upsertAndCommit(instantTime, latestCommittedInstant); - } - if (!errorTableSuccess) { - switch (errorWriteFailureStrategy) { - case ROLLBACK_COMMIT: - LOG.info("Commit " + instantTime + " failed!"); - writeClient.rollback(instantTime); - throw new HoodieStreamerWriteException("Error table commit failed"); - case LOG_ERROR: - LOG.error("Error Table write failed for instant " + instantTime); - break; - default: - throw new HoodieStreamerWriteException("Write failure strategy not implemented for " + errorWriteFailureStrategy); - } - } - } - boolean canProceed = !hasErrorRecords || commitOnErrors; - if (canProceed) { - return canProceed; - } else { - LOG.error("Delta Sync found errors when writing. Errors/Total=" + totalErroredRecords + "/" + totalRecords); - LOG.error("Printing out the top 100 errors"); - ValidationUtils.checkArgument(writeStatusesOpt.isPresent(), "RDD is expected to be present when there are errors "); - HoodieJavaRDD.getJavaRDD(writeStatusesOpt.get()).filter(WriteStatus::hasErrors).take(100).forEach(writeStatus -> { - LOG.error("Global error " + writeStatus.getGlobalError()); - if (!writeStatus.getErrors().isEmpty()) { - writeStatus.getErrors().forEach((k,v) -> { - LOG.trace("Error for key %s : %s ", k, v); - }); - } - }); - // Rolling back instant - writeClient.rollback(instantTime); - throw new HoodieStreamerWriteException("Commit " + instantTime + " failed and rolled-back !"); - } - } - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SuccessfulRecordCounter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SuccessfulRecordCounter.java new file mode 100644 index 0000000000000..01439249d760d --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SuccessfulRecordCounter.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.util.Option; + +import org.apache.spark.api.java.JavaRDD; + +import java.util.List; +import java.util.Objects; + +/** + * Computes record counts for a HoodieStreamer commit, summing across the data-table + * write statuses and (optionally) the error-table write statuses when error-table + * write unification is enabled. + * + *

    Extracted from {@code HoodieStreamerWriteStatusValidator} (issue #18750) so the + * counting logic can be invoked from the explicit pre-commit orchestration in + * {@code StreamSync} without going through the {@code WriteStatusValidator} callback.

    + */ +public final class SuccessfulRecordCounter { + + private SuccessfulRecordCounter() { + } + + /** + * Compute total / errored / successful record counts from a pre-collected list of write statuses. + * + * @param dataTableWriteStatuses Pre-collected data-table write statuses. Must not be null. + * @param errorTableWriteStatusRDDOpt Optional error-table write status RDD; only consulted + * when unification is enabled. Must not be null + * ({@link Option#empty()} when no error table). + * @param isErrorTableWriteUnificationEnabled Whether error-table records contribute to the totals. + * @return immutable {@link Counts} snapshot. + */ + public static Counts compute(List dataTableWriteStatuses, + Option> errorTableWriteStatusRDDOpt, + boolean isErrorTableWriteUnificationEnabled) { + Objects.requireNonNull(dataTableWriteStatuses, "dataTableWriteStatuses"); + Objects.requireNonNull(errorTableWriteStatusRDDOpt, "errorTableWriteStatusRDDOpt"); + + long totalRecords = 0L; + long totalErrorRecords = 0L; + for (WriteStatus ws : dataTableWriteStatuses) { + totalRecords += ws.getTotalRecords(); + totalErrorRecords += ws.getTotalErrorRecords(); + } + if (isErrorTableWriteUnificationEnabled && errorTableWriteStatusRDDOpt.isPresent()) { + JavaRDD errorRdd = errorTableWriteStatusRDDOpt.get(); + long[] sums = errorRdd.aggregate( + new long[]{0L, 0L}, + (acc, ws) -> new long[]{acc[0] + ws.getTotalRecords(), acc[1] + ws.getTotalErrorRecords()}, + (a, b) -> new long[]{a[0] + b[0], a[1] + b[1]}); + totalRecords += sums[0]; + totalErrorRecords += sums[1]; + } + return new Counts(totalRecords, totalErrorRecords); + } + + /** Immutable count snapshot. */ + public static final class Counts { + public static final Counts ZERO = new Counts(0L, 0L); + + private final long totalRecords; + private final long totalErrorRecords; + + public Counts(long totalRecords, long totalErrorRecords) { + this.totalRecords = totalRecords; + this.totalErrorRecords = totalErrorRecords; + } + + public long getTotalRecords() { + return totalRecords; + } + + public long getTotalErrorRecords() { + return totalErrorRecords; + } + + public long getTotalSuccessfulRecords() { + return totalRecords - totalErrorRecords; + } + + public boolean hasErrors() { + return totalErrorRecords > 0; + } + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java new file mode 100644 index 0000000000000..2cb7c2fe6050b --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Logs the first N errored write statuses so the operator can triage a failed commit. + * + *

    Extracted from {@code HoodieStreamerWriteStatusValidator} (#18750).

    + */ +public final class WriteErrorReporter { + + private static final Logger LOG = LoggerFactory.getLogger(WriteErrorReporter.class); + + private static final int DEFAULT_MAX_ERRORS = 100; + + private WriteErrorReporter() { + } + + public static void logTopErrors(List writeStatuses) { + logTopErrors(writeStatuses, DEFAULT_MAX_ERRORS); + } + + /** + * Log up to {@code maxErrors} errored write statuses from a pre-collected list. Each errored + * status's global error is logged at ERROR; per-key errors are logged at TRACE. The header + * line is INFO. No-op when the list is null or {@code maxErrors <= 0}. + */ + public static void logTopErrors(List writeStatuses, int maxErrors) { + if (writeStatuses == null || maxErrors <= 0) { + return; + } + LOG.info("Printing out the top {} errored write statuses", maxErrors); + writeStatuses.stream() + .filter(WriteStatus::hasErrors) + .limit(maxErrors) + .forEach(WriteErrorReporter::logOne); + } + + private static void logOne(WriteStatus writeStatus) { + LOG.error("Global error: {}", writeStatus.getGlobalError()); + if (!writeStatus.getErrors().isEmpty()) { + writeStatus.getErrors().forEach((k, v) -> LOG.trace("Error for key {} : {}", k, v)); + } + } +} diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java new file mode 100644 index 0000000000000..07591b5b79739 --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.client.validator.BasePreCommitValidator; +import org.apache.hudi.client.validator.ValidationContext; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig.ValidationFailurePolicy; +import org.apache.hudi.exception.HoodieValidationException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +/** + * Pre-commit validator that fails the commit when records failed to write. + * + *

    Equivalent of the legacy {@code HoodieStreamerWriteStatusValidator}'s boolean error check + * ({@code hasErrorRecords = totalErrorRecords > 0}), wired through the pre-commit validator + * framework (issue #18750). Pure validation: no side effects (no error-table commit, no + * top-100 error logging, no instant rollback). Those side effects are handled separately by + * {@code StreamSync}'s pre-commit orchestration.

    + * + *

    Relationship with the inline write-error gate in {@code StreamSync}: the default + * commit path in {@code StreamSync} already applies an equivalent error check via the + * {@code commitOnErrors} flag. This validator exists so that users running multiple validators + * (e.g. write-error + offset checks) can express a unified pass/fail story through a single + * {@code failure.policy} knob. Enabling this validator while leaving {@code commitOnErrors=false} + * means both checks run and either can block the commit — they are intentionally not mutually + * exclusive.

    + * + *

    Behavior mapping from the legacy HSWSV (data-table only — see caveat below):

    + *
      + *
    • {@code commitOnErrors = false} (HSWSV default) ↔ {@code failure.policy = FAIL}
      • + *
    • {@code commitOnErrors = true} ↔ {@code failure.policy = WARN_LOG}
      • + *
    + * + *

    Unification caveat: when {@code hoodie.errortable.write.unification.enabled=true}, + * HSWSV's error check summed errors across both the data-table and the error-table write + * statuses. This validator only sees data-table stats via {@link ValidationContext} (specifically + * {@link ValidationContext#getTotalWriteErrors()} / {@link ValidationContext#getTotalRecordsWritten()}, + * which are derived from {@code HoodieWriteStat} on the data table). Under unification it is + * therefore strictly weaker than HSWSV: error-table-only errors will not trip this validator. Users + * who rely on unified error counts should keep the inline {@code commitOnErrors} gate in + * {@code StreamSync} enabled (i.e. leave {@code --commit-on-errors} off), which still consults + * the unified count via {@code SuccessfulRecordCounter}.

    + * + *

    Configuration:

    + *
      + *
    • {@code hoodie.precommit.validators}: Include + * {@code org.apache.hudi.utilities.streamer.validator.SparkWriteErrorValidator}
      • + *
    • {@code hoodie.precommit.validators.failure.policy}: FAIL (default) or WARN_LOG
      • + *
    + * + *

    Like {@link SparkKafkaOffsetValidator}, this class extends {@link BasePreCommitValidator} + * and must be invoked via {@link SparkStreamerValidatorUtils} — not {@code SparkValidatorUtils}, + * which expects a different constructor signature.

    + */ +public class SparkWriteErrorValidator extends BasePreCommitValidator { + + private static final Logger LOG = LoggerFactory.getLogger(SparkWriteErrorValidator.class); + + private final ValidationFailurePolicy failurePolicy; + + public SparkWriteErrorValidator(TypedProperties config) { + super(config); + String policyStr = config.getString( + HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), + HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.defaultValue()); + try { + this.failurePolicy = ValidationFailurePolicy.valueOf(policyStr); + } catch (IllegalArgumentException e) { + throw new HoodieValidationException(String.format( + "Invalid value '%s' for %s. Allowed values: %s.", + policyStr, + HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), + Arrays.toString(ValidationFailurePolicy.values())), e); + } + } + + @Override + public void validateWithMetadata(ValidationContext context) throws HoodieValidationException { + long totalErrors = context.getTotalWriteErrors(); + long totalRecordsWritten = context.getTotalRecordsWritten(); + // Total considered for the commit = successfully-written + failed. HSWSV computed this from + // the raw WriteStatus RDD; we derive the equivalent from HoodieWriteStat fields exposed by + // ValidationContext. + long totalRecords = totalRecordsWritten + totalErrors; + + if (totalRecords == 0) { + // Empty commit (mirrors HSWSV "No new data, perform empty commit."). + LOG.info("Empty commit (no records written, no errors). Skipping write-error validation " + + "for instant {}.", context.getInstantTime()); + return; + } + + if (totalErrors == 0) { + LOG.info("Write-error validation passed for instant {}: 0 errors out of {} records.", + context.getInstantTime(), totalRecords); + return; + } + + String errorMsg = String.format( + "Write-error validation failed for instant %s. " + + "Errors: %d, Total: %d. " + + "To allow commits despite write errors, both %s=WARN_LOG (bypasses this validator) " + + "and --commit-on-errors (bypasses the StreamSync inline gate) are required.", + context.getInstantTime(), totalErrors, totalRecords, + HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key()); + + if (failurePolicy == ValidationFailurePolicy.WARN_LOG) { + LOG.warn("{} (failure policy is WARN_LOG, commit will proceed)", errorMsg); + } else { + throw new HoodieValidationException(errorMsg); + } + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableCommitter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableCommitter.java new file mode 100644 index 0000000000000..4298079743423 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableCommitter.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.util.Option; + +import org.apache.spark.api.java.JavaRDD; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ErrorTableCommitter} covering the two write paths (unification on/off), + * the success/failure pass-through contract, and the no-op when no RDD is present. + */ +public class TestErrorTableCommitter { + + private static final String INSTANT = "20260520120000000"; + + @SuppressWarnings("unchecked") + private static JavaRDD rdd() { + return (JavaRDD) Mockito.mock(JavaRDD.class); + } + + @Test + public void testUnificationCommitSuccess() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + JavaRDD rdd = rdd(); + Mockito.when(writer.commit(rdd)).thenReturn(true); + + boolean result = ErrorTableCommitter.commit(writer, Option.of(rdd), true, INSTANT, Option.empty()); + + assertTrue(result); + Mockito.verify(writer).commit(rdd); + Mockito.verify(writer, Mockito.never()).upsertAndCommit(Mockito.any(), Mockito.any()); + } + + @Test + public void testUnificationCommitFailurePropagates() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + JavaRDD rdd = rdd(); + Mockito.when(writer.commit(rdd)).thenReturn(false); + + boolean result = ErrorTableCommitter.commit(writer, Option.of(rdd), true, INSTANT, Option.empty()); + + assertFalse(result); + } + + @Test + public void testUnificationWithoutRddIsNoOpAndReturnsTrue() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + + boolean result = ErrorTableCommitter.commit(writer, Option.empty(), true, INSTANT, Option.empty()); + + assertTrue(result); + Mockito.verifyNoInteractions(writer); + } + + @Test + public void testLegacyPathUsesUpsertAndCommit() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + Option latest = Option.of("20260520115959000"); + Mockito.when(writer.upsertAndCommit(INSTANT, latest)).thenReturn(true); + + boolean result = ErrorTableCommitter.commit(writer, Option.empty(), false, INSTANT, latest); + + assertTrue(result); + Mockito.verify(writer).upsertAndCommit(INSTANT, latest); + Mockito.verify(writer, Mockito.never()).commit(Mockito.any()); + } + + @Test + public void testLegacyPathFailurePropagates() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + Mockito.when(writer.upsertAndCommit(Mockito.anyString(), Mockito.any())).thenReturn(false); + + boolean result = ErrorTableCommitter.commit(writer, Option.empty(), false, INSTANT, Option.empty()); + + assertFalse(result); + } + + @Test + public void testLegacyPathIgnoresRddEvenWhenProvided() { + // When unification is OFF, the RDD must not be touched even if accidentally passed in. + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + JavaRDD rdd = rdd(); + Mockito.when(writer.upsertAndCommit(Mockito.anyString(), Mockito.any())).thenReturn(true); + + ErrorTableCommitter.commit(writer, Option.of(rdd), false, INSTANT, Option.empty()); + + Mockito.verify(writer, Mockito.never()).commit(Mockito.any()); + Mockito.verify(writer).upsertAndCommit(INSTANT, Option.empty()); + } + + @Test + public void testNullWriterRejected() { + assertThrows(NullPointerException.class, () -> + ErrorTableCommitter.commit(null, Option.empty(), false, INSTANT, Option.empty())); + } + + @Test + public void testNullRddOptionRejected() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + assertThrows(NullPointerException.class, () -> + ErrorTableCommitter.commit(writer, null, false, INSTANT, Option.empty())); + } + + @Test + public void testNullInstantRejected() { + BaseErrorTableWriter writer = Mockito.mock(BaseErrorTableWriter.class); + assertThrows(NullPointerException.class, () -> + ErrorTableCommitter.commit(writer, Option.empty(), false, null, Option.empty())); + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestSuccessfulRecordCounter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestSuccessfulRecordCounter.java new file mode 100644 index 0000000000000..1484a1d3e58e0 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestSuccessfulRecordCounter.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.util.Option; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link SuccessfulRecordCounter}. Covers the driver-side (collected list) + * counting and error-table unification paths, plus null safety on public entry points. + */ +public class TestSuccessfulRecordCounter { + + private static JavaSparkContext jsc; + + @BeforeAll + public static void setUp() { + SparkConf conf = new SparkConf() + .setAppName("TestSuccessfulRecordCounter") + .setMaster("local[2]"); + jsc = new JavaSparkContext(conf); + } + + @AfterAll + public static void tearDown() { + if (jsc != null) { + jsc.close(); + } + } + + @Test + public void testEmptyInputReturnsZero() { + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Collections.emptyList(), Option.empty(), false); + + assertEquals(0L, counts.getTotalRecords()); + assertEquals(0L, counts.getTotalErrorRecords()); + assertEquals(0L, counts.getTotalSuccessfulRecords()); + assertFalse(counts.hasErrors()); + } + + @Test + public void testSingleWriteStatusNoErrors() { + WriteStatus ws = Mockito.mock(WriteStatus.class); + Mockito.when(ws.getTotalRecords()).thenReturn(1000L); + Mockito.when(ws.getTotalErrorRecords()).thenReturn(0L); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Collections.singletonList(ws), Option.empty(), false); + + assertEquals(1000L, counts.getTotalRecords()); + assertEquals(0L, counts.getTotalErrorRecords()); + assertEquals(1000L, counts.getTotalSuccessfulRecords()); + assertFalse(counts.hasErrors()); + } + + @Test + public void testMultipleWriteStatusesAreSummed() { + WriteStatus a = Mockito.mock(WriteStatus.class); + Mockito.when(a.getTotalRecords()).thenReturn(100L); + Mockito.when(a.getTotalErrorRecords()).thenReturn(5L); + + WriteStatus b = Mockito.mock(WriteStatus.class); + Mockito.when(b.getTotalRecords()).thenReturn(200L); + Mockito.when(b.getTotalErrorRecords()).thenReturn(10L); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Arrays.asList(a, b), Option.empty(), false); + + assertEquals(300L, counts.getTotalRecords()); + assertEquals(15L, counts.getTotalErrorRecords()); + assertEquals(285L, counts.getTotalSuccessfulRecords()); + assertTrue(counts.hasErrors()); + } + + @Test + public void testUnificationDisabledIgnoresErrorTableRdd() { + WriteStatus ws = Mockito.mock(WriteStatus.class); + Mockito.when(ws.getTotalRecords()).thenReturn(50L); + Mockito.when(ws.getTotalErrorRecords()).thenReturn(2L); + + // Even if an error-table RDD is provided, unification=false means it must be ignored. + // Pass an "always throws" mock to prove the helper never touches it. + @SuppressWarnings("unchecked") + JavaRDD rdd = (JavaRDD) Mockito.mock(JavaRDD.class); + Mockito.when(rdd.mapToDouble(Mockito.any())).thenThrow(new AssertionError("RDD must not be consulted when unification is disabled")); + Mockito.when(rdd.map(Mockito.any())).thenThrow(new AssertionError("RDD must not be consulted when unification is disabled")); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Collections.singletonList(ws), Option.of(rdd), false); + + assertEquals(50L, counts.getTotalRecords()); + assertEquals(2L, counts.getTotalErrorRecords()); + assertEquals(48L, counts.getTotalSuccessfulRecords()); + } + + @Test + public void testHasErrorsBoundary() { + WriteStatus ws = Mockito.mock(WriteStatus.class); + Mockito.when(ws.getTotalRecords()).thenReturn(10L); + Mockito.when(ws.getTotalErrorRecords()).thenReturn(1L); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Collections.singletonList(ws), Option.empty(), false); + + assertTrue(counts.hasErrors()); + } + + // ========== Unification path (real Spark) ========== + + @Test + public void testUnificationEnabledSumsErrorTable() { + WriteStatus dataA = stat(100L, 5L); + WriteStatus dataB = stat(200L, 10L); + WriteStatus errA = stat(50L, 50L); + WriteStatus errB = stat(25L, 25L); + JavaRDD errorRdd = jsc.parallelize(Arrays.asList(errA, errB)); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Arrays.asList(dataA, dataB), Option.of(errorRdd), true); + + assertEquals(375L, counts.getTotalRecords()); // 100 + 200 + 50 + 25 + assertEquals(90L, counts.getTotalErrorRecords()); // 5 + 10 + 50 + 25 + assertEquals(285L, counts.getTotalSuccessfulRecords()); + assertTrue(counts.hasErrors()); + } + + // ========== RDD-based path (real Spark) ========== + + @Test + public void testUnificationWithRealSparkErrorRdd() { + WriteStatus dataA = stat(100L, 5L); + JavaRDD errorRdd = jsc.parallelize(Collections.singletonList(stat(50L, 50L))); + + SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( + Collections.singletonList(dataA), Option.of(errorRdd), true); + + assertEquals(150L, counts.getTotalRecords()); + assertEquals(55L, counts.getTotalErrorRecords()); + assertEquals(95L, counts.getTotalSuccessfulRecords()); + } + + // ========== Null safety ========== + + @Test + public void testNullDataTableListRejected() { + assertThrows(NullPointerException.class, () -> + SuccessfulRecordCounter.compute(null, Option.empty(), false)); + } + + @Test + public void testNullErrorTableOptionRejected() { + assertThrows(NullPointerException.class, () -> + SuccessfulRecordCounter.compute(Collections.emptyList(), null, false)); + } + + // ========== Helper ========== + + private static WriteStatus stat(long totalRecords, long totalErrorRecords) { + // Use a real WriteStatus so it serializes for Spark closures (Mockito mocks are not Serializable). + // @Data on WriteStatus exposes setters for totalRecords/totalErrorRecords. + WriteStatus ws = new WriteStatus(false, 0.0); + ws.setTotalRecords(totalRecords); + ws.setTotalErrorRecords(totalErrorRecords); + return ws; + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestWriteErrorReporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestWriteErrorReporter.java new file mode 100644 index 0000000000000..d13378edb74e2 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestWriteErrorReporter.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.client.WriteStatus; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +/** + * Tests for {@link WriteErrorReporter}. Verifies the no-op contract for null/empty inputs and + * that the List overload short-circuits without touching a Spark RDD. Logging output itself is + * intentionally not asserted — the value of the logger is human triage, not test assertions. + */ +public class TestWriteErrorReporter { + + @Test + public void testNullListIsNoOp() { + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors((List) null)); + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors((List) null, 10)); + } + + @Test + public void testEmptyListIsNoOp() { + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors(Collections.emptyList())); + } + + @Test + public void testZeroMaxErrorsIsNoOp() { + WriteStatus ws = errored("global err"); + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors(Collections.singletonList(ws), 0)); + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors(Collections.singletonList(ws), -5)); + } + + @Test + public void testListWithErrorsLogsWithoutThrowing() { + List statuses = Arrays.asList( + errored("err1"), + clean(), + errored("err2")); + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors(statuses, 10)); + } + + @Test + public void testMaxCapLimitsIteration() { + // Build a list with 5 errored statuses; cap at 2. Should iterate only the first 2 + // (no throw, no interaction beyond the first 2 verified by reaching the end of the call). + WriteStatus a = errored("a"); + WriteStatus b = errored("b"); + WriteStatus c = errored("c"); + WriteStatus d = errored("d"); + WriteStatus e = errored("e"); + assertDoesNotThrow(() -> WriteErrorReporter.logTopErrors(Arrays.asList(a, b, c, d, e), 2)); + // The other statuses must not have been touched. Their getErrors() should not have been called. + Mockito.verify(c, Mockito.never()).getErrors(); + Mockito.verify(d, Mockito.never()).getErrors(); + Mockito.verify(e, Mockito.never()).getErrors(); + } + + // ========== Helpers ========== + + private static WriteStatus errored(String globalError) { + WriteStatus ws = Mockito.mock(WriteStatus.class); + Mockito.when(ws.hasErrors()).thenReturn(true); + Mockito.when(ws.getGlobalError()).thenReturn(new RuntimeException(globalError)); + Mockito.when(ws.getErrors()).thenReturn(new HashMap<>()); + return ws; + } + + private static WriteStatus clean() { + WriteStatus ws = Mockito.mock(WriteStatus.class); + Mockito.when(ws.hasErrors()).thenReturn(false); + return ws; + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkWriteErrorValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkWriteErrorValidator.java new file mode 100644 index 0000000000000..92460a94e9430 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkWriteErrorValidator.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer.validator; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodiePreCommitValidatorConfig; +import org.apache.hudi.exception.HoodieValidationException; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link SparkWriteErrorValidator}. + */ +public class TestSparkWriteErrorValidator { + + private static final String INSTANT = "20260520120000000"; + + // ========== Helpers ========== + + private static TypedProperties failConfig() { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "FAIL"); + return props; + } + + private static TypedProperties warnConfig() { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "WARN_LOG"); + return props; + } + + private static HoodieWriteStat stat(String partition, long numInserts, long numUpdates, long writeErrors) { + HoodieWriteStat s = new HoodieWriteStat(); + s.setPartitionPath(partition); + s.setNumInserts(numInserts); + s.setNumUpdateWrites(numUpdates); + s.setTotalWriteErrors(writeErrors); + return s; + } + + private static SparkValidationContext context(List writeStats) { + return new SparkValidationContext( + INSTANT, + Option.of(new HoodieCommitMetadata()), + Option.of(writeStats), + Option.empty()); + } + + // ========== Tests ========== + + @Test + public void testNoErrorsPasses() { + // 1000 records written, no errors -> passes + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 1000, 0, 0))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testErrorsWithFailPolicyThrows() { + // 500 written + 50 errors -> fails under FAIL policy (mirrors commitOnErrors=false) + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 500, 0, 50))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> validator.validateWithMetadata(ctx)); + assertTrue(ex.getMessage().contains("Errors: 50"), "message should report error count"); + assertTrue(ex.getMessage().contains("Total: 550"), "message should report total record count"); + } + + @Test + public void testErrorsWithWarnPolicyDoesNotThrow() { + // 500 written + 50 errors -> passes under WARN_LOG (mirrors commitOnErrors=true) + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 500, 0, 50))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(warnConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testEmptyCommitPasses() { + // 0 records, 0 errors -> empty commit, validation is skipped + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 0, 0, 0))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testNoWriteStatsTreatedAsEmpty() { + SparkValidationContext ctx = new SparkValidationContext( + INSTANT, + Option.of(new HoodieCommitMetadata()), + Option.empty(), + Option.empty()); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + assertDoesNotThrow(() -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testErrorsAcrossMultiplePartitionsAreSummed() { + // p1: 100 written + 5 errors. p2: 200 written + 10 errors. Total errors > 0 -> fail. + SparkValidationContext ctx = context(Arrays.asList( + stat("p1", 100, 0, 5), + stat("p2", 200, 0, 10))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> validator.validateWithMetadata(ctx)); + assertTrue(ex.getMessage().contains("Errors: 15"), + "errors should be summed across partitions, got: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("Total: 315"), + "total should be inserts + updates + errors across partitions, got: " + ex.getMessage()); + } + + @Test + public void testUpdatesCountedTowardTotal() { + // 0 inserts, 100 updates, 1 error -> 1/101 -> fail under FAIL + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 0, 100, 1))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(failConfig()); + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> validator.validateWithMetadata(ctx)); + assertTrue(ex.getMessage().contains("Total: 101"), + "updates should count toward total, got: " + ex.getMessage()); + } + + @Test + public void testDefaultPolicyIsFail() { + // No failure.policy set -> default is FAIL + TypedProperties props = new TypedProperties(); + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 10, 0, 1))); + + SparkWriteErrorValidator validator = new SparkWriteErrorValidator(props); + assertThrows(HoodieValidationException.class, () -> validator.validateWithMetadata(ctx)); + } + + @Test + public void testInvalidFailurePolicyRejected() { + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "garbage"); + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> new SparkWriteErrorValidator(props)); + assertTrue(ex.getMessage().contains("Invalid value 'garbage'"), + "message should name the bad value, got: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("FAIL") && ex.getMessage().contains("WARN_LOG"), + "message should list allowed values, got: " + ex.getMessage()); + } + + @Test + public void testLowercasePolicyRejected() { + // Java enum valueOf is case-sensitive; lowercase should fail loudly with a clear message. + TypedProperties props = new TypedProperties(); + props.setProperty(HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key(), "fail"); + assertThrows(HoodieValidationException.class, () -> new SparkWriteErrorValidator(props)); + } + + @Test + public void testErrorMessageReferencesCommitOnErrorsFlag() { + // Regression: the prior message referenced a non-existent config key + // "hoodie.streamer.commit.on.errors". The user-facing fix should mention --commit-on-errors. + SparkValidationContext ctx = context(Collections.singletonList(stat("p1", 10, 0, 1))); + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> new SparkWriteErrorValidator(failConfig()).validateWithMetadata(ctx)); + assertTrue(ex.getMessage().contains("--commit-on-errors"), + "should reference the real CLI flag, got: " + ex.getMessage()); + } +} From 6a65c1efd2c2df0e6556f15cb77721f4cb8cef3c Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Tue, 2 Jun 2026 05:15:53 +0530 Subject: [PATCH 038/255] fix: Fix NPE due to race condition while handling rocksdb handles (#18834) --------- Co-authored-by: Lokesh Jain Co-authored-by: Lokesh Jain Co-authored-by: Lokesh Jain (cherry picked from commit 8259182e7e3256d7b44263bb416ae77a848873b5) --- .../common/table/view/HoodieTableFileSystemView.java | 9 +++++++-- .../common/table/view/RocksDbBasedFileSystemView.java | 3 +++ .../table/view/SpillableMapBasedFileSystemView.java | 8 ++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java index 70f81ffa4cee7..10d8879f8233c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java @@ -438,8 +438,8 @@ protected Option getReplaceInstant(final HoodieFileGroupId fileGr } @Override - public void close() { - super.close(); + protected void closeResources() throws Exception { + super.closeResources(); this.fgIdToPendingCompaction = null; this.fgIdToPendingLogCompaction = null; this.partitionToFileGroupsMap = null; @@ -449,6 +449,11 @@ public void close() { this.closed = true; } + @Override + public void close() { + super.close(); + } + @Override public boolean isClosed() { return closed; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index c2cac1420ff34..2620a29784fcd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -598,6 +598,7 @@ private static boolean isFileSliceWithoutCompactionBarrier(FileSlice fileSlice) @Override public void close() { try { + writeLock.lock(); LOG.info("Closing Rocksdb !!"); closed = true; closeResources(); @@ -605,6 +606,8 @@ public void close() { LOG.info("Closed Rocksdb !!"); } catch (Exception e) { throw new HoodieException("Unable to close file system view", e); + } finally { + writeLock.unlock(); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java index 89687ef070579..f531907653b13 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java @@ -225,14 +225,18 @@ protected void removeReplacedFileIdsAtInstants(Set instants) { } @Override - public void close() { + protected void closeResources() throws Exception { + // Close ExternalSpillableMaps (which hold RocksDB handles) while the writeLock is held + // by AbstractTableFileSystemView.close(). This prevents a race where a concurrent reader + // holding readLock could be mid-call in RocksDBDAO.put() when the handles are cleared, + // causing a NullPointerException at RocksDB.put(null_handle, ...). + super.closeResources(); closeFileGroupsMapIfPresent(); closePendingClusteringMapIfPresent(); closePendingCompactionMapIfPresent(); closePendingLogCompactionMapIfPresent(); closeBootstrapFileMapIfPresent(); closeReplaceInstantsMapIfPresent(); - super.close(); } private void closeReplaceInstantsMapIfPresent() { From 6572fe07dd1074b09d55fb6ebe621b8174f15b41 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Tue, 2 Jun 2026 08:47:25 +0800 Subject: [PATCH 039/255] fix(flink): Trigger a failover after pending instants recommitted for both global and partitioned RLI (#18793) (cherry picked from commit ed9ea0ead908e3765b6c938da121e96ace13ba38) --- .../sink/utils/BulkInsertFunctionWrapper.java | 4 +--- .../sink/utils/InsertFunctionWrapper.java | 20 +++++++++++++++---- .../utils/StreamWriteFunctionWrapper.java | 6 ------ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java index 26f79d870d664..159ead016fb5d 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java @@ -167,9 +167,7 @@ public void checkpointComplete(long checkpointId) { } public void coordinatorFails() throws Exception { - this.coordinator.close(); - this.coordinator.start(); - this.coordinator.setExecutor(new MockCoordinatorExecutor(coordinatorContext)); + // Do nothing since there is no state recovery for bulk insert. } public void restartCoordinator() throws Exception { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/InsertFunctionWrapper.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/InsertFunctionWrapper.java index 8c0369a889c77..9a8262155973a 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/InsertFunctionWrapper.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/InsertFunctionWrapper.java @@ -49,6 +49,8 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.CompletableFuture; /** @@ -70,6 +72,7 @@ public class InsertFunctionWrapper implements TestFunctionWrapper { private final boolean asyncClustering; private ClusteringFunctionWrapper clusteringFunctionWrapper; + private final TreeMap coordinatorStateStore; /** * Append write function. @@ -97,6 +100,7 @@ public InsertFunctionWrapper(String tablePath, Configuration conf, ExecutionConf this.coordinatorContext = new MockOperatorCoordinatorContext(new OperatorID(), 1); this.coordinator = new StreamWriteOperatorCoordinator(conf, this.coordinatorContext); this.stateInitializationContext = new MockStateInitializationContext(); + this.coordinatorStateStore = new TreeMap<>(); this.asyncClustering = OptionsResolver.needsAsyncClustering(conf); StreamConfig streamConfig = new StreamConfig(conf); @@ -142,8 +146,10 @@ public OperatorEvent getNextSubTaskEvent() { } public void checkpointFunction(long checkpointId) throws Exception { + CompletableFuture completableFuture = new CompletableFuture<>(); // checkpoint the coordinator first - this.coordinator.checkpointCoordinator(checkpointId, new CompletableFuture<>()); + this.coordinator.checkpointCoordinator(checkpointId, completableFuture); + this.coordinatorStateStore.put(checkpointId, completableFuture.get()); writeFunction.snapshotState(new MockFunctionSnapshotContext(checkpointId)); stateInitializationContext.checkpointBegin(checkpointId); @@ -167,9 +173,15 @@ public void checkpointComplete(long checkpointId) { } public void coordinatorFails() throws Exception { - this.coordinator.close(); - this.coordinator.start(); - this.coordinator.setExecutor(new MockCoordinatorExecutor(coordinatorContext)); + resetCoordinatorToCheckpoint(); + } + + private void resetCoordinatorToCheckpoint() { + if (coordinatorStateStore.isEmpty()) { + return; + } + Map.Entry latestState = this.coordinatorStateStore.lastEntry(); + this.coordinator.resetToCheckpoint(latestState.getKey(), latestState.getValue()); } public void restartCoordinator() throws Exception { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java index 7b1684c8e629e..131c14602cdef 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java @@ -368,13 +368,7 @@ public void jobFailover() throws Exception { } public void coordinatorFails() throws Exception { - this.coordinator.close(); - if (isStreamingWriteIndexEnabled) { - this.coordinator.setExecutor(new MockCoordinatorExecutor(coordinatorContext)); - } resetCoordinatorToCheckpoint(); - this.coordinator.start(); - this.coordinator.setExecutor(new MockCoordinatorExecutor(coordinatorContext)); } public void restartCoordinator() throws Exception { From 0471c31172c4795fc1b253bea35472fc4670b490 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 2 Jun 2026 15:46:24 +0800 Subject: [PATCH 040/255] refactor: Add Lombok annotations to hudi-utilities (Part 1) (#17823) * refactor: Add Lombok annotations to hudi-utilities (Part 1) (cherry picked from commit 095e80a9c256959d3802b0a138e48e72747940a9) --- hudi-utilities/pom.xml | 1 + .../hudi/utilities/HiveIncrementalPuller.java | 53 ++++--- .../apache/hudi/utilities/HoodieCleaner.java | 10 +- .../hudi/utilities/HoodieClusteringJob.java | 36 +++-- .../hudi/utilities/HoodieCompactor.java | 43 +++--- .../utilities/HoodieDataTableValidator.java | 36 +++-- .../utilities/HoodieDropPartitionsTool.java | 31 ++-- .../apache/hudi/utilities/HoodieIndexer.java | 36 +++-- .../HoodieMetadataTableValidator.java | 132 ++++++++---------- .../hudi/utilities/HoodieRepairTool.java | 57 ++++---- .../utilities/HoodieSnapshotExporter.java | 21 ++- .../apache/hudi/utilities/HoodieTTLJob.java | 9 +- .../apache/hudi/utilities/TableSizeStats.java | 56 ++++---- .../apache/hudi/utilities/UtilHelpers.java | 59 ++++---- .../kafka/HoodieWriteCommitKafkaCallback.java | 18 ++- .../HoodieWriteCommitPulsarCallback.java | 14 +- .../deser/KafkaAvroSchemaDeserializer.java | 5 +- .../ingestion/HoodieIngestionException.java | 6 +- .../ingestion/HoodieIngestionService.java | 18 ++- .../utilities/multitable/ArchiveTask.java | 7 +- .../utilities/multitable/ClusteringTask.java | 6 +- .../HoodieMultiTableServicesMain.java | 12 +- .../multitable/MultiTableServiceUtils.java | 7 +- .../utilities/perf/TimelineServerPerf.java | 9 +- .../schema/DelegatingSchemaProvider.java | 10 +- .../schema/FilebasedSchemaProvider.java | 7 +- .../utilities/schema/HiveSchemaProvider.java | 18 ++- .../schema/SchemaRegistryProvider.java | 8 +- .../schema/SimpleSchemaProvider.java | 7 +- .../DeleteSupportSchemaPostProcessor.java | 8 +- .../DropColumnSchemaPostProcessor.java | 8 +- .../transform/ChainedTransformer.java | 8 +- .../transform/FlatteningTransformer.java | 7 +- .../transform/SqlFileBasedTransformer.java | 12 +- .../transform/SqlQueryBasedTransformer.java | 10 +- 35 files changed, 357 insertions(+), 428 deletions(-) diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml index 802add77a2e90..11ec8522bfa19 100644 --- a/hudi-utilities/pom.xml +++ b/hudi-utilities/pom.xml @@ -234,6 +234,7 @@ org.projectlombok lombok + provided diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java index 2510edce72a8c..538253c9f60c0 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java @@ -28,13 +28,12 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.stringtemplate.v4.ST; import java.io.File; @@ -62,10 +61,9 @@ * - Only the source table can be incrementally pulled (usually the largest table) - The incrementally pulled table * can't be referenced more than once. */ +@Slf4j public class HiveIncrementalPuller { - private static final Logger LOG = LoggerFactory.getLogger(HiveIncrementalPuller.class); - public static class Config implements Serializable { @Parameter(names = {"--hiveUrl"}) @@ -133,15 +131,14 @@ private void validateIncrementalSQL() throws IOException { incrementalSQL = scanner.useDelimiter("\\Z").next(); } if (!incrementalSQL.contains(config.sourceDb + "." + config.sourceTable)) { - LOG.error("Incremental SQL does not have " + config.sourceDb + "." + config.sourceTable - + ", which means its pulling from a different table. Fencing this from happening."); + log.error("Incremental SQL does not have {}.{}, which means its pulling from a different table. Fencing this from happening.", + config.sourceDb, config.sourceTable); throw new HoodieIncrementalPullSQLException( "Incremental SQL does not have " + config.sourceDb + "." + config.sourceTable); } if (!incrementalSQL.contains("`_hoodie_commit_time` > '%s'")) { - LOG.error("Incremental SQL : " + incrementalSQL - + " does not contain `_hoodie_commit_time` > '%s'. Please add " - + "this clause for incremental to work properly."); + log.error("Incremental SQL : {} does not contain `_hoodie_commit_time` > '%s'. Please add this clause for incremental to work properly.", + incrementalSQL); throw new HoodieIncrementalPullSQLException( "Incremental SQL does not have clause `_hoodie_commit_time` > '%s', which " + "means its not pulling incrementally"); @@ -157,14 +154,14 @@ public void saveDelta() throws IOException { try { if (config.fromCommitTime == null) { config.fromCommitTime = inferCommitTime(fs); - LOG.info("FromCommitTime inferred as " + config.fromCommitTime); + log.info("FromCommitTime inferred as {}", config.fromCommitTime); } - LOG.info("FromCommitTime - " + config.fromCommitTime); + log.info("FromCommitTime - {}", config.fromCommitTime); String sourceTableLocation = getTableLocation(config.sourceDb, config.sourceTable); String lastCommitTime = getLastCommitTimePulled(fs, sourceTableLocation); if (lastCommitTime == null) { - LOG.info("Nothing to pull. However we will continue to create a empty table"); + log.info("Nothing to pull. However we will continue to create a empty table"); lastCommitTime = config.fromCommitTime; } @@ -182,9 +179,9 @@ public void saveDelta() throws IOException { initHiveBeelineProperties(stmt); executeIncrementalSQL(tempDbTable, tempDbTablePath, stmt); - LOG.info("Finished HoodieReader execution"); + log.info("Finished HoodieReader execution"); } catch (SQLException e) { - LOG.error("Exception when executing SQL", e); + log.error("Exception when executing SQL", e); throw new IOException("Could not scan " + config.sourceTable + " incrementally", e); } finally { try { @@ -192,14 +189,14 @@ public void saveDelta() throws IOException { stmt.close(); } } catch (SQLException e) { - LOG.error("Could not close the resultSet opened ", e); + log.error("Could not close the resultSet opened ", e); } try { if (this.connection != null) { this.connection.close(); } } catch (SQLException e) { - LOG.error("Could not close the JDBC connection", e); + log.error("Could not close the JDBC connection", e); } finally { this.connection = null; } @@ -229,7 +226,7 @@ private String getStoredAsClause() { } private void initHiveBeelineProperties(Statement stmt) throws SQLException { - LOG.info("Setting up Hive JDBC Session with properties"); + log.info("Setting up Hive JDBC Session with properties"); // set the queue executeStatement("set mapred.job.queue.name=" + config.yarnQueueName, stmt); // Set the inputFormat to HoodieCombineHiveInputFormat @@ -247,18 +244,18 @@ private void initHiveBeelineProperties(Statement stmt) throws SQLException { } private boolean deleteHDFSPath(FileSystem fs, String path) throws IOException { - LOG.info("Deleting path " + path); + log.info("Deleting path {}", path); return fs.delete(new Path(path), true); } private void executeStatement(String sql, Statement stmt) throws SQLException { - LOG.info("Executing: " + sql); + log.info("Executing: {}", sql); stmt.execute(sql); } private String inferCommitTime(FileSystem fs) throws IOException { - LOG.info("FromCommitTime not specified. Trying to infer it from Hoodie table " + config.targetDb + "." - + config.targetTable); + log.info("FromCommitTime not specified. Trying to infer it from Hoodie table {}.{}", + config.targetDb, config.targetTable); String targetDataLocation = getTableLocation(config.targetDb, config.targetTable); return scanForCommitTime(fs, targetDataLocation); } @@ -272,7 +269,7 @@ private String getTableLocation(String db, String table) { resultSet = stmt.executeQuery("describe formatted `" + db + "." + table + "`"); while (resultSet.next()) { if (resultSet.getString(1).trim().equals("Location:")) { - LOG.info("Inferred table location for " + db + "." + table + " as " + resultSet.getString(2)); + log.info("Inferred table location for {}.{} as {}", db, table, resultSet.getString(2)); return resultSet.getString(2); } } @@ -287,7 +284,7 @@ private String getTableLocation(String db, String table) { resultSet.close(); } } catch (SQLException e) { - LOG.error("Could not close the resultSet opened ", e); + log.error("Could not close the resultSet opened ", e); } } return null; @@ -314,7 +311,7 @@ private String scanForCommitTime(FileSystem fs, String targetDataPath) throws IO private boolean ensureTempPathExists(FileSystem fs, String lastCommitTime) throws IOException { Path targetBaseDirPath = new Path(config.hoodieTmpDir, config.targetTable + "__" + config.sourceTable); if (!fs.exists(targetBaseDirPath)) { - LOG.info("Creating " + targetBaseDirPath + " with permission drwxrwxrwx"); + log.info("Creating {} with permission drwxrwxrwx", targetBaseDirPath); boolean result = FileSystem.mkdirs(fs, targetBaseDirPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); if (!result) { @@ -329,7 +326,7 @@ private boolean ensureTempPathExists(FileSystem fs, String lastCommitTime) throw throw new HoodieException("Could not delete existing " + targetPath); } } - LOG.info("Creating " + targetPath + " with permission drwxrwxrwx"); + log.info("Creating {} with permission drwxrwxrwx", targetPath); return FileSystem.mkdirs(fs, targetBaseDirPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); } @@ -341,17 +338,17 @@ private String getLastCommitTimePulled(FileSystem fs, String sourceTableLocation .findInstantsAfter(config.fromCommitTime, config.maxCommits).getInstantsAsStream().map(HoodieInstant::requestedTime) .collect(Collectors.toList()); if (commitsToSync.isEmpty()) { - LOG.info("Nothing to sync. All commits in {} are {} and from commit time is {}", config.sourceTable, metadata.getActiveTimeline().getCommitsTimeline() + log.info("Nothing to sync. All commits in {} are {} and from commit time is {}", config.sourceTable, metadata.getActiveTimeline().getCommitsTimeline() .filterCompletedInstants().getInstants(), config.fromCommitTime); return null; } - LOG.info("Syncing commits {}", commitsToSync); + log.info("Syncing commits {}", commitsToSync); return commitsToSync.get(commitsToSync.size() - 1); } private Connection getConnection() throws SQLException { if (connection == null) { - LOG.info("Getting Hive Connection to {}", config.hiveJDBCUrl); + log.info("Getting Hive Connection to {}", config.hiveJDBCUrl); this.connection = DriverManager.getConnection(config.hiveJDBCUrl, config.hiveUsername, config.hivePassword); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java index 63ec3354410de..90b0c7dc1515c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java @@ -27,19 +27,17 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +@Slf4j public class HoodieCleaner { - private static final Logger LOG = LoggerFactory.getLogger(HoodieCleaner.class); - /** * Config for Cleaner. */ @@ -63,7 +61,7 @@ public HoodieCleaner(Config cfg, JavaSparkContext jssc, TypedProperties props) { this.cfg = cfg; this.jssc = jssc; this.props = props; - LOG.info("Creating Cleaner with configs : " + props.toString()); + log.info("Creating Cleaner with configs : {}", props.toString()); } public void run() { @@ -124,6 +122,6 @@ public static void main(String[] args) { SparkAdapterSupport$.MODULE$.sparkAdapter().stopSparkContext(jssc, exitCode); } - LOG.info("Cleaner ran successfully"); + log.info("Cleaner ran successfully"); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java index c378bce9026c3..67e35aea78311 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java @@ -37,9 +37,8 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; @@ -53,9 +52,9 @@ import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE; import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE_AND_EXECUTE; +@Slf4j public class HoodieClusteringJob { - private static final Logger LOG = LoggerFactory.getLogger(HoodieClusteringJob.class); private final Config cfg; private final TypedProperties props; private final JavaSparkContext jsc; @@ -168,7 +167,7 @@ public static void main(String[] args) { if (result != 0) { throw new HoodieException(resultMsg + " failed"); } - LOG.info(resultMsg + " success"); + log.info("{} success", resultMsg); jsc.stop(); } @@ -187,28 +186,28 @@ public int cluster(int retry) { return UtilHelpers.retry(retry, () -> { switch (cfg.runningMode.toLowerCase()) { case SCHEDULE: { - LOG.info("Running Mode: [" + SCHEDULE + "]; Do schedule"); + log.info("Running Mode: [{}]; Do schedule", SCHEDULE); Option instantTime = doSchedule(jsc); int result = instantTime.isPresent() ? 0 : -1; if (result == 0) { - LOG.info("The schedule instant time is " + instantTime.get()); + log.info("The schedule instant time is {}", instantTime.get()); } return result; } case SCHEDULE_AND_EXECUTE: { - LOG.info("Running Mode: [" + SCHEDULE_AND_EXECUTE + "]"); + log.info("Running Mode: [{}]", SCHEDULE_AND_EXECUTE); return doScheduleAndCluster(jsc); } case EXECUTE: { - LOG.info("Running Mode: [" + EXECUTE + "]; Do cluster"); + log.info("Running Mode: [{}]; Do cluster", EXECUTE); return doCluster(jsc); } case PURGE_PENDING_INSTANT: { - LOG.info("Running Mode: [" + PURGE_PENDING_INSTANT + "];"); + log.info("Running Mode: [{}];", PURGE_PENDING_INSTANT); return doPurgePendingInstant(jsc); } default: { - LOG.error("Unsupported running mode [" + cfg.runningMode + "], quit the job directly"); + log.error("Unsupported running mode [{}], quit the job directly", cfg.runningMode); return -1; } } @@ -224,10 +223,9 @@ private int doCluster(JavaSparkContext jsc) throws Exception { metaClient.getActiveTimeline().getFirstPendingClusterInstant(); if (firstClusteringInstant.isPresent()) { cfg.clusteringInstantTime = firstClusteringInstant.get().requestedTime(); - LOG.info("Found the earliest scheduled clustering instant which will be executed: " - + cfg.clusteringInstantTime); + log.info("Found the earliest scheduled clustering instant which will be executed: {}", cfg.clusteringInstantTime); } else { - LOG.info("There is no scheduled clustering in the table."); + log.info("There is no scheduled clustering in the table."); return 0; } } @@ -255,7 +253,7 @@ private Option doSchedule(SparkRDDWriteClient clien } private int doScheduleAndCluster(JavaSparkContext jsc) throws Exception { - LOG.info("Step 1: Do schedule"); + log.info("Step 1: Do schedule"); metaClient = HoodieTableMetaClient.reload(metaClient); String schemaStr = UtilHelpers.getSchemaFromLatestInstant(metaClient); try (SparkRDDWriteClient client = UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) { @@ -265,19 +263,19 @@ private int doScheduleAndCluster(JavaSparkContext jsc) throws Exception { Option staleInstant = TableServiceUtils.findStaleInflightInstant( metaClient, HoodieTimeline.CLUSTERING_ACTION, cfg.maxProcessingTimeMs); if (staleInstant.isPresent()) { - LOG.info("Found failed clustering instant at : " + staleInstant.get() + "; Will rollback the failed clustering and re-trigger again."); + log.info("Found failed clustering instant at : {}; Will rollback the failed clustering and re-trigger again.", staleInstant.get()); instantTime = Option.of(staleInstant.get().requestedTime()); } } instantTime = instantTime.isPresent() ? instantTime : doSchedule(client); if (!instantTime.isPresent()) { - LOG.info("Couldn't generate cluster plan"); + log.info("Couldn't generate cluster plan"); return -1; } - LOG.info("The schedule instant time is " + instantTime.get()); - LOG.info("Step 2: Do cluster"); + log.info("The schedule instant time is {}", instantTime.get()); + log.info("Step 2: Do cluster"); Option metadata = client.cluster(instantTime.get()).getCommitMetadata(); clean(client); return UtilHelpers.handleErrors(metadata.get(), instantTime.get()); @@ -345,7 +343,7 @@ public static void rollbackFailedClusteringForPartitions( metaClient.reloadActiveTimeline(); if (metaClient.getActiveTimeline().filterInflightsAndRequested() .containsInstant(instant.requestedTime())) { - LOG.info("Rolling back expired clustering instant {}", instant.requestedTime()); + log.info("Rolling back expired clustering instant {}", instant.requestedTime()); client.rollback(instant.requestedTime()); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java index 3ba8808dedac4..dcd60133e82a5 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java @@ -38,20 +38,19 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Objects; +@Slf4j public class HoodieCompactor { - private static final Logger LOG = LoggerFactory.getLogger(HoodieCompactor.class); public static final String EXECUTE = "execute"; public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_AND_EXECUTE = "scheduleandexecute"; @@ -75,11 +74,12 @@ public HoodieCompactor(JavaSparkContext jsc, Config cfg, TypedProperties props, this.props.put(HoodieCleanConfig.ASYNC_CLEAN.key(), false); if (this.metaClient.getTableConfig().isMetadataTableAvailable()) { // add default lock config options if MDT is enabled. - UtilHelpers.addLockOptions(cfg.basePath, this.metaClient.getBasePath().toUri().getScheme(), this.props); + UtilHelpers.addLockOptions(cfg.basePath, this.metaClient.getBasePath().toUri().getScheme(), this.props); } } public static class Config implements Serializable { + @Parameter(names = {"--base-path", "-sp"}, description = "Base path for the table", required = true) public String basePath = null; @Parameter(names = {"--table-name", "-tn"}, description = "Table name", required = true) @@ -196,7 +196,7 @@ public static void main(String[] args) { if (ret != 0) { throw new HoodieException("Fail to run compaction for " + cfg.tableName + ", return code: " + ret); } - LOG.info("Success to run compaction for " + cfg.tableName); + log.info("Success to run compaction for {}", cfg.tableName); jsc.stop(); } @@ -204,29 +204,29 @@ public int compact(int retry) { this.fs = HadoopFSUtils.getFs(cfg.basePath, jsc.hadoopConfiguration()); // need to do validate in case that users call compact() directly without setting cfg.runningMode validateRunningMode(cfg); - LOG.info(cfg.toString()); + log.info(cfg.toString()); int ret = UtilHelpers.retry(retry, () -> { switch (cfg.runningMode.toLowerCase()) { case SCHEDULE: { - LOG.info("Running Mode: [" + SCHEDULE + "]; Do schedule"); + log.info("Running Mode: [{}] Do schedule", SCHEDULE); Option instantTime = doSchedule(jsc); int result = instantTime.isPresent() ? 0 : -1; if (result == 0) { - LOG.info("The schedule instant time is " + instantTime.get()); + log.info("The schedule instant time is {}", instantTime.get()); } return result; } case SCHEDULE_AND_EXECUTE: { - LOG.info("Running Mode: [" + SCHEDULE_AND_EXECUTE + "]"); + log.info("Running Mode: [{}]", SCHEDULE_AND_EXECUTE); return doScheduleAndCompact(jsc); } case EXECUTE: { - LOG.info("Running Mode: [" + EXECUTE + "]; Do compaction"); + log.info("Running Mode: [{}]; Do compaction", EXECUTE); return doCompact(jsc); } default: { - LOG.info("Unsupported running mode [" + cfg.runningMode + "], quit the job directly"); + log.info("Unsupported running mode [{}], quit the job directly", cfg.runningMode); return -1; } } @@ -235,7 +235,7 @@ public int compact(int retry) { } private Integer doScheduleAndCompact(JavaSparkContext jsc) throws Exception { - LOG.info("Step 1: Do schedule"); + log.info("Step 1: Do schedule"); metaClient = HoodieTableMetaClient.reload(metaClient); Option instantTime = Option.empty(); @@ -243,20 +243,20 @@ private Integer doScheduleAndCompact(JavaSparkContext jsc) throws Exception { Option staleInstant = TableServiceUtils.findStaleInflightInstant( metaClient, HoodieTimeline.COMPACTION_ACTION, cfg.maxProcessingTimeMs); if (staleInstant.isPresent()) { - LOG.info("Found failed compaction instant at : " + staleInstant.get() + "; Will rollback the failed compaction and re-trigger again."); + log.info("Found failed compaction instant at : {}; Will rollback the failed compaction and re-trigger again.", staleInstant.get()); instantTime = Option.of(staleInstant.get().requestedTime()); } } instantTime = instantTime.isPresent() ? instantTime : doSchedule(jsc); if (!instantTime.isPresent()) { - LOG.error("Couldn't do schedule"); + log.error("Couldn't do schedule"); return -1; } cfg.compactionInstantTime = instantTime.get(); - LOG.info("The schedule instant time is {}", instantTime.get()); - LOG.info("Step 2: Do compaction"); + log.info("The schedule instant time is {}", instantTime.get()); + log.info("Step 2: Do compaction"); return doCompact(jsc); } @@ -278,10 +278,10 @@ private int doCompact(JavaSparkContext jsc) throws Exception { } else { schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile); } - LOG.info("Schema --> : " + schemaStr); + log.info("Schema --> : {}", schemaStr); try (SparkRDDWriteClient client = - UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) { + UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) { // If no compaction instant is provided by --instant-time, find the earliest scheduled compaction // instant from the active timeline if (StringUtils.isNullOrEmpty(cfg.compactionInstantTime)) { @@ -289,10 +289,9 @@ private int doCompact(JavaSparkContext jsc) throws Exception { Option firstCompactionInstant = metaClient.getActiveTimeline().filterPendingCompactionTimeline().firstInstant(); if (firstCompactionInstant.isPresent()) { cfg.compactionInstantTime = firstCompactionInstant.get().requestedTime(); - LOG.info("Found the earliest scheduled compaction instant which will be executed: " - + cfg.compactionInstantTime); + log.info("Found the earliest scheduled compaction instant which will be executed: {}", cfg.compactionInstantTime); } else { - LOG.info("There is no scheduled compaction in the table."); + log.info("There is no scheduled compaction in the table."); return 0; } } @@ -305,7 +304,7 @@ private int doCompact(JavaSparkContext jsc) throws Exception { private Option doSchedule(JavaSparkContext jsc) { try (SparkRDDWriteClient client = - UtilHelpers.createHoodieClient(jsc, cfg.basePath, "", cfg.parallelism, Option.of(cfg.strategyClassName), props)) { + UtilHelpers.createHoodieClient(jsc, cfg.basePath, "", cfg.parallelism, Option.of(cfg.strategyClassName), props)) { return client.scheduleCompaction(Option.empty()); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java index 521f94bbafc59..f34fe8650401e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java @@ -38,10 +38,9 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; @@ -98,10 +97,10 @@ * --min-validate-interval-seconds 60 * ``` */ +@Slf4j public class HoodieDataTableValidator implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieDataTableValidator.class); // Spark context private transient JavaSparkContext jsc; @@ -253,7 +252,7 @@ public static void main(String[] args) { try { validator.run(); } catch (Throwable throwable) { - LOG.error("Fail to do hoodie Data table validation for " + validator.cfg, throwable); + log.error("Fail to do hoodie Data table validation for " + validator.cfg, throwable); } finally { jsc.stop(); } @@ -261,12 +260,12 @@ public static void main(String[] args) { public void run() { try { - LOG.info(cfg.toString()); + log.info(cfg.toString()); if (cfg.continuous) { - LOG.info(" ****** do hoodie data table validation in CONTINUOUS mode ******"); + log.info(" ****** do hoodie data table validation in CONTINUOUS mode ******"); doHoodieDataTableValidationContinuous(); } else { - LOG.info(" ****** do hoodie data table validation once ******"); + log.info(" ****** do hoodie data table validation once ******"); doHoodieDataTableValidationOnce(); } } catch (Exception e) { @@ -283,7 +282,7 @@ private void doHoodieDataTableValidationOnce() { try { doDataTableValidation(); } catch (HoodieValidationException e) { - LOG.error("Metadata table validation failed to HoodieValidationException", e); + log.error("Metadata table validation failed to HoodieValidationException", e); if (!cfg.ignoreFailed) { throw e; } @@ -320,9 +319,8 @@ public void doDataTableValidation() { }).collect(Collectors.toList()); if (!danglingFilePaths.isEmpty() && danglingFilePaths.size() > 0) { - LOG.error("Data table validation failed due to dangling files count " - + danglingFilePaths.size() + ", found before active timeline"); - danglingFilePaths.forEach(entry -> LOG.error("Dangling file: " + entry.toString())); + log.error("Data table validation failed due to dangling files count {}, found before active timeline", danglingFilePaths.size()); + danglingFilePaths.forEach(entry -> log.error("Dangling file: " + entry.toString())); finalResult = false; if (!cfg.ignoreFailed) { throw new HoodieValidationException( @@ -354,8 +352,8 @@ public void doDataTableValidation() { }, hoodieInstants.size()).stream().collect(Collectors.toList()); if (!danglingFiles.isEmpty()) { - LOG.error("Data table validation failed due to extra files found for completed commits {}", danglingFiles.size()); - danglingFiles.forEach(entry -> LOG.error("Dangling file: {}", entry)); + log.error("Data table validation failed due to extra files found for completed commits {}", danglingFiles.size()); + danglingFiles.forEach(entry -> log.error("Dangling file: {}", entry)); finalResult = false; if (!cfg.ignoreFailed) { throw new HoodieValidationException("Data table validation failed due to dangling files " + danglingFiles.size()); @@ -363,16 +361,16 @@ public void doDataTableValidation() { } } } catch (Exception e) { - LOG.error("Data table validation failed", e); + log.error("Data table validation failed", e); if (!cfg.ignoreFailed) { throw new HoodieValidationException("Data table validation failed due to " + e.getMessage(), e); } } if (finalResult) { - LOG.info("Data table validation succeeded."); + log.info("Data table validation succeeded."); } else { - LOG.error("Data table validation failed."); + log.error("Data table validation failed."); } } @@ -389,12 +387,12 @@ protected Pair startService() { long toSleepMs = cfg.minValidateIntervalSeconds * 1000 - (System.currentTimeMillis() - start); if (toSleepMs > 0) { - LOG.info("Last validate ran less than min validate interval: " + cfg.minValidateIntervalSeconds + " s, sleep: " - + toSleepMs + " ms."); + log.info("Last validate ran less than min validate interval: {} s, sleep: {} ms.", + cfg.minValidateIntervalSeconds, toSleepMs); Thread.sleep(toSleepMs); } } catch (HoodieValidationException e) { - LOG.error("Shutting down AsyncDataTableValidateService due to HoodieValidationException", e); + log.error("Shutting down AsyncDataTableValidateService due to HoodieValidationException", e); if (!cfg.ignoreFailed) { throw e; } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java index 9e2e7d035b9a2..b9dadae7d87a4 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java @@ -38,13 +38,12 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; @@ -101,10 +100,10 @@ * * Also you can use --help to find more configs to use. */ +@Slf4j public class HoodieDropPartitionsTool implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieDropPartitionsTool.class); // Spark context private final transient JavaSparkContext jsc; // config @@ -282,7 +281,7 @@ public static void main(String[] args) { try { tool.run(); } catch (Throwable throwable) { - LOG.error("Fail to run deleting table partitions for " + cfg, throwable); + log.error("Fail to run deleting table partitions for {}", cfg, throwable); } finally { jsc.stop(); } @@ -290,21 +289,21 @@ public static void main(String[] args) { public void run() { try { - LOG.info(cfg.toString()); + log.info(cfg.toString()); Mode mode = Mode.valueOf(cfg.runningMode.toUpperCase()); switch (mode) { case DELETE: - LOG.info(" ****** The Hoodie Drop Partitions Tool is in delete mode ****** "); + log.info(" ****** The Hoodie Drop Partitions Tool is in delete mode ****** "); doDeleteTablePartitions(); syncToHiveIfNecessary(); break; case DRY_RUN: - LOG.info(" ****** The Hoodie Drop Partitions Tool is in dry-run mode ****** "); + log.info(" ****** The Hoodie Drop Partitions Tool is in dry-run mode ****** "); dryRun(); break; default: - LOG.info("Unsupported running mode [" + cfg.runningMode + "], quit the job directly"); + log.info("Unsupported running mode [{}], quit the job directly", cfg.runningMode); } } catch (Exception e) { throw new HoodieException("Unable to delete table partitions in " + cfg.basePath, e); @@ -368,19 +367,17 @@ private void verifyHiveConfigs() { } private void syncHive(HiveSyncConfig hiveSyncConfig) { - LOG.info("Syncing target hoodie table with hive table(" - + hiveSyncConfig.getStringOrDefault(HoodieSyncConfig.META_SYNC_TABLE_NAME) - + "). Hive metastore URL :" - + hiveSyncConfig.getStringOrDefault(HiveSyncConfigHolder.HIVE_URL) - + ", basePath :" + cfg.basePath); - LOG.info("Hive Sync Conf => " + hiveSyncConfig); + log.info("Syncing target hoodie table with hive table({}). Hive metastore URL :{}, basePath :{}", + hiveSyncConfig.getStringOrDefault(HoodieSyncConfig.META_SYNC_TABLE_NAME), + hiveSyncConfig.getStringOrDefault(HiveSyncConfigHolder.HIVE_URL), cfg.basePath); + log.info("Hive Sync Conf => {}", hiveSyncConfig); FileSystem fs = HadoopFSUtils.getFs(cfg.basePath, jsc.hadoopConfiguration()); HiveConf hiveConf = new HiveConf(); if (!StringUtils.isNullOrEmpty(cfg.hiveHMSUris)) { hiveConf.set("hive.metastore.uris", cfg.hiveHMSUris); } hiveConf.addResource(fs.getConf()); - LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString()); + log.info("Hive Conf => {}", hiveConf.getAllProperties().toString()); try (HiveSyncTool hiveSyncTool = new HiveSyncTool(hiveSyncConfig.getProps(), hiveConf)) { hiveSyncTool.syncHoodieTable(); } @@ -392,9 +389,9 @@ private void syncHive(HiveSyncConfig hiveSyncConfig) { * @param partitionToReplaceFileIds */ private void printDeleteFilesInfo(Map> partitionToReplaceFileIds) { - LOG.info("Data files and partitions to delete : "); + log.info("Data files and partitions to delete : "); for (Map.Entry> entry : partitionToReplaceFileIds.entrySet()) { - LOG.info(String.format("Partitions : %s, corresponding data file IDs : %s", entry.getKey(), entry.getValue())); + log.info("Partitions : {}, corresponding data file IDs : {}", entry.getKey(), entry.getValue()); } } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java index b3d743693eec0..2c7bab98900fb 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java @@ -35,10 +35,9 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; @@ -87,9 +86,9 @@ * hoodie.write.concurrency.mode=optimistic_concurrency_control * hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider */ +@Slf4j public class HoodieIndexer { - private static final Logger LOG = LoggerFactory.getLogger(HoodieIndexer.class); static final String DROP_INDEX = "dropindex"; private final HoodieIndexer.Config cfg; @@ -165,21 +164,21 @@ public static void main(String[] args) { if (result != 0) { throw new HoodieException(resultMsg + " failed"); } - LOG.info(resultMsg + " success"); + log.info("{} success", resultMsg); jsc.stop(); } public int start(int retry) { // indexing should be done only if metadata is enabled if (!props.getBoolean(HoodieMetadataConfig.ENABLE.key())) { - LOG.error(String.format("Metadata is not enabled. Please set %s to true.", HoodieMetadataConfig.ENABLE.key())); + log.error("Metadata is not enabled. Please set {} to true.", HoodieMetadataConfig.ENABLE.key()); return -1; } // all inflight or completed metadata partitions have already been initialized // so enable corresponding indexes in the props so that they're not deleted Set initializedMetadataPartitions = getInflightAndCompletedMetadataPartitions(metaClient.getTableConfig()); - LOG.info("Setting props for: " + initializedMetadataPartitions); + log.info("Setting props for: {}", initializedMetadataPartitions); initializedMetadataPartitions.forEach(p -> { if (PARTITION_NAME_COLUMN_STATS.equals(p)) { props.setProperty(ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true"); @@ -195,28 +194,28 @@ public int start(int retry) { return UtilHelpers.retry(retry, () -> { switch (cfg.runningMode.toLowerCase()) { case SCHEDULE: { - LOG.info("Running Mode: [" + SCHEDULE + "]; Do schedule"); + log.info("Running Mode: [{}]; Do schedule", SCHEDULE); Option instantTime = scheduleIndexing(jsc); int result = instantTime.isPresent() ? 0 : -1; if (result == 0) { - LOG.info("The schedule instant time is " + instantTime.get()); + log.info("The schedule instant time is {}", instantTime.get()); } return result; } case SCHEDULE_AND_EXECUTE: { - LOG.info("Running Mode: [" + SCHEDULE_AND_EXECUTE + "]"); + log.info("Running Mode: [{}]", SCHEDULE_AND_EXECUTE); return scheduleAndRunIndexing(jsc); } case EXECUTE: { - LOG.info("Running Mode: [" + EXECUTE + "];"); + log.info("Running Mode: [{}];", EXECUTE); return runIndexing(jsc); } case DROP_INDEX: { - LOG.info("Running Mode: [" + DROP_INDEX + "];"); + log.info("Running Mode: [{}];", DROP_INDEX); return dropIndex(jsc); } default: { - LOG.info("Unsupported running mode [" + cfg.runningMode + "], quit the job directly"); + log.info("Unsupported running mode [{}], quit the job directly", cfg.runningMode); return -1; } } @@ -248,7 +247,7 @@ private Option doSchedule(SparkRDDWriteClient clien Option indexingInstant = client.scheduleIndexing(partitionTypes, Collections.emptyList()); if (!indexingInstant.isPresent()) { - LOG.error("Scheduling of index action did not return any instant."); + log.error("Scheduling of index action did not return any instant."); } return indexingInstant; } @@ -264,7 +263,7 @@ private boolean indexExists(List partitionTypes) { Set requestedIndexPartitionPaths = partitionTypes.stream().map(MetadataPartitionType::getPartitionPath).collect(Collectors.toSet()); requestedIndexPartitionPaths.retainAll(indexedMetadataPartitions); if (!requestedIndexPartitionPaths.isEmpty()) { - LOG.error("Following indexes already built: " + requestedIndexPartitionPaths); + log.error("Following indexes already built: {}", requestedIndexPartitionPaths); return true; } return false; @@ -286,8 +285,7 @@ private int runIndexing(JavaSparkContext jsc) throws Exception { .firstInstant(); if (earliestPendingIndexInstant.isPresent()) { cfg.indexInstantTime = earliestPendingIndexInstant.get().requestedTime(); - LOG.info("Found the earliest scheduled indexing instant which will be executed: " - + cfg.indexInstantTime); + log.info("Found the earliest scheduled indexing instant which will be executed: {}", cfg.indexInstantTime); } else { throw new HoodieIndexException("There is no scheduled indexing in the table."); } @@ -316,18 +314,18 @@ private int dropIndex(JavaSparkContext jsc) throws Exception { client.dropIndex(partitionTypes); return 0; } catch (Exception e) { - LOG.error("Failed to drop index. ", e); + log.error("Failed to drop index. ", e); return -1; } } private boolean handleResponse(Option commitMetadata) { if (!commitMetadata.isPresent()) { - LOG.error("Indexing failed as no commit metadata present."); + log.error("Indexing failed as no commit metadata present."); return false; } List indexPartitionInfos = commitMetadata.get().getIndexPartitionInfos(); - LOG.info("Indexing complete for partitions: {}", indexPartitionInfos.stream().map(HoodieIndexPartitionInfo::getMetadataPartitionPath).collect(Collectors.toList())); + log.info("Indexing complete for partitions: {}", indexPartitionInfos.stream().map(HoodieIndexPartitionInfo::getMetadataPartitionPath).collect(Collectors.toList())); return isIndexBuiltForAllRequestedTypes(indexPartitionInfos); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java index daf9370c35509..d6876bea6e695 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java @@ -97,6 +97,8 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkException; import org.apache.spark.api.java.JavaPairRDD; @@ -105,8 +107,6 @@ import org.apache.spark.api.java.Optional; import org.apache.spark.sql.functions; import org.apache.spark.storage.StorageLevel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -204,10 +204,10 @@ * --min-validate-interval-seconds 60 * ``` */ +@Slf4j public class HoodieMetadataTableValidator implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieMetadataTableValidator.class); // Spark context private transient JavaSparkContext jsc; @@ -222,6 +222,7 @@ public class HoodieMetadataTableValidator implements Serializable { private final String taskLabels; + @Getter private final List throwables = new ArrayList<>(); public HoodieMetadataTableValidator(JavaSparkContext jsc, Config cfg) { @@ -240,21 +241,13 @@ public HoodieMetadataTableValidator(JavaSparkContext jsc, Config cfg) { .build()); } catch (TableNotFoundException tbe) { // Suppress the TableNotFound exception, table not yet created for a new stream - LOG.warn("Data table is not found. Skip current validation for: {}", cfg.basePath); + log.warn("Data table is not found. Skip current validation for: {}", cfg.basePath); } this.asyncMetadataTableValidateService = cfg.continuous ? Option.of(new AsyncMetadataTableValidateService()) : Option.empty(); this.taskLabels = generateValidationTaskLabels(); } - /** - * Returns list of Throwable which were encountered during validation. This method is useful - * when ignoreFailed parameter is set to true. - */ - public List getThrowables() { - return throwables; - } - /** * Returns true if there is a validation failure encountered during validation. * This method is useful when ignoreFailed parameter is set to true. @@ -514,7 +507,7 @@ public static void main(String[] args) { HoodieMetadataTableValidator validator = new HoodieMetadataTableValidator(jsc, cfg); validator.run(); } catch (Throwable throwable) { - LOG.error("Fail to do hoodie metadata table validation for {}", cfg, throwable); + log.error("Fail to do hoodie metadata table validation for {}", cfg, throwable); } finally { jsc.stop(); } @@ -522,17 +515,17 @@ public static void main(String[] args) { public boolean run() { if (!metaClientOpt.isPresent()) { - LOG.warn("Data table is not available to read for now, skip current validation for: {}", cfg.basePath); + log.warn("Data table is not available to read for now, skip current validation for: {}", cfg.basePath); return true; } boolean result = false; try { - LOG.info(cfg.toString()); + log.info(cfg.toString()); if (cfg.continuous) { - LOG.info(" ****** do hoodie metadata table validation in CONTINUOUS mode - {} ******", taskLabels); + log.info(" ****** do hoodie metadata table validation in CONTINUOUS mode - {} ******", taskLabels); doHoodieMetadataTableValidationContinuous(); } else { - LOG.info(" ****** do hoodie metadata table validation once - {} ******", taskLabels); + log.info(" ****** do hoodie metadata table validation once - {} ******", taskLabels); result = doHoodieMetadataTableValidationOnce(); } return result; @@ -554,7 +547,7 @@ private boolean doHoodieMetadataTableValidationOnce() { try { return doMetadataTableValidation(); } catch (Throwable e) { - LOG.error("Metadata table validation failed to HoodieValidationException {}", taskLabels, e); + log.error("Metadata table validation failed to HoodieValidationException {}", taskLabels, e); if (!cfg.ignoreFailed) { throw e; } @@ -577,7 +570,7 @@ private void doHoodieMetadataTableValidationContinuous() { public boolean doMetadataTableValidation() { boolean finalResult = true; if (!metaClientOpt.isPresent()) { - LOG.warn("Data table is not available to read for now, skip current validation."); + log.warn("Data table is not available to read for now, skip current validation."); return true; } HoodieTableMetaClient metaClient = this.metaClientOpt.get(); @@ -619,7 +612,7 @@ public boolean doMetadataTableValidation() { List allPartitions = validatePartitions(engineContext, basePath, metaClient); if (allPartitions.isEmpty()) { - LOG.warn("The result of getting all partitions is null or empty, skip current validation. {}", taskLabels); + log.warn("The result of getting all partitions is null or empty, skip current validation. {}", taskLabels); return true; } @@ -632,10 +625,10 @@ public boolean doMetadataTableValidation() { engineContext.parallelize(allPartitions, allPartitions.size()).map(partitionPath -> { try { validateFilesInPartition(metadataTableBasedContext, fsBasedContext, partitionPath, finalBaseFilesForCleaning); - LOG.info("Metadata table validation succeeded for partition {} (partition {})", partitionPath, taskLabels); + log.info("Metadata table validation succeeded for partition {} (partition {})", partitionPath, taskLabels); return Pair.of(true, null); } catch (HoodieValidationException e) { - LOG.error("Metadata table validation failed for partition {} due to HoodieValidationException (partition {})", + log.error("Metadata table validation failed for partition {} due to HoodieValidationException (partition {})", partitionPath, taskLabels, e); if (!cfg.ignoreFailed) { throw e; @@ -672,7 +665,7 @@ public boolean doMetadataTableValidation() { for (Pair res : result) { finalResult &= res.getKey(); if (res.getKey().equals(false)) { - LOG.error("Metadata Validation failed for table: {}", cfg.basePath, res.getValue()); + log.error("Metadata Validation failed for table: {}", cfg.basePath, res.getValue()); if (res.getRight() != null) { throwables.add(res.getRight()); } @@ -680,10 +673,10 @@ public boolean doMetadataTableValidation() { } if (finalResult) { - LOG.info("Metadata table validation succeeded ({}).", taskLabels); + log.info("Metadata table validation succeeded ({}).", taskLabels); return true; } else { - LOG.error("Metadata table validation failed ({}).", taskLabels); + log.error("Metadata table validation failed ({}).", taskLabels); return false; } } catch (HoodieValidationException validationException) { @@ -697,14 +690,14 @@ public boolean doMetadataTableValidation() { throw new HoodieValidationException("Unexpected spark failure", sparkException); } } catch (Exception e) { - LOG.warn("Error closing HoodieMetadataValidationContext, " + log.warn("Error closing HoodieMetadataValidationContext, " + "ignoring the error as the validation is successful.", e); return true; } } private void handleValidationException(HoodieValidationException e, List> result, String errorMsg) { - LOG.error("{} for table: {} ", errorMsg, cfg.basePath, e); + log.error("{} for table: {} ", errorMsg, cfg.basePath, e); if (!cfg.ignoreFailed) { throw e; } @@ -725,7 +718,7 @@ private boolean checkMetadataTableIsAvailable() { int finishedInstants = mdtMetaClient.getCommitsTimeline().filterCompletedInstants().countInstants(); if (finishedInstants == 0) { if (metaClientOpt.get().getCommitsTimeline().filterCompletedInstants().countInstants() == 0) { - LOG.info("There is no completed commit in both metadata table and corresponding data table: {}", taskLabels); + log.info("There is no completed commit in both metadata table and corresponding data table: {}", taskLabels); return false; } else { throw new HoodieValidationException("There is no completed instant for metadata table: " + cfg.basePath); @@ -734,10 +727,10 @@ private boolean checkMetadataTableIsAvailable() { return true; } catch (TableNotFoundException tbe) { // Suppress the TableNotFound exception if Metadata table is not available to read for now - LOG.warn("Metadata table is not found for table: {}. Skip current validation.", cfg.basePath); + log.warn("Metadata table is not found for table: {}. Skip current validation.", cfg.basePath); return false; } catch (Exception ex) { - LOG.warn("Metadata table is not available to read for now for table: {}, ", cfg.basePath, ex); + log.warn("Metadata table is not available to read for now for table: {}, ", cfg.basePath, ex); return false; } } @@ -776,7 +769,7 @@ List validatePartitions(HoodieSparkEngineContext engineContext, StorageP Option lastInstant = completedTimeline.lastInstant(); if (lastInstant.isPresent() && InstantComparison.compareTimestamps(partitionCreationTimeOpt.get(), GREATER_THAN, lastInstant.get().requestedTime())) { - LOG.info("Ignoring additional partition {}, as it was deduced to be part of a " + log.info("Ignoring additional partition {}, as it was deduced to be part of a " + "latest completed commit which was inflight when FS based listing was polled.", partitionFromMDT); actualAdditionalPartitionsInMDT.remove(partitionFromMDT); } @@ -804,7 +797,7 @@ List validatePartitions(HoodieSparkEngineContext engineContext, StorageP }).collect(Collectors.toList()); additionalFromFS.removeAll(emptyPartitions); if (additionalFromFS.isEmpty()) { - LOG.info("All out of sync partitions turned out to be empty {}", emptyPartitions); + log.info("All out of sync partitions turned out to be empty {}", emptyPartitions); misMatch.set(false); } else { misMatch.set(true); @@ -819,7 +812,7 @@ List validatePartitions(HoodieSparkEngineContext engineContext, StorageP + toStringWithThreshold(actualAdditionalPartitionsInMDT, cfg.logDetailMaxLength) + "\".\n All " + allPartitionPathsFromFS.size() + " partitions from FS listing " + toStringWithThreshold(allPartitionPathsFromFS, cfg.logDetailMaxLength); - LOG.error(message); + log.error(message); throw new HoodieValidationException(message); } } @@ -1252,10 +1245,10 @@ private void validateRecordIndexCount(HoodieSparkEngineContext sparkEngineContex if (countKeyFromTable != countKeyFromRecordIndex) { String message = String.format("Validation of record index count failed: %s entries from record index metadata, %s keys from the data table: %s", countKeyFromRecordIndex, countKeyFromTable, cfg.basePath); - LOG.error(message); + log.error(message); throw new HoodieValidationException(message); } else { - LOG.info("Validation of record index count succeeded: {} entries. Table: {}", countKeyFromRecordIndex, cfg.basePath); + log.info("Validation of record index count succeeded: {} entries. Table: {}", countKeyFromRecordIndex, cfg.basePath); } } @@ -1338,10 +1331,10 @@ private void validateRecordIndexContent(HoodieSparkEngineContext sparkEngineCont + "%s keys (total %s) from the data table have wrong location in record index " + "metadata. Table: %s Sample mismatches: %s", diffCount, countKey, cfg.basePath, String.join(";", result.getRight())); - LOG.error(message); + log.error(message); throw new HoodieValidationException(message); } else { - LOG.info("Validation of record index content succeeded: {} entries. Table: {}", countKey, cfg.basePath); + log.info("Validation of record index content succeeded: {} entries. Table: {}", countKey, cfg.basePath); } } @@ -1469,10 +1462,10 @@ void validate( if (mismatch) { String message = String.format("Validation of %s for partition %s failed for table: %s. %s", label, partitionPath, cfg.basePath, errorDetails); - LOG.error(message); + log.error(message); throw new HoodieValidationException(message); } else { - LOG.info("Validation of {} succeeded for partition {} for table: {}", label, partitionPath, cfg.basePath); + log.info("Validation of {} succeeded for partition {} for table: {}", label, partitionPath, cfg.basePath); } } @@ -1522,7 +1515,7 @@ void validateFileSlices( mismatch = true; break; } else { - LOG.info("There are uncommitted log files in the latest file slices but the committed log files match: {} {}", fileSlice1, fileSlice2); + log.info("There are uncommitted log files in the latest file slices but the committed log files match: {} {}", fileSlice1, fileSlice2); } } } @@ -1530,10 +1523,10 @@ void validateFileSlices( if (mismatch) { String message = String.format("Validation of %s for partition %s failed for table: %s. %s", label, partitionPath, cfg.basePath, errorDetails); - LOG.error(message); + log.error(message); throw new HoodieValidationException(message); } else { - LOG.info("Validation of {} succeeded for partition {} for table: {}", label, partitionPath, cfg.basePath); + log.info("Validation of {} succeeded for partition {} for table: {}", label, partitionPath, cfg.basePath); } } @@ -1565,16 +1558,16 @@ static String computeDiffSummary(List fileSliceListFromMetadataTable, // truncate start instant since range is not Set missingCommits = nonActiveInstantTimes.stream().filter(instant -> !archivedInstants.contains(instant)).collect(Collectors.toSet()); if (!missingCommits.isEmpty()) { - LOG.warn("File slices in file system belong to missing commits: {}", String.join(",", missingCommits)); + log.warn("File slices in file system belong to missing commits: {}", String.join(",", missingCommits)); activeTimeline.getRollbackTimeline().getInstantsAsStream().forEach(instant -> { HoodieInstant requestedInstant = metaClient.getInstantGenerator().getRollbackRequestedInstant(instant); try { HoodieRollbackPlan rollbackPlan = activeTimeline.readInstantContent(requestedInstant, HoodieRollbackPlan.class); if (missingCommits.contains(rollbackPlan.getInstantToRollback().getCommitTime())) { - LOG.warn("Missing commit ({}) is part of rollback plan: {}", rollbackPlan.getInstantToRollback().getCommitTime(), rollbackPlan); + log.warn("Missing commit ({}) is part of rollback plan: {}", rollbackPlan.getInstantToRollback().getCommitTime(), rollbackPlan); } } catch (IOException ex) { - LOG.warn("Failed to deserialize rollback plan for instant: {}", requestedInstant, ex); + log.warn("Failed to deserialize rollback plan for instant: {}", requestedInstant, ex); } }); } @@ -1663,7 +1656,7 @@ Pair hasCommittedLogFiles( try { HoodieSchema readerSchema = TableSchemaResolver.readSchemaFromLogFile(storage, new StoragePath(logFilePathStr)); if (readerSchema == null) { - LOG.warn("Cannot read schema from log file {}. Skip the check as it's likely being written by an inflight instant.", logFilePathStr); + log.warn("Cannot read schema from log file {}. Skip the check as it's likely being written by an inflight instant.", logFilePathStr); continue; } reader = @@ -1700,7 +1693,7 @@ Pair hasCommittedLogFiles( "Log file is committed in an instant in active timeline: instantTime=%s %s", instantTime, logFilePathStr)); } else { - LOG.warn("Log file is uncommitted in a completed instant, likely due to retry: instantTime={} {}", instantTime, logFilePathStr); + log.warn("Log file is uncommitted in a completed instant, likely due to retry: instantTime={} {}", instantTime, logFilePathStr); } } else if (completedInstantsTimeline.isBeforeTimelineStarts(instantTime)) { // The instant is in archived timeline @@ -1710,18 +1703,18 @@ Pair hasCommittedLogFiles( } else if (inflightInstantsTimeline.containsInstant(instantTime)) { // The instant is inflight in active timeline // hit an uncommitted block possibly from a failed write - LOG.warn("Log file is uncommitted because of an inflight instant: instantTime={} {}", instantTime, logFilePathStr); + log.warn("Log file is uncommitted because of an inflight instant: instantTime={} {}", instantTime, logFilePathStr); } else { // The instant is after the start of the active timeline, // but it cannot be found in the active timeline - LOG.warn("Log file is uncommitted because the instant is after the start of the active timeline but absent or in requested in the active timeline: instantTime={} {}", + log.warn("Log file is uncommitted because the instant is after the start of the active timeline but absent or in requested in the active timeline: instantTime={} {}", instantTime, logFilePathStr); } } else { - LOG.warn("There is no log block in {}", logFilePathStr); + log.warn("There is no log block in {}", logFilePathStr); } } catch (IOException e) { - LOG.warn("Cannot read log file {}. Skip the check as it's likely being written by an inflight instant.", + log.warn("Cannot read log file {}. Skip the check as it's likely being written by an inflight instant.", logFilePathStr, e); } finally { FileIOUtils.closeQuietly(reader); @@ -1756,11 +1749,11 @@ protected Pair startService() { long toSleepMs = cfg.minValidateIntervalSeconds * 1000 - (System.currentTimeMillis() - start); if (toSleepMs > 0) { - LOG.info("Last validate ran less than min validate interval: {} s, sleep: {} ms.", cfg.minValidateIntervalSeconds, toSleepMs); + log.info("Last validate ran less than min validate interval: {} s, sleep: {} ms.", cfg.minValidateIntervalSeconds, toSleepMs); Thread.sleep(toSleepMs); } } catch (HoodieValidationException e) { - LOG.error("Shutting down AsyncMetadataTableValidateService due to HoodieValidationException", e); + log.error("Shutting down AsyncMetadataTableValidateService due to HoodieValidationException", e); if (!cfg.ignoreFailed) { throw e; } @@ -1816,15 +1809,18 @@ public int compare(HoodieColumnRangeMetadata o1, HoodieColumnRangeMe * the same information regardless of whether metadata table is enabled, which is * verified in the {@link HoodieMetadataTableValidator}. */ + @Slf4j private static class HoodieMetadataValidationContext implements AutoCloseable, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieMetadataValidationContext.class); - private final Properties props; + @Getter private final HoodieTableMetaClient metaClient; + @Getter private final HoodieMetadataConfig metadataConfig; + @Getter private final HoodieSchema schema; private final HoodieTableFileSystemView fileSystemView; + @Getter private final HoodieTableMetadata tableMetadata; private final boolean enableMetadataTable; private List allColumnNameList; @@ -1865,10 +1861,10 @@ private HoodieTableFileSystemView getFileSystemView(HoodieEngineContext context, FileSystemViewStorageConfig viewConf, HoodieCommonConfig commonConfig) { switch (viewConf.getStorageType()) { case SPILLABLE_DISK: - LOG.debug("Creating Spillable Disk based Table View"); + log.debug("Creating Spillable Disk based Table View"); break; case MEMORY: - LOG.debug("Creating in-memory based Table View"); + log.debug("Creating in-memory based Table View"); break; default: throw new HoodieException("Unsupported storage type " + viewConf.getStorageType() + ", used with HoodieMetadataTableValidator"); @@ -1876,22 +1872,6 @@ private HoodieTableFileSystemView getFileSystemView(HoodieEngineContext context, return (HoodieTableFileSystemView) FileSystemViewManager.createViewManager(context, metadataConfig, viewConf, commonConfig, unused -> tableMetadata).getFileSystemView(metaClient); } - public HoodieTableMetaClient getMetaClient() { - return metaClient; - } - - public HoodieMetadataConfig getMetadataConfig() { - return metadataConfig; - } - - public HoodieSchema getSchema() { - return schema; - } - - public HoodieTableMetadata getTableMetadata() { - return tableMetadata; - } - public List getSortedLatestBaseFileList(String partitionPath) { return fileSystemView.getLatestBaseFiles(partitionPath) .sorted(new HoodieBaseFileComparator()).collect(Collectors.toList()); @@ -1909,7 +1889,7 @@ public List getSortedAllFileGroupList(String partitionPath) { @SuppressWarnings({"rawtypes", "unchecked"}) public List> getSortedColumnStatsList(String partitionPath, List fileNames, HoodieSchema readerSchema) { - LOG.info("All column names for getting column stats: {}", allColumnNameList); + log.info("All column names for getting column stats: {}", allColumnNameList); if (enableMetadataTable) { List> partitionFileNameList = fileNames.stream() .map(filename -> Pair.of(partitionPath, filename)).collect(Collectors.toList()); @@ -1993,11 +1973,11 @@ private Option readBloomFilterFromFile(String partitionPath, St .getFileReader(new HoodieConfig(), path)) { bloomFilter = fileReader.readBloomFilter(); if (bloomFilter == null) { - LOG.error("Failed to read bloom filter for {}", path); + log.error("Failed to read bloom filter for {}", path); return Option.empty(); } } catch (IOException e) { - LOG.error("Failed to get file reader for {} {}", path, e); + log.error("Failed to get file reader for {} {}", path, e); return Option.empty(); } return Option.of(BloomFilterData.builder() diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java index 88c2b33454e9c..6ea90039cc36a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java @@ -41,11 +41,10 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -139,9 +138,9 @@ * --backup-path backup_path * ``` */ +@Slf4j public class HoodieRepairTool { - private static final Logger LOG = LoggerFactory.getLogger(HoodieRepairTool.class); private static final String BACKUP_DIR_PREFIX = "hoodie_repair_backup_"; // Repair config private final Config cfg; @@ -175,41 +174,39 @@ public boolean run() { Option endingInstantOption = Option.ofNullable(cfg.endingInstantTime); if (startingInstantOption.isPresent() && endingInstantOption.isPresent()) { - LOG.info(String.format("Start repairing completed instants between %s and %s (inclusive)", - startingInstantOption.get(), endingInstantOption.get())); + log.info("Start repairing completed instants between {} and {} (inclusive)", + startingInstantOption.get(), endingInstantOption.get()); } else if (startingInstantOption.isPresent()) { - LOG.info(String.format("Start repairing completed instants from %s (inclusive)", - startingInstantOption.get())); + log.info("Start repairing completed instants from {} (inclusive)", startingInstantOption.get()); } else if (endingInstantOption.isPresent()) { - LOG.info(String.format("Start repairing completed instants till %s (inclusive)", - endingInstantOption.get())); + log.info("Start repairing completed instants till {} (inclusive)", endingInstantOption.get()); } else { - LOG.info("Start repairing all completed instants"); + log.info("Start repairing all completed instants"); } try { Mode mode = Mode.valueOf(cfg.runningMode.toUpperCase()); switch (mode) { case REPAIR: - LOG.info(" ****** The repair tool is in REPAIR mode, dangling data and logs files " + log.info(" ****** The repair tool is in REPAIR mode, dangling data and logs files " + "not belonging to any commit are going to be DELETED from the table ******"); if (checkBackupPathForRepair() < 0) { - LOG.error("Backup path check failed."); + log.error("Backup path check failed."); return false; } return doRepair(startingInstantOption, endingInstantOption, false); case DRY_RUN: - LOG.info(" ****** The repair tool is in DRY_RUN mode, " + log.info(" ****** The repair tool is in DRY_RUN mode, " + "only LOOKING FOR dangling data and log files from the table ******"); return doRepair(startingInstantOption, endingInstantOption, true); case UNDO: if (checkBackupPathAgainstBasePath() < 0) { - LOG.error("Backup path check failed."); + log.error("Backup path check failed."); return false; } return undoRepair(); default: - LOG.info("Unsupported running mode [" + cfg.runningMode + "], quit the job directly"); + log.info("Unsupported running mode [{}], quit the job directly", cfg.runningMode); return false; } } catch (IOException e) { @@ -229,7 +226,7 @@ public static void main(String[] args) { try { new HoodieRepairTool(jsc, cfg).run(); } catch (Throwable throwable) { - LOG.error("Fail to run table repair for " + cfg.basePath, throwable); + log.error("Fail to run table repair for {}", cfg.basePath, throwable); } finally { jsc.stop(); } @@ -264,8 +261,7 @@ static boolean copyFiles( } } catch (IOException e) { // Copy Fail - LOG.error(String.format("Copying file fails: source [%s], destination [%s]", - sourcePath, destPath)); + log.error("Copying file fails: source [{}], destination [{}]", sourcePath, destPath); } finally { results.add(success); } @@ -321,7 +317,7 @@ static boolean deleteFiles( try { success = fs.delete(new Path(basePath, relativeFilePath), false); } catch (IOException e) { - LOG.error("Failed to delete file {}", relativeFilePath); + log.error("Failed to delete file {}", relativeFilePath); } finally { results.add(success); } @@ -378,12 +374,12 @@ boolean doRepair( .collect(Collectors.toList()); if (relativeFilePathsToDelete.size() > 0) { if (!backupFiles(relativeFilePathsToDelete)) { - LOG.error("Error backing up dangling files. Exiting..."); + log.error("Error backing up dangling files. Exiting..."); return false; } return deleteFiles(context, cfg.basePath, relativeFilePathsToDelete); } - LOG.info(String.format("Table repair on %s is successful", cfg.basePath)); + log.info("Table repair on {} is successful", cfg.basePath); } return true; } @@ -398,14 +394,14 @@ boolean undoRepair() throws IOException { String backupPathStr = cfg.backupPath; StoragePath backupPath = new StoragePath(backupPathStr); if (!storage.exists(backupPath)) { - LOG.error("Cannot find backup path: " + backupPath); + log.error("Cannot find backup path: {}", backupPath); return false; } List allPartitionPaths = tableMetadata.getAllPartitionPaths(); if (allPartitionPaths.isEmpty()) { - LOG.error("Cannot get one partition path since there is no partition available"); + log.error("Cannot get one partition path since there is no partition available"); return false; } @@ -446,7 +442,7 @@ int checkBackupPathForRepair() throws IOException { StoragePath backupPath = new StoragePath(cfg.backupPath); if (metaClient.getStorage().exists(backupPath) && metaClient.getStorage().listDirectEntries(backupPath).size() > 0) { - LOG.error(String.format("Cannot use backup path %s: it is not empty", cfg.backupPath)); + log.error("Cannot use backup path {}: it is not empty", cfg.backupPath); return -1; } @@ -461,13 +457,12 @@ int checkBackupPathForRepair() throws IOException { */ int checkBackupPathAgainstBasePath() { if (cfg.backupPath == null) { - LOG.error("Backup path is not configured"); + log.error("Backup path is not configured"); return -1; } if (cfg.backupPath.contains(cfg.basePath)) { - LOG.error(String.format("Cannot use backup path %s: it resides in the base path %s", - cfg.backupPath, cfg.basePath)); + log.error("Cannot use backup path {}: it resides in the base path {}", cfg.backupPath, cfg.basePath); return -1; } return 0; @@ -502,11 +497,11 @@ boolean restoreFiles(List relativeFilePaths) { private void printRepairInfo( List instantTimesToRepair, List>> instantsWithDanglingFiles) { int numInstantsToRepair = instantsWithDanglingFiles.size(); - LOG.info("Number of instants verified based on the base and log files: {}", instantTimesToRepair.size()); - LOG.info("Instant timestamps: {}", instantTimesToRepair); - LOG.info("Number of instants to repair: {}", numInstantsToRepair); + log.info("Number of instants verified based on the base and log files: {}", instantTimesToRepair.size()); + log.info("Instant timestamps: {}", instantTimesToRepair); + log.info("Number of instants to repair: {}", numInstantsToRepair); if (numInstantsToRepair > 0) { - instantsWithDanglingFiles.forEach(e -> LOG.info(" ** Removing files: {}", e.getValue())); + instantsWithDanglingFiles.forEach(e -> log.info(" ** Removing files: {}", e.getValue())); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java index 38794c5345fde..78e72ee31c3bb 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java @@ -49,6 +49,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -62,8 +63,6 @@ import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -79,6 +78,7 @@ /** * Export the latest records of Hudi dataset to a set of external files (e.g., plain parquet files). */ +@Slf4j public class HoodieSnapshotExporter { @FunctionalInterface @@ -88,8 +88,6 @@ public interface Partitioner { } - private static final Logger LOG = LoggerFactory.getLogger(HoodieSnapshotExporter.class); - public static class OutputFormatValidator implements IValueValidator { public static final String HUDI = "hudi"; @@ -159,8 +157,7 @@ public void export(JavaSparkContext jsc, Config cfg) throws IOException { .orElseThrow(() -> { throw new HoodieSnapshotExporterException("No commits present. Nothing to snapshot."); }); - LOG.info(String.format("Starting to snapshot latest version files which are also no-late-than %s.", - latestCommitTimestamp)); + log.info("Starting to snapshot latest version files which are also no-late-than {}.", latestCommitTimestamp); final HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc); final List partitions = getPartitions(engineContext, cfg, HoodieStorageUtils.getStorage( @@ -169,7 +166,7 @@ public void export(JavaSparkContext jsc, Config cfg) throws IOException { if (partitions.isEmpty()) { throw new HoodieSnapshotExporterException("The source dataset has 0 partition to snapshot."); } - LOG.info(String.format("The job needs to export %d partitions.", partitions.size())); + log.info("The job needs to export {} partitions.", partitions.size()); if (cfg.outputFormat.equals(OutputFormatValidator.HUDI)) { exportAsHudi(jsc, sourceFs, cfg, partitions, latestCommitTimestamp, tableMetadata); @@ -194,7 +191,7 @@ private List getPartitions(HoodieEngineContext engineContext, Config cfg private void createSuccessTag(FileSystem fs, Config cfg) throws IOException { Path successTagPath = new Path(cfg.targetOutputPath + "/_SUCCESS"); if (!fs.exists(successTagPath)) { - LOG.info(String.format("Creating _SUCCESS under target output path: %s", cfg.targetOutputPath)); + log.info("Creating _SUCCESS under target output path: {}", cfg.targetOutputPath); fs.createNewFile(successTagPath); } } @@ -285,7 +282,7 @@ private void exportAsHudi(JavaSparkContext jsc, FileSystem sourceFs, }, parallelism); // Also copy the .commit files - LOG.info(String.format("Copying .commit files which are no-late-than %s.", latestCommitTimestamp)); + log.info("Copying .commit files which are no-late-than {}.", latestCommitTimestamp); List commitFilesListToCopy = Arrays.stream(sourceFs.listStatus(new Path(cfg.sourceBasePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTableMetaClient.TIMELINEFOLDER_NAME))) .filter(fileStatus -> { @@ -344,7 +341,7 @@ public static void main(String[] args) throws IOException { } JavaSparkContext jsc = UtilHelpers.buildSparkContext("Hoodie-snapshot-exporter", "local[*]", cfg.enableHiveSupport); - LOG.info("Initializing spark job."); + log.info("Initializing spark job."); try { new HoodieSnapshotExporter().export(jsc, cfg); @@ -359,13 +356,13 @@ public static boolean areTransformerOptionsValid(Config config) { switch (config.transformerClassName) { case "org.apache.hudi.utilities.transform.SqlQueryBasedTransformer": if (StringUtils.isNullOrEmpty(config.transformerSql)) { - LOG.error("--transformer-sql is required when using SqlQueryBasedTransformer"); + log.error("--transformer-sql is required when using SqlQueryBasedTransformer"); valid = false; } break; case "org.apache.hudi.utilities.transform.SqlFileBasedTransformer": if (StringUtils.isNullOrEmpty(config.transformerSqlFile)) { - LOG.error("--transformer-sql-file is required when using SqlFileBasedTransformer"); + log.error("--transformer-sql-file is required when using SqlFileBasedTransformer"); valid = false; } break; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieTTLJob.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieTTLJob.java index 519ce482d6eca..3d61449760286 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieTTLJob.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieTTLJob.java @@ -31,10 +31,9 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; @@ -43,9 +42,9 @@ /** * Utility class to run TTL management. */ +@Slf4j public class HoodieTTLJob { - private static final Logger LOG = LoggerFactory.getLogger(HoodieTTLJob.class); private final Config cfg; private final TypedProperties props; private final JavaSparkContext jsc; @@ -61,7 +60,7 @@ public HoodieTTLJob(JavaSparkContext jsc, Config cfg, TypedProperties props, Hoo this.jsc = jsc; this.props = props; this.metaClient = metaClient; - LOG.info("Creating TTL job with configs : " + props.toString()); + log.info("Creating TTL job with configs : {}", props.toString()); // Disable async cleaning, will trigger synchronous cleaning manually. this.props.put(HoodieCleanConfig.ASYNC_CLEAN.key(), false); if (this.metaClient.getTableConfig().isMetadataTableAvailable()) { @@ -129,7 +128,7 @@ public static void main(String[] args) { SparkAdapterSupport$.MODULE$.sparkAdapter().stopSparkContext(jssc, exitCode); } - LOG.info("Hoodie TTL job ran successfully"); + log.info("Hoodie TTL job ran successfully"); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java index 2f7636bfa5b3d..29eab4b472056 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java @@ -41,12 +41,11 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; import com.codahale.metrics.UniformReservoir; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -97,10 +96,10 @@ * --base-path \ * --num-days */ +@Slf4j public class TableSizeStats implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(TableSizeStats.class); // Date formatter for parsing partition dates (example: 2023/5/5/ or 2023-5-5). private static final DateTimeFormatter DATE_FORMATTER = @@ -138,6 +137,7 @@ private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cf } public static class Config implements Serializable { + @Parameter(names = {"--base-path", "-bp"}, description = "Base path for the table", required = false) public String basePath = null; @@ -241,9 +241,9 @@ public static void main(String[] args) { TableSizeStats tableSizeStats = new TableSizeStats(jsc, cfg); tableSizeStats.run(); } catch (TableNotFoundException e) { - LOG.warn("The Hudi data table is not found: [{}].", cfg.basePath, e); + log.warn("The Hudi data table is not found: [{}].", cfg.basePath, e); } catch (Throwable throwable) { - LOG.error("Failed to get table size stats for {}", cfg, throwable); + log.error("Failed to get table size stats for {}", cfg, throwable); } finally { jsc.stop(); } @@ -251,8 +251,8 @@ public static void main(String[] args) { public void run() { try { - LOG.info(cfg.toString()); - LOG.info(" ****** Fetching table size stats ******"); + log.info(cfg.toString()); + log.info(" ****** Fetching table size stats ******"); // Determine starting and ending date intervals for filtering data files. LocalDate[] dateInterval = getUserSpecifiedDateInterval(cfg); @@ -276,7 +276,7 @@ public void run() { private void logTableStats(String basePath, LocalDate[] dateInterval) throws IOException { - LOG.info("Processing table {}", basePath); + log.info("Processing table {}", basePath); HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder() .enable(isMetadataEnabled(basePath, jsc)) .build(); @@ -351,7 +351,7 @@ private void logTableStats(String basePath, LocalDate[] dateInterval) throws IOE logStats("Table stats [path: " + basePath + "]", tableHistogram); } else { // Display only total talbe size - LOG.info("Total size: {}", getFileSizeUnit(Arrays.stream(tableHistogram.getSnapshot().getValues()).sum())); + log.info("Total size: {}", getFileSizeUnit(Arrays.stream(tableHistogram.getSnapshot().getValues()).sum())); } } @@ -378,7 +378,7 @@ private static List getFilePaths(String propsPath, Configuration hadoopC line = reader.readLine(); } } catch (IOException ioe) { - LOG.error("Error reading in properties from dfs from file." + propsPath); + log.error("Error reading in properties from dfs from file. {}", propsPath); throw new HoodieIOException("Cannot read properties from dfs from file " + propsPath, ioe); } return filePaths; @@ -390,12 +390,12 @@ private static LocalDate[] getUserSpecifiedDateInterval(Config cfg) { if (cfg.endDate != null) { try { endDate = LocalDate.parse(cfg.endDate, DATE_FORMATTER); - LOG.info("Setting ending date to {}. ", endDate); + log.info("Setting ending date to {}.", endDate); } catch (DateTimeParseException dtpe) { throw new HoodieException("Unable to parse --end-date. ", dtpe); } } else { - LOG.info("End date is not specified: {}.", endDate); + log.info("End date is not specified: {}.", endDate); } // Set startDate to null by default. @@ -404,14 +404,14 @@ private static LocalDate[] getUserSpecifiedDateInterval(Config cfg) { // Set startDate to cfg.startDate if specified. cfg.startDate takes priority over cfg.numDays if both are specified. if (cfg.startDate != null) { startDate = LocalDate.parse(cfg.startDate, DATE_FORMATTER); - LOG.info("Setting starting date to {}.", startDate); + log.info("Setting starting date to {}.", startDate); } else { if (cfg.numDays == 0) { - LOG.info("Start date not specified: {}.", startDate); + log.info("Start date not specified: {}.", startDate); } else if (cfg.numDays > 0) { endDate = LocalDate.now(); startDate = endDate.minusDays(cfg.numDays); - LOG.info("Setting starting date to {} ({} - {} days). ", startDate, endDate, cfg.numDays); + log.info("Setting starting date to {} ({} - {} days). ", startDate, endDate, cfg.numDays); } else { throw new HoodieException("--num-days must specify a positive value."); } @@ -422,7 +422,7 @@ private static LocalDate[] getUserSpecifiedDateInterval(Config cfg) { throw new HoodieException("Starting date must be before ending date. Start Date: " + startDate + ", End Date: " + endDate); } - return startDate == null && endDate == null ? null : new LocalDate[]{startDate, endDate}; + return startDate == null && endDate == null ? null : new LocalDate[] {startDate, endDate}; } @Nullable @@ -442,7 +442,7 @@ private static LocalDate getPartitionDate(String partition) { try { return LocalDate.parse(dateString, DATE_FORMATTER); } catch (DateTimeParseException dtpe) { - LOG.error("Partition name {} must conform to date format if --start-date, --end-date, or --num-days are specified. ", partition, dtpe); + log.error("Partition name {} must conform to date format if --start-date, --end-date, or --num-days are specified. ", partition, dtpe); } return partitionDate; } @@ -458,17 +458,17 @@ private static String getFileSizeUnit(double size) { } private static void logStats(String header, Histogram histogram) { - LOG.info(header); + log.info(header); Snapshot snapshot = histogram.getSnapshot(); - LOG.info("Number of files: {}", snapshot.size()); - LOG.info("Total size: {}", getFileSizeUnit(Arrays.stream(snapshot.getValues()).sum())); - LOG.info("Minimum file size: {}", getFileSizeUnit(snapshot.getMin())); - LOG.info("Maximum file size: {}", getFileSizeUnit(snapshot.getMax())); - LOG.info("Average file size: {}", getFileSizeUnit(snapshot.getMean())); - LOG.info("Median file size: {}", getFileSizeUnit(snapshot.getMedian())); - LOG.info("P50 file size: {}", getFileSizeUnit(snapshot.getValue(0.5))); - LOG.info("P90 file size: {}", getFileSizeUnit(snapshot.getValue(0.9))); - LOG.info("P95 file size: {}", getFileSizeUnit(snapshot.getValue(0.95))); - LOG.info("P99 file size: {}", getFileSizeUnit(snapshot.getValue(0.99))); + log.info("Number of files: {}", snapshot.size()); + log.info("Total size: {}", getFileSizeUnit(Arrays.stream(snapshot.getValues()).sum())); + log.info("Minimum file size: {}", getFileSizeUnit(snapshot.getMin())); + log.info("Maximum file size: {}", getFileSizeUnit(snapshot.getMax())); + log.info("Average file size: {}", getFileSizeUnit(snapshot.getMean())); + log.info("Median file size: {}", getFileSizeUnit(snapshot.getMedian())); + log.info("P50 file size: {}", getFileSizeUnit(snapshot.getValue(0.5))); + log.info("P90 file size: {}", getFileSizeUnit(snapshot.getValue(0.9))); + log.info("P95 file size: {}", getFileSizeUnit(snapshot.getValue(0.95))); + log.info("P99 file size: {}", getFileSizeUnit(snapshot.getValue(0.99))); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java index 9d04641f71b49..22bf3f4b4dcb2 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java @@ -76,6 +76,7 @@ import org.apache.hudi.utilities.transform.ErrorTableAwareChainedTransformer; import org.apache.hudi.utilities.transform.Transformer; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -93,8 +94,6 @@ import org.apache.spark.sql.jdbc.JdbcDialects; import org.apache.spark.sql.types.StructType; import org.apache.spark.util.LongAccumulator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.IOException; @@ -126,6 +125,7 @@ /** * Bunch of helper methods. */ +@Slf4j public class UtilHelpers { public static final String EXECUTE = "execute"; @@ -133,8 +133,6 @@ public class UtilHelpers { public static final String SCHEDULE_AND_EXECUTE = "scheduleandexecute"; public static final String PURGE_PENDING_INSTANT = "purge_pending_instant"; - private static final Logger LOG = LoggerFactory.getLogger(UtilHelpers.class); - public static HoodieRecordMerger createRecordMerger(Properties props) { return HoodieRecordUtils.createRecordMerger(null, EngineType.SPARK, StringUtils.split(ConfigUtils.getStringWithAltKeys(props, HoodieWriteConfig.RECORD_MERGE_IMPL_CLASSES, null), ","), @@ -142,7 +140,7 @@ public static HoodieRecordMerger createRecordMerger(Properties props) { } public static Source createSource(String sourceClass, TypedProperties cfg, JavaSparkContext jssc, - SparkSession sparkSession, HoodieIngestionMetrics metrics, StreamContext streamContext) throws IOException { + SparkSession sparkSession, HoodieIngestionMetrics metrics, StreamContext streamContext) throws IOException { // All possible constructors. Class[] constructorArgsStreamContextMetrics = new Class[] {TypedProperties.class, JavaSparkContext.class, SparkSession.class, HoodieIngestionMetrics.class, StreamContext.class}; Class[] constructorArgsStreamContext = new Class[] {TypedProperties.class, JavaSparkContext.class, SparkSession.class, StreamContext.class}; @@ -168,7 +166,7 @@ public static Source createSource(String sourceClass, TypedProperties cfg, JavaS String constructorSignature = Arrays.stream(constructor.getLeft()) .map(Class::getSimpleName) .collect(Collectors.joining(", ", "[", "]")); - LOG.error("Unexpected error while loading source class {} with constructor signature {}", sourceClass, constructorSignature, e); + log.error("Unexpected error while loading source class {} with constructor signature {}", sourceClass, constructorSignature, e); } catch (Throwable t) { throw new IOException("Could not load source class due to unexpected error " + sourceClass, t); } @@ -197,7 +195,7 @@ public static JsonKafkaSourcePostProcessor createJsonKafkaSourcePostProcessor(St } public static SchemaProvider createSchemaProvider(String schemaProviderClass, TypedProperties cfg, - JavaSparkContext jssc) throws IOException { + JavaSparkContext jssc) throws IOException { try { return StringUtils.isNullOrEmpty(schemaProviderClass) ? null : (SchemaProvider) ReflectionUtils.loadClass(schemaProviderClass, cfg, jssc); @@ -233,7 +231,7 @@ public static StructType getSourceSchema(SchemaProvider schemaProvider) { } public static Option createTransformer(Option> classNamesOpt, Supplier> sourceSchemaSupplier, - boolean isErrorTableWriterEnabled) throws IOException { + boolean isErrorTableWriterEnabled) throws IOException { try { Function, Transformer> chainedTransformerFunction = classNames -> @@ -255,13 +253,13 @@ public static InitialCheckPointProvider createInitialCheckpointProvider( } public static DFSPropertiesConfiguration readConfig(Configuration hadoopConfig, - Path cfgPath, - List overriddenProps) { + Path cfgPath, + List overriddenProps) { StoragePath storagePath = convertToStoragePath(cfgPath); DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration(hadoopConfig, storagePath); try { if (!overriddenProps.isEmpty()) { - LOG.info("Adding overridden properties to file properties."); + log.info("Adding overridden properties to file properties."); conf.addPropsFromStream(new BufferedReader(new StringReader(String.join("\n", overriddenProps))), storagePath); } } catch (IOException ioe) { @@ -275,7 +273,7 @@ public static DFSPropertiesConfiguration getConfig(List overriddenProps) DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration(); try { if (!overriddenProps.isEmpty()) { - LOG.info("Adding overridden properties to file properties."); + log.info("Adding overridden properties to file properties."); conf.addPropsFromStream(new BufferedReader(new StringReader(String.join("\n", overriddenProps))), null); } } catch (IOException ioe) { @@ -289,7 +287,7 @@ public static TypedProperties buildProperties(Configuration hadoopConf, String p return StringUtils.isNullOrEmpty(propsFilePath) ? UtilHelpers.buildProperties(props) : UtilHelpers.readConfig(hadoopConf, new Path(propsFilePath), props) - .getProps(true); + .getProps(true); } public static TypedProperties buildProperties(List props) { @@ -310,7 +308,7 @@ public static void validateAndAddProperties(String[] configs, SparkLauncher spar /** * Parse Schema from file. * - * @param fs File System + * @param fs File System * @param schemaFile Schema File */ public static String parseSchema(FileSystem fs, String schemaFile) throws Exception { @@ -412,9 +410,9 @@ public static JavaSparkContext getJavaSparkContextFromSparkConf(SparkConf sparkC /** * Build Hoodie write client. * - * @param jsc Java Spark Context - * @param basePath Base Path - * @param schemaStr Schema + * @param jsc Java Spark Context + * @param basePath Base Path + * @param schemaStr Schema * @param parallelism Parallelism */ public static SparkRDDWriteClient createHoodieClient(JavaSparkContext jsc, String basePath, String schemaStr, @@ -440,14 +438,14 @@ public static int handleErrors(JavaSparkContext jsc, String instantTime, JavaRDD writeResponse.foreach(writeStatus -> { if (writeStatus.hasErrors()) { errors.add(1); - LOG.error("Error processing records :writeStatus:{}", writeStatus.getStat().toString()); + log.error("Error processing records :writeStatus:{}", writeStatus.getStat().toString()); } }); if (errors.value() == 0) { - LOG.info("Table imported into hoodie with {} instant time.", instantTime); + log.info("Table imported into hoodie with {} instant time.", instantTime); return 0; } - LOG.error("Import failed with {} errors.", errors.value()); + log.error("Import failed with {} errors.", errors.value()); return -1; } @@ -455,11 +453,11 @@ public static int handleErrors(HoodieCommitMetadata metadata, String instantTime List writeStats = metadata.getWriteStats(); long errorsCount = writeStats.stream().mapToLong(HoodieWriteStat::getTotalWriteErrors).sum(); if (errorsCount == 0) { - LOG.info("Finish job with {} instant time.", instantTime); + log.info("Finish job with {} instant time.", instantTime); return 0; } - LOG.error("Job failed with {} errors.", errorsCount); + log.error("Job failed with {} errors.", errorsCount); return -1; } @@ -571,7 +569,7 @@ public static SchemaProvider getOriginalSchemaProvider(SchemaProvider schemaProv } public static SchemaProvider wrapSchemaProviderWithPostProcessor(SchemaProvider provider, - TypedProperties cfg, JavaSparkContext jssc, List transformerClassNames) { + TypedProperties cfg, JavaSparkContext jssc, List transformerClassNames) { if (provider == null) { return null; @@ -601,16 +599,16 @@ public static SchemaProvider getSchemaProviderForKafkaSource(SchemaProvider prov } public static SchemaProvider createRowBasedSchemaProvider(StructType structType, - TypedProperties cfg, - JavaSparkContext jssc) { + TypedProperties cfg, + JavaSparkContext jssc) { SchemaProvider rowSchemaProvider = new RowBasedSchemaProvider(structType); return wrapSchemaProviderWithPostProcessor(rowSchemaProvider, cfg, jssc, null); } public static Option getLatestTableSchema(JavaSparkContext jssc, - HoodieStorage storage, - String basePath, - HoodieTableMetaClient tableMetaClient) { + HoodieStorage storage, + String basePath, + HoodieTableMetaClient tableMetaClient) { try { if (FSUtils.isTableExists(basePath, storage)) { TableSchemaResolver tableSchemaResolver = new TableSchemaResolver(tableMetaClient); @@ -618,7 +616,7 @@ public static Option getLatestTableSchema(JavaSparkContext jssc, return tableSchemaResolver.getTableSchemaFromLatestCommit(false); } } catch (Exception e) { - LOG.warn("Failed to fetch latest table's schema", e); + log.warn("Failed to fetch latest table's schema", e); } return Option.empty(); @@ -655,6 +653,7 @@ public static StructType extractSchemaFromDataset(Dataset dataset, TypedProperti @FunctionalInterface public interface CheckedSupplier { + T get() throws Throwable; } @@ -665,7 +664,7 @@ public static int retry(int maxRetryCount, CheckedSupplier supplier, St ret = supplier.get(); } while (ret != 0 && maxRetryCount-- > 0); } catch (Throwable t) { - LOG.error(errorMessage, t); + log.error(errorMessage, t); throw new RuntimeException("Failed in retry", t); } return ret; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java index df466a0a7dd6d..382fe27b5d1d5 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java @@ -25,13 +25,12 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieWriteConfig; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Properties; @@ -45,10 +44,9 @@ /** * Kafka implementation of {@link HoodieWriteCommitCallback}. */ +@Slf4j public class HoodieWriteCommitKafkaCallback implements HoodieWriteCommitCallback { - private static final Logger LOG = LoggerFactory.getLogger(HoodieWriteCommitKafkaCallback.class); - private final HoodieConfig hoodieConfig; private final String bootstrapServers; private final String topic; @@ -66,9 +64,9 @@ public void call(HoodieWriteCommitCallbackMessage callbackMessage) { try (KafkaProducer producer = createProducer(hoodieConfig)) { ProducerRecord record = buildProducerRecord(hoodieConfig, callbackMsg); producer.send(record).get(); - LOG.info("Send callback message succeed"); + log.info("Send callback message succeed"); } catch (Exception e) { - LOG.error("Send kafka callback msg failed : ", e); + log.error("Send kafka callback msg failed : ", e); } } @@ -94,8 +92,8 @@ public KafkaProducer createProducer(HoodieConfig hoodieConfig) { kafkaProducerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - LOG.debug("Callback kafka producer init with configs: " - + HoodieWriteCommitCallbackUtil.convertToJsonString(kafkaProducerProps)); + log.debug("Callback kafka producer init with configs: {}", + HoodieWriteCommitCallbackUtil.convertToJsonString(kafkaProducerProps)); return new KafkaProducer(kafkaProducerProps); } @@ -138,11 +136,11 @@ private static class ProducerSendCallback implements Callback { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (null != metadata) { - LOG.info("message offset={} partition={} timestamp={} topic={}", + log.info("message offset={} partition={} timestamp={} topic={}", metadata.offset(), metadata.partition(), metadata.timestamp(), metadata.topic()); } if (null != exception) { - LOG.error("Send kafka callback msg failed : ", exception); + log.error("Send kafka callback msg failed: ", exception); } } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/pulsar/HoodieWriteCommitPulsarCallback.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/pulsar/HoodieWriteCommitPulsarCallback.java index af4bbbf49e46f..379aba2d19ecd 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/pulsar/HoodieWriteCommitPulsarCallback.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/pulsar/HoodieWriteCommitPulsarCallback.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieWriteConfig; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -32,8 +33,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; @@ -56,10 +55,9 @@ /** * Pulsar implementation of {@link HoodieWriteCommitCallback}. */ +@Slf4j public class HoodieWriteCommitPulsarCallback implements HoodieWriteCommitCallback, Closeable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieWriteCommitPulsarCallback.class); - private final String serviceUrl; private final String topic; @@ -85,9 +83,9 @@ public void call(HoodieWriteCommitCallbackMessage callbackMessage) { String callbackMsg = HoodieWriteCommitCallbackUtil.convertToJsonString(callbackMessage); try { producer.newMessage().key(callbackMessage.getTableName()).value(callbackMsg).send(); - LOG.info("Send callback message succeed"); + log.info("Send callback message succeed"); } catch (Exception e) { - LOG.error("Send pulsar callback msg failed : ", e); + log.error("Send pulsar callback msg failed: ", e); } } @@ -165,7 +163,7 @@ public void close() throws IOException { try { producer.close(); } catch (Throwable t) { - LOG.warn("Could not properly close the producer.", t); + log.warn("Could not properly close the producer.", t); } } @@ -173,7 +171,7 @@ public void close() throws IOException { try { client.close(); } catch (Throwable t) { - LOG.warn("Could not properly close the client.", t); + log.warn("Could not properly close the client.", t); } } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java index 365a01b604094..a2ffb4878454e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java @@ -23,6 +23,7 @@ import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.KafkaAvroDeserializer; +import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.kafka.common.errors.SerializationException; @@ -35,13 +36,11 @@ /** * Extending {@link KafkaAvroSchemaDeserializer} as we need to be able to inject reader schema during deserialization. */ +@NoArgsConstructor public class KafkaAvroSchemaDeserializer extends KafkaAvroDeserializer { private Schema sourceSchema; - public KafkaAvroSchemaDeserializer() { - } - public KafkaAvroSchemaDeserializer(SchemaRegistryClient client, Map props) { super(client, props); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java index 04174f740767d..57f3e7910c8e2 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java @@ -20,14 +20,14 @@ import org.apache.hudi.exception.HoodieException; +import lombok.NoArgsConstructor; + /** * The root exception class for any failure with {@link HoodieIngestionService}. */ +@NoArgsConstructor public class HoodieIngestionException extends HoodieException { - public HoodieIngestionException() { - } - public HoodieIngestionException(String message) { super(message); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java index 5f2c15f090c44..2769d851b107c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java @@ -27,8 +27,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.utilities.streamer.PostWriteTerminationStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -40,10 +39,9 @@ /** * A generic service to facilitate running data ingestion. */ +@Slf4j public abstract class HoodieIngestionService extends HoodieAsyncService { - private static final Logger LOG = LoggerFactory.getLogger(HoodieIngestionService.class); - protected HoodieIngestionConfig ingestionConfig; public HoodieIngestionService(HoodieIngestionConfig ingestionConfig) { @@ -59,18 +57,18 @@ public HoodieIngestionService(HoodieIngestionConfig ingestionConfig) { */ public void startIngestion() { if (ingestionConfig.getBoolean(INGESTION_IS_CONTINUOUS)) { - LOG.info("Ingestion service starts running in continuous mode"); + log.info("Ingestion service starts running in continuous mode"); start(this::onIngestionCompletes); try { waitForShutdown(); } catch (Exception e) { throw new HoodieIngestionException("Ingestion service was shut down with exception.", e); } - LOG.info("Ingestion service (continuous mode) has been shut down."); + log.info("Ingestion service (continuous mode) has been shut down."); } else { - LOG.info("Ingestion service starts running in run-once mode"); + log.info("Ingestion service starts running in run-once mode"); ingestOnce(); - LOG.info("Ingestion service (run-once mode) has been shut down."); + log.info("Ingestion service (run-once mode) has been shut down."); } } @@ -121,8 +119,8 @@ protected void sleepBeforeNextIngestion(long ingestionStartEpochMillis) { long minSyncInternalSeconds = ingestionConfig.getLongOrDefault(INGESTION_MIN_SYNC_INTERNAL_SECONDS); long sleepMs = minSyncInternalSeconds * 1000 - (System.currentTimeMillis() - ingestionStartEpochMillis); if (sleepMs > 0) { - LOG.info(String.format("Last ingestion took less than min sync interval: %d s; sleep for %.2f s", - minSyncInternalSeconds, sleepMs / 1000.0)); + log.info("Last ingestion took less than min sync interval: {} s; sleep for {} s", + minSyncInternalSeconds, sleepMs / 1000.0); Thread.sleep(sleepMs); } } catch (InterruptedException e) { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ArchiveTask.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ArchiveTask.java index fe5decd7005c1..03d67dd2d9694 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ArchiveTask.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ArchiveTask.java @@ -25,21 +25,20 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.utilities.UtilHelpers; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Archive task to run in TableServicePipeline. * * @see HoodieMultiTableServicesMain */ +@Slf4j class ArchiveTask extends TableServiceTask { - private static final Logger LOG = LoggerFactory.getLogger(ArchiveTask.class); @Override void run() { - LOG.info("Run Archive with props: " + props); + log.info("Run Archive with props: {}", props); HoodieWriteConfig hoodieCfg = HoodieWriteConfig.newBuilder().withPath(basePath).withProps(props).build(); try (SparkRDDWriteClient client = new SparkRDDWriteClient<>(new HoodieSparkEngineContext(jsc), hoodieCfg)) { UtilHelpers.retry(retry, () -> { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ClusteringTask.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ClusteringTask.java index 66efbd475dc47..34d585b903d61 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ClusteringTask.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/ClusteringTask.java @@ -23,6 +23,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.utilities.HoodieClusteringJob; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import org.apache.spark.api.java.JavaSparkContext; /** @@ -71,6 +73,7 @@ public static Builder newBuilder() { /** * Builder class for {@link ClusteringTask}. */ + @NoArgsConstructor(access = AccessLevel.PRIVATE) public static final class Builder { /** @@ -110,9 +113,6 @@ public static final class Builder { */ private HoodieTableMetaClient metaClient; - private Builder() { - } - public Builder withProps(TypedProperties props) { this.props = props; return this; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/HoodieMultiTableServicesMain.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/HoodieMultiTableServicesMain.java index aade3d0a48546..9d4af77d3f422 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/HoodieMultiTableServicesMain.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/HoodieMultiTableServicesMain.java @@ -28,11 +28,10 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -51,8 +50,9 @@ /** * Main function for executing multi-table services. */ +@Slf4j public class HoodieMultiTableServicesMain { - private static final Logger LOG = LoggerFactory.getLogger(HoodieMultiTableServicesMain.class); + final Config cfg; final TypedProperties props; @@ -107,7 +107,7 @@ public HoodieMultiTableServicesMain(JavaSparkContext jsc, Config cfg) { } public void startServices() throws ExecutionException, InterruptedException { - LOG.info("StartServices Config: " + cfg); + log.info("StartServices Config: {}", cfg); List tablePaths; if (cfg.autoDiscovery) { // We support defining multi base paths @@ -118,7 +118,7 @@ public void startServices() throws ExecutionException, InterruptedException { } else { tablePaths = MultiTableServiceUtils.getTablesToBeServedFromProps(jsc, props); } - LOG.info("All table paths: " + String.join(",", tablePaths)); + log.info("All table paths: {}", String.join(",", tablePaths)); if (cfg.batch) { batchRunTableServices(tablePaths); } else { @@ -253,7 +253,7 @@ public static void main(String[] args) { try { new HoodieMultiTableServicesMain(jsc, cfg).startServices(); } catch (Throwable throwable) { - LOG.error("Fail to run table services, ", throwable); + log.error("Fail to run table services, ", throwable); } finally { jsc.stop(); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/MultiTableServiceUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/MultiTableServiceUtils.java index 41bd7248f0f3b..3c1f1453f54eb 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/MultiTableServiceUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/multitable/MultiTableServiceUtils.java @@ -29,13 +29,12 @@ import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.utilities.UtilHelpers; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -49,8 +48,8 @@ /** * Utils for executing multi-table services. */ +@Slf4j public class MultiTableServiceUtils { - private static final Logger LOG = LoggerFactory.getLogger(MultiTableServiceUtils.class); public static class Constants { public static final String TABLES_TO_BE_SERVED_PROP = "hoodie.tableservice.tablesToServe"; @@ -79,7 +78,7 @@ public static List getTablesToBeServedFromProps(JavaSparkContext jsc, Ty return true; } else { // Log the wrong path in console. - LOG.info("Hoodie table not found in path {}, skip", tablePath); + log.info("Hoodie table not found in path {}, skip", tablePath); return false; } }).collect(Collectors.toList()); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java index 330eece72e9ab..39efad86c3d35 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java @@ -40,10 +40,9 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; import com.codahale.metrics.UniformReservoir; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; @@ -62,10 +61,10 @@ import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; +@Slf4j public class TimelineServerPerf implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(TimelineServerPerf.class); private final Config cfg; private transient TimelineService timelineServer; private final boolean useExternalTimelineServer; @@ -84,10 +83,10 @@ public TimelineServerPerf(Config cfg) { private void setHostAddrFromSparkConf(SparkConf sparkConf) { String hostAddr = sparkConf.get("spark.driver.host", null); if (hostAddr != null) { - LOG.info("Overriding hostIp to ({}) found in spark-conf. It was {}", hostAddr, this.hostAddr); + log.info("Overriding hostIp to ({}) found in spark-conf. It was {}", hostAddr, this.hostAddr); this.hostAddr = hostAddr; } else { - LOG.warn("Unable to find driver bind address from spark config"); + log.warn("Unable to find driver bind address from spark config"); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/DelegatingSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/DelegatingSchemaProvider.java index 5662c46169ce1..dd7924a62a567 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/DelegatingSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/DelegatingSchemaProvider.java @@ -21,11 +21,13 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.schema.HoodieSchema; +import lombok.Getter; import org.apache.spark.api.java.JavaSparkContext; /** * SchemaProvider which uses separate Schema Providers for source and target. */ +@Getter public final class DelegatingSchemaProvider extends SchemaProvider { private final SchemaProvider sourceSchemaProvider; @@ -48,12 +50,4 @@ public HoodieSchema getSourceHoodieSchema() { public HoodieSchema getTargetHoodieSchema() { return targetSchemaProvider.getTargetHoodieSchema(); } - - public SchemaProvider getSourceSchemaProvider() { - return sourceSchemaProvider; - } - - public SchemaProvider getTargetSchemaProvider() { - return targetSchemaProvider; - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java index 23c9e76c67e17..542c983ba1677 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java @@ -31,6 +31,7 @@ import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.json.JsonSchema; +import lombok.Getter; import org.apache.avro.Schema; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -54,6 +55,7 @@ public class FilebasedSchemaProvider extends SchemaProvider { private final String sourceFile; private final String targetFile; + @Getter protected Schema sourceSchema; protected Schema targetSchema; @@ -74,11 +76,6 @@ private Schema parseSchema(String schemaFile) { return readSchemaFromFile(schemaFile, this.fs, config); } - @Override - public Schema getSourceSchema() { - return sourceSchema; - } - @Override public Schema getTargetSchema() { if (targetSchema != null) { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/HiveSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/HiveSchemaProvider.java index 6c58fb2739dc4..7ec566b33db3c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/HiveSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/HiveSchemaProvider.java @@ -25,6 +25,7 @@ import org.apache.hudi.utilities.config.HiveSchemaProviderConfig; import org.apache.hudi.utilities.exception.HoodieSchemaFetchException; +import lombok.Getter; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; @@ -42,10 +43,11 @@ /** * A schema provider to get data schema through user specified hive table. */ +@Getter public class HiveSchemaProvider extends SchemaProvider { - private final HoodieSchema sourceSchema; - private HoodieSchema targetSchema; + private final HoodieSchema sourceHoodieSchema; + private HoodieSchema targetHoodieSchema; public HiveSchemaProvider(TypedProperties props, JavaSparkContext jssc) { super(props, jssc); @@ -58,7 +60,7 @@ public HiveSchemaProvider(TypedProperties props, JavaSparkContext jssc) { try { TableIdentifier sourceSchemaTable = new TableIdentifier(sourceSchemaTableName, scala.Option.apply(sourceSchemaDatabaseName)); StructType sourceSchema = spark.sessionState().catalog().getTableMetadata(sourceSchemaTable).schema(); - this.sourceSchema = HoodieSchemaConversionUtils.convertStructTypeToHoodieSchema( + this.sourceHoodieSchema = HoodieSchemaConversionUtils.convertStructTypeToHoodieSchema( sourceSchema, sourceSchemaTableName, "hoodie." + sourceSchemaDatabaseName); @@ -73,7 +75,7 @@ public HiveSchemaProvider(TypedProperties props, JavaSparkContext jssc) { try { TableIdentifier targetSchemaTable = new TableIdentifier(targetSchemaTableName, scala.Option.apply(targetSchemaDatabaseName)); StructType targetSchema = spark.sessionState().catalog().getTableMetadata(targetSchemaTable).schema(); - this.targetSchema = HoodieSchemaConversionUtils.convertStructTypeToHoodieSchema( + this.targetHoodieSchema = HoodieSchemaConversionUtils.convertStructTypeToHoodieSchema( targetSchema, targetSchemaTableName, "hoodie." + targetSchemaDatabaseName); @@ -84,14 +86,16 @@ public HiveSchemaProvider(TypedProperties props, JavaSparkContext jssc) { } @Override + @Deprecated public Schema getSourceSchema() { - return sourceSchema.toAvroSchema(); + return getSourceHoodieSchema().toAvroSchema(); } @Override + @Deprecated public Schema getTargetSchema() { - if (targetSchema != null) { - return targetSchema.toAvroSchema(); + if (getTargetHoodieSchema() != null) { + return getTargetHoodieSchema().toAvroSchema(); } else { return super.getTargetSchema(); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java index b23f7d24fbc1f..159634c8818d0 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java @@ -40,13 +40,12 @@ import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; @@ -79,8 +78,9 @@ *

    * https://github.com/confluentinc/schema-registry */ +@Slf4j public class SchemaRegistryProvider extends SchemaProvider { - private static final Logger LOG = LoggerFactory.getLogger(SchemaRegistryProvider.class); + private static final Pattern URL_PATTERN = Pattern.compile("(.*/)subjects/(.*)/versions/(.*)"); private static final String LATEST = "latest"; @@ -195,7 +195,7 @@ public String fetchSchemaFromRegistry(String registryUrl) { } catch (IllegalAccessError error) { // If we're not processing Protobuf schema, fall back to the legacy method if (!ProtobufSchema.TYPE.equalsIgnoreCase(schemaType)) { - LOG.warn("Falling back to legacy schema retrieval due to IllegalAccessError", error); + log.warn("Falling back to legacy schema retrieval due to IllegalAccessError", error); return fetchSchemaUsingLegacyMethod(registryUrl); } // Otherwise, rethrow the error diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java index 854486e0343b5..c9698feb9ad8d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java @@ -21,9 +21,11 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.schema.HoodieSchema; +import lombok.Getter; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; +@Getter public class SimpleSchemaProvider extends SchemaProvider { private final Schema sourceSchema; @@ -32,9 +34,4 @@ public SimpleSchemaProvider(JavaSparkContext jssc, HoodieSchema sourceSchema, Ty super(props, jssc); this.sourceSchema = sourceSchema.toAvroSchema(); } - - @Override - public Schema getSourceSchema() { - return sourceSchema; - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java index acc9ad78cb2de..cb2079a9d25c0 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java @@ -26,10 +26,9 @@ import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.utilities.schema.SchemaPostProcessor; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -38,10 +37,9 @@ * An implementation of {@link SchemaPostProcessor} which will add a column named "_hoodie_is_deleted" to the end of * a given schema. */ +@Slf4j public class DeleteSupportSchemaPostProcessor extends SchemaPostProcessor { - private static final Logger LOG = LoggerFactory.getLogger(DeleteSupportSchemaPostProcessor.class); - public DeleteSupportSchemaPostProcessor(TypedProperties props, JavaSparkContext jssc) { super(props, jssc); } @@ -55,7 +53,7 @@ public Schema processSchema(Schema schema) { @Override public HoodieSchema processSchema(HoodieSchema schema) { if (schema.getField(HoodieRecord.HOODIE_IS_DELETED_FIELD).isPresent()) { - LOG.warn("column {} already exists!", HoodieRecord.HOODIE_IS_DELETED_FIELD); + log.warn("column {} already exists!", HoodieRecord.HOODIE_IS_DELETED_FIELD); return schema; } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java index 703dea14544fb..bb3de9344ea7e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java @@ -27,10 +27,9 @@ import org.apache.hudi.utilities.exception.HoodieSchemaPostProcessException; import org.apache.hudi.utilities.schema.SchemaPostProcessor; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.LinkedList; @@ -49,10 +48,9 @@ *

    * properties.put("hoodie.streamer.schemaprovider.schema_post_processor.delete.columns", "column1,column2"). */ +@Slf4j public class DropColumnSchemaPostProcessor extends SchemaPostProcessor { - private static final Logger LOG = LoggerFactory.getLogger(DropColumnSchemaPostProcessor.class); - public DropColumnSchemaPostProcessor(TypedProperties props, JavaSparkContext jssc) { super(props, jssc); } @@ -77,7 +75,7 @@ public HoodieSchema processSchema(HoodieSchema schema) { this.config, SchemaProviderPostProcessorConfig.DELETE_COLUMN_POST_PROCESSOR_COLUMN); if (StringUtils.isNullOrEmpty(columnToDeleteStr)) { - LOG.warn("Param {} is null or empty, return original schema", + log.warn("Param {} is null or empty, return original schema", SchemaProviderPostProcessorConfig.DELETE_COLUMN_POST_PROCESSOR_COLUMN.key()); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java index 726770e5ad4a8..79ad5ec3dcf19 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java @@ -27,6 +27,8 @@ import org.apache.hudi.utilities.exception.HoodieTransformPlanException; import org.apache.hudi.utilities.streamer.HoodieStreamer; +import lombok.AccessLevel; +import lombok.Getter; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -141,6 +143,8 @@ public StructType transformedSchema(JavaSparkContext jsc, SparkSession sparkSess } protected static class TransformerInfo { + + @Getter(AccessLevel.PROTECTED) private final Transformer transformer; private final Option idOpt; @@ -154,10 +158,6 @@ private TransformerInfo(Transformer transformer) { this.idOpt = Option.empty(); } - protected Transformer getTransformer() { - return transformer; - } - private boolean hasIdentifier() { return idOpt.isPresent(); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java index 1256491528d1b..76f21a308e39c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java @@ -21,24 +21,23 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.utilities.exception.HoodieTransformExecutionException; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.UUID; /** * Transformer that can flatten nested objects. It currently doesn't unnest arrays. */ +@Slf4j public class FlatteningTransformer implements Transformer { private static final String TMP_TABLE = "HUDI_SRC_TMP_TABLE_"; - private static final Logger LOG = LoggerFactory.getLogger(FlatteningTransformer.class); /** * Configs supported. @@ -49,7 +48,7 @@ public Dataset apply(JavaSparkContext jsc, SparkSession sparkSession, Datas try { // tmp table name doesn't like dashes String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); - LOG.info("Registering tmp table : " + tmpTable); + log.info("Registering tmp table: {}", tmpTable); rowDataset.createOrReplaceTempView(tmpTable); Dataset transformed = sparkSession.sql("select " + flattenSchema(rowDataset.schema(), null) + " from " + tmpTable); sparkSession.catalog().dropTempView(tmpTable); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java index cdef1677e587c..1e8af287f8090 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java @@ -23,14 +23,13 @@ import org.apache.hudi.utilities.config.SqlTransformerConfig; import org.apache.hudi.utilities.exception.HoodieTransformExecutionException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Scanner; @@ -56,10 +55,9 @@ *

    * SELECT * FROM tmp_personal_trips; */ +@Slf4j public class SqlFileBasedTransformer implements Transformer { - private static final Logger LOG = LoggerFactory.getLogger(SqlFileBasedTransformer.class); - private static final String SRC_PATTERN = ""; private static final String TMP_TABLE = "HOODIE_SRC_TMP_TABLE_"; @@ -75,19 +73,19 @@ public Dataset apply( final FileSystem fs = HadoopFSUtils.getFs(sqlFile, jsc.hadoopConfiguration(), true); // tmp table name doesn't like dashes final String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); - LOG.info("Registering tmp table: {}", tmpTable); + log.info("Registering tmp table: {}", tmpTable); rowDataset.createOrReplaceTempView(tmpTable); try (final Scanner scanner = new Scanner(fs.open(new Path(sqlFile)), "UTF-8")) { Dataset rows = null; // each sql statement is separated with semicolon hence set that as delimiter. scanner.useDelimiter(";"); - LOG.info("SQL Query for transformation:"); + log.info("SQL Query for transformation:"); while (scanner.hasNext()) { String sqlStr = scanner.next(); sqlStr = sqlStr.replaceAll(SRC_PATTERN, tmpTable).trim(); if (!sqlStr.isEmpty()) { - LOG.info(sqlStr); + log.info(sqlStr); // overwrite the same dataset object until the last statement then return. rows = sparkSession.sql(sqlStr); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java index 290e3a69432ad..9013ab04b9b3f 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java @@ -22,12 +22,11 @@ import org.apache.hudi.utilities.config.SqlTransformerConfig; import org.apache.hudi.utilities.exception.HoodieTransformExecutionException; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.UUID; @@ -38,10 +37,9 @@ * * The query should reference the source as a table named "\" */ +@Slf4j public class SqlQueryBasedTransformer implements Transformer { - private static final Logger LOG = LoggerFactory.getLogger(SqlQueryBasedTransformer.class); - private static final String SRC_PATTERN = ""; private static final String TMP_TABLE = "HOODIE_SRC_TMP_TABLE_"; @@ -53,10 +51,10 @@ public Dataset apply(JavaSparkContext jsc, SparkSession sparkSession, Datas try { // tmp table name doesn't like dashes String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); - LOG.info("Registering tmp table: {}", tmpTable); + log.info("Registering tmp table: {}", tmpTable); rowDataset.createOrReplaceTempView(tmpTable); String sqlStr = transformerSQL.replaceAll(SRC_PATTERN, tmpTable); - LOG.debug("SQL Query for transformation: {}", sqlStr); + log.debug("SQL Query for transformation: {}", sqlStr); Dataset transformed = sparkSession.sql(sqlStr); sparkSession.catalog().dropTempView(tmpTable); return transformed; From 7bc38af1be0bad97bec6780f055c000dc3779fcd Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 2 Jun 2026 19:08:47 +0800 Subject: [PATCH 041/255] refactor: Add Lombok Builders (#17781) - Add Lombok Builder to HoodieFileGroupReader - Add Lombok Builder to InputSplit - Add Lombok Builder to ReaderParameters (cherry picked from commit 7964202a751dc24d65fa0bcd1f300278a453e6bd) --- .../cli/commands/HoodieLogFileCommand.java | 6 +- .../apache/hudi/index/HoodieIndexUtils.java | 8 +- .../io/FileGroupReaderBasedAppendHandle.java | 20 +- .../io/FileGroupReaderBasedMergeHandle.java | 6 +- .../HoodieBackedTableMetadataWriter.java | 8 +- .../SecondaryIndexRecordGenerationUtils.java | 10 +- .../strategy/ClusteringExecutionStrategy.java | 17 +- .../client/TestJavaHoodieBackedMetadata.java | 2 +- .../utils/SparkMetadataWriterUtils.java | 2 +- .../TestSparkRDDMetadataWriteClient.java | 6 +- .../TestHoodieBackedTableMetadata.java | 2 +- .../table/read/HoodieFileGroupReader.java | 296 ++++++------------ .../hudi/common/table/read/InputSplit.java | 64 ++-- .../common/table/read/ReaderParameters.java | 82 ++--- .../DefaultFileGroupRecordBufferLoader.java | 6 +- .../buffer/LogScanningRecordBufferLoader.java | 2 +- .../ReusableFileGroupRecordBufferLoader.java | 2 +- .../StreamingFileGroupRecordBufferLoader.java | 4 +- .../metadata/HoodieBackedTableMetadata.java | 6 +- .../metadata/HoodieTableMetadataUtil.java | 8 +- .../read/TestHoodieFileGroupReaderBase.java | 8 +- .../buffer/BaseTestFileGroupRecordBuffer.java | 2 +- .../TestFileGroupRecordBufferLoader.java | 4 +- ...stSortedKeyBasedFileGroupRecordBuffer.java | 2 +- .../apache/hudi/table/format/FormatUtils.java | 8 +- .../HoodieFileGroupReaderTestHarness.java | 7 +- ...oodieFileGroupReaderBasedRecordReader.java | 6 +- .../apache/hudi/HoodieMergeOnReadRDDV2.scala | 8 +- .../hudi/cdc/CDCFileGroupIterator.scala | 8 +- ...HoodieFileGroupReaderBasedFileFormat.scala | 8 +- .../PartitionBucketIndexManager.scala | 8 +- ...tMetadataUtilRLIandSIRecordGeneration.java | 2 +- .../functional/TestHoodieBackedMetadata.java | 2 +- 33 files changed, 279 insertions(+), 351 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java index 6138e1fb38396..d6d2546e2cb11 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java @@ -242,10 +242,12 @@ storage, new StoragePath(logFilePathPattern)).stream() Option.empty(), Option.empty(), fileGroupReaderProperties); - try (HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + try (HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(HoodieCLI.getTableMetaClient()) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(readerSchema) .withRequestedSchema(readerSchema) .withLatestCommitTime(client.getActiveTimeline().getCommitAndReplaceTimeline().lastInstant().map(HoodieInstant::requestedTime).orElse(HoodieInstantTimeGenerator.getCurrentInstantTimeStr())) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java index d9fe1068e4218..d2cc530295f2b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java @@ -319,14 +319,16 @@ private static HoodieData> getExistingRecords( Option internalSchemaOption = SerDeHelper.fromJson(config.getInternalSchema()); FileSlice fileSlice = fileSliceOption.get(); HoodieReaderContext readerContext = readerContextFactory.getContext(); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(instantTime.get()) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(dataSchema) .withRequestedSchema(dataSchema) - .withInternalSchema(internalSchemaOption) + .withInternalSchemaOpt(internalSchemaOption) .withProps(metaClient.getTableConfig().getProps()) .build(); try { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedAppendHandle.java index a081709f6fc22..c40ce0158a3cd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedAppendHandle.java @@ -82,10 +82,20 @@ public void doAppend() { new HoodieLogFile(new StoragePath(FSUtils.constructAbsolutePath( config.getBasePath(), operation.getPartitionPath()), logFileName))); // Initializes the record iterator, log compaction requires writing the deletes into the delete block of the resulting log file. - try (HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder().withReaderContext(readerContext).withHoodieTableMetaClient(hoodieTable.getMetaClient()) - .withLatestCommitTime(instantTime).withPartitionPath(partitionPath).withLogFiles(logFiles).withBaseFileOption(Option.empty()).withDataSchema(writeSchemaWithMetaFields) - .withRequestedSchema(writeSchemaWithMetaFields).withInternalSchema(internalSchemaOption).withProps(props).withEmitDelete(true) - .withShouldUseRecordPosition(usePosition).withSortOutput(hoodieTable.requireSortedRecords()) + try (HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(hoodieTable.getMetaClient()) + .withLatestCommitTime(instantTime) + .withPartitionPath(partitionPath) + .withLogFiles(logFiles) + .withBaseFileOption(Option.empty()) + .withDataSchema(writeSchemaWithMetaFields) + .withRequestedSchema(writeSchemaWithMetaFields) + .withInternalSchemaOpt(internalSchemaOption) + .withProps(props) + .withEmitDelete(true) + .withShouldUseRecordPosition(usePosition) + .withSortOutput(hoodieTable.requireSortedRecords()) // instead of using config.enableOptimizedLogBlocksScan(), we set to true as log compaction blocks only supported in scanV2 .build()) { recordItr = new CloseableMappingIterator<>(fileGroupReader.getLogRecordsOnly(), record -> { @@ -96,7 +106,7 @@ public void doAppend() { header.put(HoodieLogBlock.HeaderMetadataType.COMPACTED_BLOCK_TIMES, StringUtils.join(fileGroupReader.getValidBlockInstants(), ",")); super.doAppend(); - this.readStats = fileGroupReader.getStats(); + this.readStats = fileGroupReader.getReadStats(); } catch (IOException e) { throw new HoodieIOException("Failed to initialize file group reader for " + fileId, e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java index d708c15f33845..2893913a0d13a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java @@ -301,7 +301,7 @@ public void doMerge() { // The stats of inserts, updates, and deletes are updated once at the end // These will be set in the write stat when closing the merge handle - this.readStats = fileGroupReader.getStats(); + this.readStats = fileGroupReader.getReadStats(); this.insertRecordsWritten = readStats.getNumInserts(); this.updatedRecordsWritten = readStats.getNumUpdates(); this.recordsDeleted = readStats.getNumDeletes(); @@ -318,10 +318,10 @@ protected long getMaxMemoryForMerge() { private HoodieFileGroupReader getFileGroupReader(boolean usePosition, Option internalSchemaOption, TypedProperties props, Option> logFileStreamOpt, Iterator> incomingRecordsItr) { - HoodieFileGroupReader.Builder fileGroupBuilder = HoodieFileGroupReader.newBuilder().withReaderContext(readerContext).withHoodieTableMetaClient(hoodieTable.getMetaClient()) + HoodieFileGroupReader.HoodieFileGroupReaderBuilder fileGroupBuilder = HoodieFileGroupReader.builder().withReaderContext(readerContext).withHoodieTableMetaClient(hoodieTable.getMetaClient()) .withLatestCommitTime(maxInstantTime).withPartitionPath(partitionPath).withBaseFileOption(Option.ofNullable(baseFileToMerge)) .withDataSchema(writeSchemaWithMetaFields).withRequestedSchema(writeSchemaWithMetaFields) - .withInternalSchema(internalSchemaOption).withProps(props) + .withInternalSchemaOpt(internalSchemaOption).withProps(props) .withShouldUseRecordPosition(usePosition).withSortOutput(hoodieTable.requireSortedRecords()) .withFileGroupUpdateCallback(createCallback()); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 7035c40c9b42b..9ed26c586ef13 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -997,14 +997,16 @@ private static HoodieData readRecordKeysFromFileSliceSnapshot( HoodieSchema requestedSchema = metaClient.getTableConfig().populateMetaFields() ? getRecordKeySchema() : HoodieSchemaUtils.projectSchema(dataSchema, Arrays.asList(metaClient.getTableConfig().getRecordKeyFields().orElse(new String[0]))); Option internalSchemaOption = SerDeHelper.fromJson(dataWriteConfig.getInternalSchema()); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withLatestCommitTime(instantTime.get()) .withDataSchema(dataSchema) .withRequestedSchema(requestedSchema) - .withInternalSchema(internalSchemaOption) + .withInternalSchemaOpt(internalSchemaOption) .withShouldUseRecordPosition(false) .withProps(metaClient.getTableConfig().getProps()) .build(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java index 9cb9b8ca7df5f..b86688b81e609 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java @@ -123,7 +123,7 @@ public static HoodieData convertWriteStatsToSecondaryIndexReco // validate that for a given fileId, either we have 1 parquet file or N log files. AtomicInteger totalParquetFiles = new AtomicInteger(); AtomicInteger totalLogFiles = new AtomicInteger(); - writeStats.stream().forEach(writeStat -> { + writeStats.forEach(writeStat -> { if (FSUtils.isLogFile(new StoragePath(basePath, writeStat.getPath()))) { totalLogFiles.getAndIncrement(); } else { @@ -156,7 +156,7 @@ public static HoodieData convertWriteStatsToSecondaryIndexReco } else { // log files are added in current commit // add new log files to existing latest file slice and compute the secondary index to primary key mapping. FileSlice latestFileSlice = fileSliceOption.get(); - writeStats.stream().forEach(writeStat -> { + writeStats.forEach(writeStat -> { StoragePathInfo logFile = new StoragePathInfo(new StoragePath(basePath, writeStat.getPath()), writeStat.getFileSizeInBytes(), false, (short) 0, 0, 0); latestFileSlice.addLogFile(new HoodieLogFile(logFile)); }); @@ -286,9 +286,11 @@ private static ClosableIterator> createSecondaryIndexRe boolean allowInflightInstants) throws IOException { String secondaryKeyField = indexDefinition.getSourceFieldsKey(); HoodieSchema requestedSchema = getRequestedSchemaForSecondaryIndex(metaClient, tableSchema, secondaryKeyField); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withHoodieTableMetaClient(metaClient) .withProps(props) .withLatestCommitTime(instantTime) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java index d34a85a97bd41..cc3c4a02eb227 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java @@ -142,9 +142,18 @@ protected static HoodieFileGroupReader getFileGroupReader(HoodieTableMeta ReaderContextFactory readerContextFactory, String instantTime, TypedProperties properties, boolean usePosition) { HoodieReaderContext readerContext = readerContextFactory.getContext(); - return HoodieFileGroupReader.newBuilder() - .withReaderContext(readerContext).withHoodieTableMetaClient(metaClient).withLatestCommitTime(instantTime) - .withFileSlice(fileSlice).withDataSchema(readerSchema).withRequestedSchema(readerSchema).withInternalSchema(internalSchemaOption) - .withShouldUseRecordPosition(usePosition).withProps(properties).build(); + return HoodieFileGroupReader.builder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(metaClient) + .withLatestCommitTime(instantTime) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) + .withDataSchema(readerSchema) + .withRequestedSchema(readerSchema) + .withInternalSchemaOpt(internalSchemaOption) + .withShouldUseRecordPosition(usePosition) + .withProps(properties) + .build(); } } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index 9563439c39eb9..feda056472dc8 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -946,7 +946,7 @@ private void verifyMetadataMergedRecords(HoodieTableMetaClient metadataMetaClien HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())); HoodieAvroReaderContext readerContext = new HoodieAvroReaderContext(metadataMetaClient.getStorageConf(), metadataMetaClient.getTableConfig(), Option.empty(), Option.empty(), new TypedProperties()); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metadataMetaClient) .withLogFiles(logFiles.stream()) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java index 47155899e2048..2184c24ad136a 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java @@ -335,7 +335,7 @@ private static Iterator getExpressionIndexRecordsIterator(HoodieReaderConte baseFileOption = Option.empty(); logFileStream = Stream.of(new HoodieLogFile(filePath)); } - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withDataSchema(tableSchema) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDMetadataWriteClient.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDMetadataWriteClient.java index 8a88dd8135887..65e45ce96f368 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDMetadataWriteClient.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDMetadataWriteClient.java @@ -218,10 +218,12 @@ private void readFromMDTFileSliceAndValidate(HoodieTableMetaClient metadataMetaC HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(metadataSchema); HoodieAvroReaderContext readerContext = new HoodieAvroReaderContext(metadataMetaClient.getStorageConf(), metadataMetaClient.getTableConfig(), Option.of(instantRange), Option.of(predicate)); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metadataMetaClient) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withLatestCommitTime(validMetadataInstant) .withRequestedSchema(metadataSchema) .withDataSchema(schema) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java index 6bb71f07de614..fcbd689f74471 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java @@ -560,7 +560,7 @@ private void verifyMetadataRawRecords(HoodieTable table, List log private void verifyMetadataMergedRecords(HoodieTableMetaClient metadataMetaClient, List logFiles, String latestCommitTimestamp, HoodieWriteConfig metadataTableWriteConfig) { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())); HoodieAvroReaderContext readerContext = new HoodieAvroReaderContext(metadataMetaClient.getStorageConf(), metadataMetaClient.getTableConfig(), Option.empty(), Option.empty()); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metadataMetaClient) .withLogFiles(logFiles.stream()) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java index 2b35eaff4f30d..0b536b511db10 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java @@ -23,7 +23,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.model.BaseFile; -import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; @@ -35,7 +34,6 @@ import org.apache.hudi.common.table.read.buffer.FileGroupRecordBufferLoader; import org.apache.hudi.common.table.read.buffer.HoodieFileGroupRecordBuffer; import org.apache.hudi.common.util.ConfigUtils; -import org.apache.hudi.common.util.Either; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; @@ -49,6 +47,10 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -67,7 +69,9 @@ * @param The type of engine-specific record representation, e.g.,{@code InternalRow} * in Spark and {@code RowData} in Flink. */ +@AllArgsConstructor public final class HoodieFileGroupReader implements Closeable { + private final HoodieReaderContext readerContext; private final HoodieTableMetaClient metaClient; private final InputSplit inputSplit; @@ -81,25 +85,113 @@ public final class HoodieFileGroupReader implements Closeable { private HoodieFileGroupRecordBuffer recordBuffer; private ClosableIterator baseFileIterator; private final Option> outputConverter; + @Getter private final HoodieReadStats readStats; // Callback to run custom logic on updates to the base files for the file group private final Option> fileGroupUpdateCallback; // The list of instant times read from the log blocks, this value is used by the log-compaction to allow optimized log-block scans + @Getter private List validBlockInstants = Collections.emptyList(); private BufferedRecordConverter bufferedRecordConverter; - private HoodieFileGroupReader(HoodieReaderContext readerContext, HoodieStorage storage, String tablePath, - String latestCommitTime, HoodieSchema dataSchema, HoodieSchema requestedSchema, - Option internalSchemaOpt, HoodieTableMetaClient hoodieTableMetaClient, TypedProperties props, - ReaderParameters readerParameters, InputSplit inputSplit, Option> updateCallback, - FileGroupRecordBufferLoader recordBufferLoader) { + @Builder(setterPrefix = "with") + private HoodieFileGroupReader( + HoodieReaderContext readerContext, + String latestCommitTime, + HoodieSchema dataSchema, + HoodieSchema requestedSchema, + Option internalSchemaOpt, + HoodieTableMetaClient hoodieTableMetaClient, + TypedProperties props, + Option baseFileOption, + Stream logFiles, + String partitionPath, + Long start, + Long length, + Iterator recordIterator, + Boolean shouldUseRecordPosition, + Boolean allowInflightInstants, + Boolean emitDelete, + Boolean sortOutput, + Option> fileGroupUpdateCallback, + FileGroupRecordBufferLoader recordBufferLoader) { + + // Validations + ValidationUtils.checkArgument(readerContext != null, "Reader context is required"); + ValidationUtils.checkArgument(hoodieTableMetaClient != null, "Hoodie table meta client is required"); + ValidationUtils.checkArgument(latestCommitTime != null, "Latest commit time is required"); + ValidationUtils.checkArgument(dataSchema != null, "Data schema is required"); + ValidationUtils.checkArgument(requestedSchema != null, "Requested schema is required"); + ValidationUtils.checkArgument(props != null, "Props is required"); + ValidationUtils.checkArgument(partitionPath != null, "Partition path is required"); + + // Handle defaults + if (internalSchemaOpt == null) { + internalSchemaOpt = Option.empty(); + } + if (baseFileOption == null) { + baseFileOption = Option.empty(); + } + if (start == null) { + start = 0L; + } + if (length == null) { + length = Long.MAX_VALUE; + } + if (shouldUseRecordPosition == null) { + shouldUseRecordPosition = false; + } + if (allowInflightInstants == null) { + allowInflightInstants = false; + } + if (emitDelete == null) { + emitDelete = false; + } + if (sortOutput == null) { + sortOutput = false; + } + if (fileGroupUpdateCallback == null) { + fileGroupUpdateCallback = Option.empty(); + } + + // Derive tablePath + String tablePath = hoodieTableMetaClient.getBasePath().toString(); + + // Set the storage with the readerContext's storage configuration + HoodieStorage storage = hoodieTableMetaClient.getStorage().newInstance(new StoragePath(tablePath), readerContext.getStorageConfiguration()); + + // Handle recordBufferLoader default + if (recordBufferLoader == null) { + if (recordIterator != null) { + recordBufferLoader = FileGroupRecordBufferLoader.createStreamingRecordsBufferLoader(); + } else { + recordBufferLoader = FileGroupRecordBufferLoader.createDefault(); + } + } + + // Build composite objects using static helpers + this.readerParameters = ReaderParameters.builder() + .shouldUseRecordPosition(shouldUseRecordPosition) + .emitDeletes(emitDelete) + .sortOutputs(sortOutput) + .inflightInstantsAllowed(allowInflightInstants) + .build(); + this.inputSplit = InputSplit.builder() + .baseFileOption(baseFileOption) + .logFileStream(logFiles) + .recordIterator((Iterator) recordIterator) + .partitionPath(partitionPath) + .start(start) + .length(length) + .build(); + + // Initialize fields this.readerContext = readerContext; this.recordBufferLoader = recordBufferLoader; - this.fileGroupUpdateCallback = updateCallback; + this.fileGroupUpdateCallback = fileGroupUpdateCallback; this.metaClient = hoodieTableMetaClient; this.storage = storage; - this.readerParameters = readerParameters; - this.inputSplit = inputSplit; + readerContext.setHasLogFiles(this.inputSplit.hasLogFiles()); readerContext.getRecordContext().setPartitionPath(inputSplit.getPartitionPath()); if (readerContext.getHasLogFiles() && inputSplit.getStart() != 0) { @@ -112,7 +204,7 @@ private HoodieFileGroupReader(HoodieReaderContext readerContext, HoodieStorag readerContext.setTablePath(tablePath); readerContext.setLatestCommitTime(latestCommitTime); boolean isSkipMerge = ConfigUtils.getStringWithAltKeys(props, HoodieReaderConfig.MERGE_TYPE, true).equalsIgnoreCase(HoodieReaderConfig.REALTIME_SKIP_MERGE); - readerContext.setShouldMergeUseRecordPosition(readerParameters.useRecordPosition() && !isSkipMerge && readerContext.getHasLogFiles() && inputSplit.isParquetBaseFile()); + readerContext.setShouldMergeUseRecordPosition(readerParameters.shouldUseRecordPosition() && !isSkipMerge && readerContext.getHasLogFiles() && inputSplit.isParquetBaseFile()); readerContext.setHasBootstrapBaseFile(inputSplit.getBaseFileOption().flatMap(HoodieBaseFile::getBootstrapBaseFile).isPresent()); readerContext.setSchemaHandler(readerContext.getRecordContext().supportsParquetRowIndex() ? new ParquetRowIndexBasedSchemaHandler<>(readerContext, dataSchema, requestedSchema, internalSchemaOpt, props, metaClient) @@ -248,13 +340,6 @@ boolean hasNext() throws IOException { } } - /** - * @return statistics of reading a file group. - */ - public HoodieReadStats getStats() { - return readStats; - } - /** * @return The next record after calling {@link #hasNext}. */ @@ -266,10 +351,6 @@ BufferedRecord next() { return nextVal; } - public List getValidBlockInstants() { - return validBlockInstants; - } - /** * Notifies a write failure with the given record key. */ @@ -355,175 +436,4 @@ public void close() { } } } - - public static Builder newBuilder() { - return new Builder<>(); - } - - public static class Builder { - private HoodieReaderContext readerContext; - private HoodieStorage storage; - private String tablePath; - private String latestCommitTime; - private HoodieSchema dataSchema; - private HoodieSchema requestedSchema; - private Option internalSchemaOpt = Option.empty(); - private HoodieTableMetaClient hoodieTableMetaClient; - private TypedProperties props; - private Option baseFileOption; - private Stream logFiles; - private String partitionPath; - private long start = 0; - private long length = Long.MAX_VALUE; - private Iterator recordIterator; - private boolean shouldUseRecordPosition = false; - private boolean allowInflightInstants = false; - private boolean emitDelete; - private boolean sortOutput = false; - private Option> fileGroupUpdateCallback = Option.empty(); - private FileGroupRecordBufferLoader recordBufferLoader; - - public Builder withReaderContext(HoodieReaderContext readerContext) { - this.readerContext = readerContext; - return this; - } - - public Builder withLatestCommitTime(String latestCommitTime) { - this.latestCommitTime = latestCommitTime; - return this; - } - - public Builder withFileSlice(FileSlice fileSlice) { - this.baseFileOption = fileSlice.getBaseFile(); - this.logFiles = fileSlice.getLogFiles(); - this.partitionPath = fileSlice.getPartitionPath(); - return this; - } - - public Builder withBaseFileOption(Option baseFileOption) { - this.baseFileOption = baseFileOption; - return this; - } - - public Builder withLogFiles(Stream logFiles) { - this.logFiles = logFiles; - return this; - } - - public Builder withRecordIterator(Iterator recordIterator) { - this.recordIterator = (Iterator) recordIterator; - this.recordBufferLoader = FileGroupRecordBufferLoader.createStreamingRecordsBufferLoader(); - return this; - } - - public Builder withPartitionPath(String partitionPath) { - this.partitionPath = partitionPath; - return this; - } - - public Builder withDataSchema(HoodieSchema dataSchema) { - this.dataSchema = dataSchema; - return this; - } - - public Builder withRequestedSchema(HoodieSchema requestedSchema) { - this.requestedSchema = requestedSchema; - return this; - } - - public Builder withInternalSchema(Option internalSchemaOpt) { - this.internalSchemaOpt = internalSchemaOpt; - return this; - } - - public Builder withHoodieTableMetaClient(HoodieTableMetaClient hoodieTableMetaClient) { - this.hoodieTableMetaClient = hoodieTableMetaClient; - this.tablePath = hoodieTableMetaClient.getBasePath().toString(); - return this; - } - - public Builder withProps(TypedProperties props) { - this.props = props; - return this; - } - - public Builder withStart(long start) { - this.start = start; - return this; - } - - public Builder withLength(long length) { - this.length = length; - return this; - } - - public Builder withShouldUseRecordPosition(boolean shouldUseRecordPosition) { - this.shouldUseRecordPosition = shouldUseRecordPosition; - return this; - } - - public Builder withAllowInflightInstants(boolean allowInflightInstants) { - this.allowInflightInstants = allowInflightInstants; - return this; - } - - public Builder withEmitDelete(boolean emitDelete) { - this.emitDelete = emitDelete; - return this; - } - - public Builder withFileGroupUpdateCallback(Option> fileGroupUpdateCallback) { - this.fileGroupUpdateCallback = fileGroupUpdateCallback; - return this; - } - - /** - * If true, the output of the merge will be sorted instead of appending log records to end of the iterator if they do not have matching keys in the base file. - * This assumes that the base file is already sorted by key. - * @param sortOutput whether to sort the output iterator - * @return this builder instance - */ - public Builder withSortOutput(boolean sortOutput) { - this.sortOutput = sortOutput; - return this; - } - - public Builder withRecordBufferLoader(FileGroupRecordBufferLoader recordBufferLoader) { - this.recordBufferLoader = recordBufferLoader; - return this; - } - - public HoodieFileGroupReader build() { - ValidationUtils.checkArgument(readerContext != null, "Reader context is required"); - ValidationUtils.checkArgument(hoodieTableMetaClient != null, "Hoodie table meta client is required"); - ValidationUtils.checkArgument(tablePath != null, "Table path is required"); - // set the storage with the readerContext's storage configuration - this.storage = hoodieTableMetaClient.getStorage().newInstance(new StoragePath(tablePath), readerContext.getStorageConfiguration()); - - ValidationUtils.checkArgument(storage != null, "Storage is required"); - ValidationUtils.checkArgument(latestCommitTime != null, "Latest commit time is required"); - ValidationUtils.checkArgument(dataSchema != null, "Data schema is required"); - ValidationUtils.checkArgument(requestedSchema != null, "Requested schema is required"); - ValidationUtils.checkArgument(props != null, "Props is required"); - ValidationUtils.checkArgument(baseFileOption != null, "Base file option is required"); - ValidationUtils.checkArgument(partitionPath != null, "Partition path is required"); - - if (recordBufferLoader == null) { - recordBufferLoader = FileGroupRecordBufferLoader.createDefault(); - } - - ReaderParameters readerParameters = ReaderParameters.builder() - .shouldUseRecordPosition(shouldUseRecordPosition) - .emitDeletes(emitDelete) - .sortOutputs(sortOutput) - .allowInflightInstants(allowInflightInstants) - .build(); - InputSplit inputSplit = new InputSplit(baseFileOption, recordIterator != null ? Either.right(recordIterator) : Either.left(logFiles == null ? Stream.empty() : logFiles), - partitionPath, start, length); - return new HoodieFileGroupReader<>( - readerContext, storage, tablePath, latestCommitTime, dataSchema, requestedSchema, internalSchemaOpt, hoodieTableMetaClient, - props, readerParameters, inputSplit, fileGroupUpdateCallback, recordBufferLoader); - } - } - } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/InputSplit.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/InputSplit.java index 6d7b578e15b3e..a0f1a3d04e905 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/InputSplit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/InputSplit.java @@ -24,10 +24,13 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.cdc.HoodieCDCUtils; -import org.apache.hudi.common.util.Either; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; + import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -38,9 +41,13 @@ * Represents a split of input data for reading, which includes the partition path the data belongs to along with an optional base file and a list of log files. * If there is only a base file, it is possible for the reader to specify a particular range of the file with the start and length parameters. */ +@Getter public class InputSplit { + private final Option baseFileOption; + @Getter(AccessLevel.NONE) private final List logFiles; + @Getter(AccessLevel.NONE) private final Option> recordIterator; private final String partitionPath; // Byte offset to start reading from the base file @@ -48,26 +55,41 @@ public class InputSplit { // Length of bytes to read from the base file private final long length; - InputSplit(Option baseFileOption, - Either, Iterator> recordsToMerge, - String partitionPath, long start, long length) { - this.baseFileOption = baseFileOption; - if (recordsToMerge.isLeft()) { - this.logFiles = recordsToMerge.asLeft().sorted(HoodieLogFile.getLogFileComparator()) + @Builder + private InputSplit( + Option baseFileOption, + Stream logFileStream, + Iterator recordIterator, + String partitionPath, + long start, + long length) { + + // Ensure we do not have both sources of data to merge + // i.e. logFileStream and recordIterator cannot be both non-null + ValidationUtils.checkArgument(!(logFileStream != null && recordIterator != null), + "Cannot provide both logFileStream and recordIterator"); + + this.baseFileOption = Option.ofNullable(baseFileOption).orElse(Option.empty()); + this.partitionPath = partitionPath; + this.start = start; + this.length = length; + + if (logFileStream != null) { + // Process Log Files (if provided) + this.logFiles = logFileStream + .sorted(HoodieLogFile.getLogFileComparator()) .filter(logFile -> !logFile.getFileName().endsWith(HoodieCDCUtils.CDC_LOGFILE_SUFFIX)) .collect(Collectors.toList()); this.recordIterator = Option.empty(); + } else if (recordIterator != null) { + // Process Record Iterator (if provided) + this.logFiles = Collections.emptyList(); + this.recordIterator = Option.of(recordIterator); } else { + // Handle Case with neither (Base file only read) this.logFiles = Collections.emptyList(); - this.recordIterator = Option.of(recordsToMerge.asRight()); + this.recordIterator = Option.empty(); } - this.partitionPath = partitionPath; - this.start = start; - this.length = length; - } - - public Option getBaseFileOption() { - return baseFileOption; } public List getLogFiles() { @@ -79,18 +101,6 @@ public boolean hasLogFiles() { return !logFiles.isEmpty(); } - public String getPartitionPath() { - return partitionPath; - } - - public long getStart() { - return start; - } - - public long getLength() { - return length; - } - public boolean isParquetBaseFile() { return baseFileOption.map(baseFile -> HoodieFileFormat.fromFileExtension(baseFile.getStoragePath().getFileExtension()) == HoodieFileFormat.PARQUET).orElse(false); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/ReaderParameters.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/ReaderParameters.java index bfb791fe5fa78..d19ad39418b34 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/ReaderParameters.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/ReaderParameters.java @@ -19,78 +19,38 @@ package org.apache.hudi.common.table.read; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + /** * Parameters for how the reader should process the FileGroup while reading. */ +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Getter +@Builder public class ReaderParameters { + // Rely on the position of the record in the file instead of the record keys while merging data between base and log files - private final boolean useRecordPosition; + @Getter(AccessLevel.NONE) + @Builder.Default + private final boolean shouldUseRecordPosition = false; // Whether to emit delete records while reading - private final boolean emitDelete; + @Builder.Default + private final boolean emitDeletes = false; // Whether to sort the output records while reading, this implicitly requires the base file to be sorted - private final boolean sortOutput; + @Builder.Default + private final boolean sortOutputs = false; // Allows to consider inflight instants while merging log records using HoodieMergedLogRecordReader // The inflight instants need to be considered while updating RLI records. RLI needs to fetch the revived // and deleted keys from the log files written as part of active data commit. During the RLI update, - // the allowInflightInstants flag would need to be set to true. This would ensure the HoodieMergedLogRecordReader + // the inflightInstantsAllowed flag would need to be set to true. This would ensure the HoodieMergedLogRecordReader // considers the log records which are inflight. - private final boolean allowInflightInstants; - - private ReaderParameters(boolean useRecordPosition, boolean emitDelete, boolean sortOutput, boolean allowInflightInstants) { - this.useRecordPosition = useRecordPosition; - this.emitDelete = emitDelete; - this.sortOutput = sortOutput; - this.allowInflightInstants = allowInflightInstants; - } - - public boolean useRecordPosition() { - return useRecordPosition; - } - - public boolean emitDeletes() { - return emitDelete; - } - - public boolean sortOutputs() { - return sortOutput; - } - - public boolean allowInflightInstants() { - return allowInflightInstants; - } - - static Builder builder() { - return new Builder(); - } - - static class Builder { - private boolean shouldUseRecordPosition = false; - private boolean emitDelete = false; - private boolean sortOutput = false; - private boolean allowInflightInstants = false; - - public Builder shouldUseRecordPosition(boolean shouldUseRecordPosition) { - this.shouldUseRecordPosition = shouldUseRecordPosition; - return this; - } - - public Builder emitDeletes(boolean emitDelete) { - this.emitDelete = emitDelete; - return this; - } - - public Builder sortOutputs(boolean sortOutput) { - this.sortOutput = sortOutput; - return this; - } - - public Builder allowInflightInstants(boolean allowInflightInstants) { - this.allowInflightInstants = allowInflightInstants; - return this; - } + @Builder.Default + private final boolean inflightInstantsAllowed = false; - public ReaderParameters build() { - return new ReaderParameters(shouldUseRecordPosition, emitDelete, sortOutput, allowInflightInstants); - } + public boolean shouldUseRecordPosition() { + return shouldUseRecordPosition; } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java index 3e0609003ad33..e84921c080ae7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java @@ -62,15 +62,15 @@ public Pair, List> getRecordBuffer(Hoodie Option> fileGroupUpdateCallback) { boolean isSkipMerge = ConfigUtils.getStringWithAltKeys(props, HoodieReaderConfig.MERGE_TYPE, true).equalsIgnoreCase(HoodieReaderConfig.REALTIME_SKIP_MERGE); Option partialUpdateModeOpt = hoodieTableMetaClient.getTableConfig().getPartialUpdateMode(); - UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.emitDeletes(), fileGroupUpdateCallback, props); + UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.isEmitDeletes(), fileGroupUpdateCallback, props); FileGroupRecordBuffer recordBuffer; if (isSkipMerge) { recordBuffer = new UnmergedFileGroupRecordBuffer<>( readerContext, hoodieTableMetaClient, readerContext.getMergeMode(), partialUpdateModeOpt, props, readStats); - } else if (readerParameters.sortOutputs()) { + } else if (readerParameters.isSortOutputs()) { recordBuffer = new SortedKeyBasedFileGroupRecordBuffer<>( readerContext, hoodieTableMetaClient, readerContext.getMergeMode(), partialUpdateModeOpt, props, orderingFieldNames, updateProcessor); - } else if (readerParameters.useRecordPosition() && inputSplit.getBaseFileOption().isPresent()) { + } else if (readerParameters.shouldUseRecordPosition() && inputSplit.getBaseFileOption().isPresent()) { recordBuffer = new PositionBasedFileGroupRecordBuffer<>( readerContext, hoodieTableMetaClient, readerContext.getMergeMode(), partialUpdateModeOpt, inputSplit.getBaseFileOption().get().getCommitTime(), props, orderingFieldNames, updateProcessor); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/LogScanningRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/LogScanningRecordBufferLoader.java index b6638b1ecd043..1c4f6082e4f15 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/LogScanningRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/LogScanningRecordBufferLoader.java @@ -48,7 +48,7 @@ protected List scanLogFiles(HoodieReaderContext readerContext, Ho .withInstantRange(readerContext.getInstantRange()) .withPartition(inputSplit.getPartitionPath()) .withRecordBuffer(recordBuffer) - .withAllowInflightInstants(readerParameters.allowInflightInstants()) + .withAllowInflightInstants(readerParameters.isInflightInstantsAllowed()) .withMetaClient(hoodieTableMetaClient) .build()) { readStats.setTotalLogReadTimeMs(logRecordReader.getTotalTimeTakenToReadAndMergeBlocks()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java index aa3fbd22cf94b..bd3b3ec55bb36 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java @@ -58,7 +58,7 @@ public synchronized Pair, List> getRecord ReaderParameters readerParameters, HoodieReadStats readStats, Option> fileGroupUpdateCallback) { - UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.emitDeletes(), fileGroupUpdateCallback, props); + UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.isEmitDeletes(), fileGroupUpdateCallback, props); Option partialUpdateModeOpt = hoodieTableMetaClient.getTableConfig().getPartialUpdateMode(); if (cachedResults == null) { // Create an initial buffer to process the log files diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/StreamingFileGroupRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/StreamingFileGroupRecordBufferLoader.java index 02c1d5b4cb9c1..2087b7dceaf8e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/StreamingFileGroupRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/StreamingFileGroupRecordBufferLoader.java @@ -67,9 +67,9 @@ public Pair, List> getRecordBuffer(Hoodie readerContext.getSchemaHandler().setSchemaForUpdates(recordSchema); HoodieTableConfig tableConfig = hoodieTableMetaClient.getTableConfig(); Option partialUpdateModeOpt = tableConfig.getPartialUpdateMode(); - UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.emitDeletes(), fileGroupUpdateCallback, props); + UpdateProcessor updateProcessor = UpdateProcessor.create(readStats, readerContext, readerParameters.isEmitDeletes(), fileGroupUpdateCallback, props); FileGroupRecordBuffer recordBuffer; - if (readerParameters.sortOutputs()) { + if (readerParameters.isSortOutputs()) { recordBuffer = new SortedKeyBasedFileGroupRecordBuffer<>( readerContext, hoodieTableMetaClient, readerContext.getMergeMode(), partialUpdateModeOpt, props, orderingFieldNames, updateProcessor); } else { diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java index 1fabe8c97786f..70b136f2a646d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java @@ -601,11 +601,13 @@ private ClosableIterator readSliceWithFilter(Predicate predicate, baseFileReaders, fileGroupReaderProps); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metadataMetaClient) .withLatestCommitTime(latestMetadataInstantTime) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(SCHEMA) .withRequestedSchema(SCHEMA) .withProps(fileGroupReaderProps) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index 59e2d6dcd83f7..0ede3ae5015e5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -1809,7 +1809,7 @@ public static List> getLogFileColumnRangeM properties.setProperty(HoodieReaderConfig.MERGE_TYPE.key(), REALTIME_SKIP_MERGE); // Currently only avro is fully supported for extracting column ranges (see HUDI-8585) HoodieReaderContext readerContext = new HoodieAvroReaderContext(datasetMetaClient.getStorageConf(), datasetMetaClient.getTableConfig(), Option.empty(), Option.empty()); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(datasetMetaClient) .withLogFiles(Stream.of(logFile)) @@ -2568,10 +2568,12 @@ public static HoodieData readRecordKeysFromFileSlices(HoodieEn final String partition = partitionAndBaseFile.getKey(); final FileSlice fileSlice = partitionAndBaseFile.getValue(); if (!fileSlice.getBaseFile().isPresent()) { - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContextFactory.getContext()) .withHoodieTableMetaClient(metaClient) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(tableSchema) .withRequestedSchema(HoodieSchemaUtils.getRecordKeySchema()) .withLatestCommitTime(latestCommitTime) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java index ffcda26a37968..158ef475d8431 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java @@ -1040,15 +1040,17 @@ private HoodieFileGroupReader getHoodieFileGroupReader(StorageConfigurationnewBuilder() + return HoodieFileGroupReader.builder() .withReaderContext(getHoodieReaderContext(tablePath, schema, storageConf, metaClient)) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(metaClient.getActiveTimeline().lastInstant().get().requestedTime()) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(schema) .withRequestedSchema(schema) .withProps(props) - .withStart(start) + .withStart((long) start) .withLength(fileSlice.getTotalFileSize()) .withShouldUseRecordPosition(false) .withAllowInflightInstants(false) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java index e6ee09b3e6add..b2869df3310dc 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java @@ -149,7 +149,7 @@ protected static KeyBasedFileGroupRecordBuffer buildKeyBasedFileG when(inputSplit.hasNoRecordsToMerge()).thenReturn(false); when(inputSplit.getRecordIterator()).thenReturn(fileGroupRecordBufferItrOpt.get()); ReaderParameters readerParameters = mock(ReaderParameters.class); - when(readerParameters.sortOutputs()).thenReturn(false); + when(readerParameters.isSortOutputs()).thenReturn(false); return (KeyBasedFileGroupRecordBuffer) recordBufferLoader.getRecordBuffer(readerContext, mockMetaClient.getStorage(), inputSplit, orderingFieldNames, mockMetaClient, props, readerParameters, readStats, Option.empty()).getKey(); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java index 51dad1c752af6..e1c810c56910c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java @@ -90,12 +90,12 @@ public void testDefaultFileGroupBufferRecordLoader(String fileGroupRecordBufferT } ReaderParameters readerParameters = mock(ReaderParameters.class); if (fileGroupRecordBufferType.contains("Sorted")) { - when(readerParameters.sortOutputs()).thenReturn(true); + when(readerParameters.isSortOutputs()).thenReturn(true); } if (fileGroupRecordBufferType.contains("Position")) { HoodieBaseFile baseFile = mock(HoodieBaseFile.class); when(inputSplit.getBaseFileOption()).thenReturn(Option.of(baseFile)); - when(readerParameters.useRecordPosition()).thenReturn(true); + when(readerParameters.shouldUseRecordPosition()).thenReturn(true); } Option fileGroupUpdateCallback = Option.empty(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java index 0d43c1d463ca7..611128a963098 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java @@ -141,7 +141,7 @@ void readWithStreamingRecordBufferLoaderAndEventTimeOrdering() throws IOExceptio when(inputSplit.hasNoRecordsToMerge()).thenReturn(false); when(inputSplit.getRecordIterator()).thenReturn(inputRecords.iterator()); ReaderParameters readerParameters = mock(ReaderParameters.class); - when(readerParameters.sortOutputs()).thenReturn(true); + when(readerParameters.isSortOutputs()).thenReturn(true); SortedKeyBasedFileGroupRecordBuffer fileGroupRecordBuffer = (SortedKeyBasedFileGroupRecordBuffer) recordBufferLoader .getRecordBuffer(readerContext, mockMetaClient.getStorage(), inputSplit, Collections.singletonList("ts"), mockMetaClient, properties, readerParameters, readStats, Option.empty()).getKey(); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java index d62763ef64af7..0bd7f53b611f1 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java @@ -132,14 +132,16 @@ public static HoodieFileGroupReader createFileGroupReader( final TypedProperties typedProps = FlinkClientUtil.getReadProps(metaClient.getTableConfig(), writeConfig); typedProps.put(HoodieReaderConfig.MERGE_TYPE.key(), mergeType); - return HoodieFileGroupReader.newBuilder() + return HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(latestInstant) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(tableSchema) .withRequestedSchema(requiredSchema) - .withInternalSchema(Option.ofNullable(internalSchemaManager.getQuerySchema())) + .withInternalSchemaOpt(Option.ofNullable(internalSchemaManager.getQuerySchema())) .withProps(typedProps) .withShouldUseRecordPosition(false) .withEmitDelete(emitDelete) diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java index b7c63d3dd81cf..9259d75adb24f 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java @@ -137,11 +137,14 @@ protected ClosableIterator getFileGroupIterator(int numFiles, boo properties.setProperty(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.key(), basePath + "/" + HoodieTableMetaClient.TEMPFOLDER_NAME); properties.setProperty(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.key(), ExternalSpillableMap.DiskMapType.ROCKS_DB.name()); properties.setProperty(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.key(), "false"); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + FileSlice fileSlice = fileSliceOpt.orElseThrow(() -> new IllegalArgumentException("FileSlice is not present")); + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime("1000") // Not used internally. - .withFileSlice(fileSliceOpt.orElseThrow(() -> new IllegalArgumentException("FileSlice is not present"))) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(HOODIE_SCHEMA) .withRequestedSchema(HOODIE_SCHEMA) .withProps(properties) diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java index 1fbb6ad4a9826..22d5ba7017114 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java @@ -145,11 +145,13 @@ public HoodieFileGroupReaderBasedRecordReader(HiveReaderCreator readerCreator, LOG.debug("Creating HoodieFileGroupReaderRecordReader with tableBasePath={}, latestCommitTime={}, fileSplit={}", tableBasePath, latestCommitTime, fileSplit.getPath()); FileSlice fileSlice = getFileSliceFromSplit(fileSplit, getFs(tableBasePath, jobConfCopy), tableBasePath); this.containsBaseFile = fileSlice.getBaseFile().isPresent(); - this.recordIterator = HoodieFileGroupReader.newBuilder() + this.recordIterator = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(latestCommitTime) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) .withDataSchema(tableSchema) .withRequestedSchema(requestedSchema) .withProps(props) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDDV2.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDDV2.scala index 3d9b908c4d310..eadbdd80bd31c 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDDV2.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDDV2.scala @@ -171,7 +171,7 @@ class HoodieMergeOnReadRDDV2(@transient sc: SparkContext, val requestedSchema = requiredSchema.schema val instantRange = InstantRange.builder().rangeType(RangeType.EXACT_MATCH).explicitInstants(validInstants.value).build() val readerContext = new HoodieAvroReaderContext(storageConf, metaClient.getTableConfig, HOption.of(instantRange), HOption.empty().asInstanceOf[HOption[HPredicate]]) - val fileGroupReader: HoodieFileGroupReader[IndexedRecord] = HoodieFileGroupReader.newBuilder() + val fileGroupReader: HoodieFileGroupReader[IndexedRecord] = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(tableState.latestCommitTimestamp.orNull) @@ -181,13 +181,13 @@ class HoodieMergeOnReadRDDV2(@transient sc: SparkContext, .withProps(properties) .withDataSchema(tableSchema.schema) .withRequestedSchema(requestedSchema) - .withInternalSchema(HOption.ofNullable(tableSchema.internalSchema.orNull)) + .withInternalSchemaOpt(HOption.ofNullable(tableSchema.internalSchema.orNull)) .build() convertAvroToRowIterator(fileGroupReader.getClosableIterator, requestedSchema) } else { val readerContext = new SparkFileFormatInternalRowReaderContext(fileGroupBaseFileReader.value, optionalFilters, Seq.empty, storageConf, metaClient.getTableConfig) - val fileGroupReader = HoodieFileGroupReader.newBuilder() + val fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(tableState.latestCommitTimestamp.orNull) @@ -197,7 +197,7 @@ class HoodieMergeOnReadRDDV2(@transient sc: SparkContext, .withProps(properties) .withDataSchema(tableSchema.schema) .withRequestedSchema(requiredSchema.schema) - .withInternalSchema(HOption.ofNullable(tableSchema.internalSchema.orNull)) + .withInternalSchemaOpt(HOption.ofNullable(tableSchema.internalSchema.orNull)) .build() convertCloseableIterator(fileGroupReader.getClosableIterator) } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala index 7a97367bb3c59..4970be3c249b3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala @@ -513,13 +513,15 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit, } private def loadFileSlice(fileSlice: FileSlice, readerContext: SparkFileFormatInternalRowReaderContext): Iterator[BufferedRecord[InternalRow]] = { - val fileGroupReader = HoodieFileGroupReader.newBuilder() + val fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile) + .withLogFiles(fileSlice.getLogFiles) + .withPartitionPath(fileSlice.getPartitionPath) .withDataSchema(schema) .withRequestedSchema(schema) - .withInternalSchema(toJavaOption(originTableSchema.internalSchema)) + .withInternalSchemaOpt(toJavaOption(originTableSchema.internalSchema)) .withProps(readerProperties) .withLatestCommitTime(split.changes.last.getInstant) .build() diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala index de4ffb400d4c2..da5bef9f785ca 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala @@ -313,14 +313,16 @@ class HoodieFileGroupReaderBasedFileFormat(tablePath: String, } else { 0 } - val reader = HoodieFileGroupReader.newBuilder() + val reader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(queryTimestamp) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile) + .withLogFiles(fileSlice.getLogFiles) + .withPartitionPath(fileSlice.getPartitionPath) .withDataSchema(dataSchema) .withRequestedSchema(requestedSchema) - .withInternalSchema(internalSchemaOpt) + .withInternalSchemaOpt(internalSchemaOpt) .withProps(props) .withStart(file.start) .withLength(baseFileLength) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/PartitionBucketIndexManager.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/PartitionBucketIndexManager.scala index 6eaade123a649..4e044ecb6786f 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/PartitionBucketIndexManager.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/PartitionBucketIndexManager.scala @@ -225,14 +225,16 @@ class PartitionBucketIndexManager extends BaseProcedure // instantiate other supporting cast val internalSchemaOption: Option[InternalSchema] = Option.empty() // instantiate FG reader - val fileGroupReader = HoodieFileGroupReader.newBuilder() + val fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContextFactory.getContext) .withHoodieTableMetaClient(metaClient) .withLatestCommitTime(latestInstantTime.requestedTime()) - .withFileSlice(fileSlice) + .withBaseFileOption(fileSlice.getBaseFile) + .withLogFiles(fileSlice.getLogFiles) + .withPartitionPath(fileSlice.getPartitionPath) .withDataSchema(tableSchemaWithMetaFields) .withRequestedSchema(tableSchemaWithMetaFields) - .withInternalSchema(internalSchemaOption) // not support evolution of schema for now + .withInternalSchemaOpt(internalSchemaOption) // not support evolution of schema for now .withProps(metaClient.getTableConfig.getProps) .withShouldUseRecordPosition(false) .build() diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java index b35da0450cfdc..7dbd7e7aaf96d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java @@ -712,7 +712,7 @@ Set getRecordKeys(String partition, String baseInstantTime, String fileI TypedProperties properties = new TypedProperties(); // configure un-merged log file reader HoodieReaderContext readerContext = context.getReaderContextFactory(metaClient).getContext(); - HoodieFileGroupReader reader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader reader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withDataSchema(writerSchemaOpt.get()) .withRequestedSchema(writerSchemaOpt.get()) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java index b3beca5fa18c2..b0a4aab2e7899 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java @@ -1664,7 +1664,7 @@ private void verifyMetadataMergedRecords(HoodieTableMetaClient metadataMetaClien String latestCommitTimestamp) { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())); HoodieAvroReaderContext readerContext = new HoodieAvroReaderContext(metadataMetaClient.getStorageConf(), metadataMetaClient.getTableConfig(), Option.empty(), Option.empty()); - HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() .withReaderContext(readerContext) .withHoodieTableMetaClient(metadataMetaClient) .withLogFiles(logFiles.stream()) From 5cdfefe4118a99b4fbf25cce2df437a7eba819db Mon Sep 17 00:00:00 2001 From: fhan Date: Wed, 3 Jun 2026 18:44:39 +0800 Subject: [PATCH 042/255] fix(spark): align CTAS partition fields by table partition order (#18899) * fix(spark): fix CTAS partition field order * fix(spark): add a short comment --------- Co-authored-by: fhan (cherry picked from commit ba8c4c7b82c7c94405f712e5d1964226412f4e13) --- .../sql/hudi/analysis/HoodieAnalysis.scala | 34 +++++++++++++++- .../spark/sql/hudi/ddl/TestCreateTable.scala | 40 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala index d01627ae1fd56..46f9ed36ad780 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala @@ -423,7 +423,8 @@ case class ResolveImplementationsEarly(spark: SparkSession) extends Rule[Logical // Convert to CreateHoodieTableAsSelectCommand case ct @ CreateTable(table, mode, Some(query)) if sparkAdapter.isHoodieTable(table) && ct.query.forall(_.resolved) => - val alignedQuery = stripMetaFieldAttributes(query) + val alignedQuery = alignCtasQueryByPartitionOrder( + stripMetaFieldAttributes(query), table.partitionColumnNames) CreateHoodieTableAsSelectCommand(table, mode, alignedQuery) case ct: CreateTable => @@ -445,6 +446,37 @@ case class ResolveImplementationsEarly(spark: SparkSession) extends Rule[Logical case _ => plan } } + + private def alignCtasQueryByPartitionOrder(query: LogicalPlan, partitionColumns: Seq[String]): LogicalPlan = { + if (partitionColumns.isEmpty) { + query + } else { + val resolver = spark.sessionState.conf.resolver + val (dataAttrs, partitionAttrs) = query.output.partition { attr => + !partitionColumns.exists(partition => resolver(partition, attr.name)) + } + + if (partitionAttrs.size != partitionColumns.size) { + throw new HoodieAnalysisException(s"Partition columns ${partitionColumns.mkString("[", ", ", "]")} " + + s"do not match query output ${query.output.map(_.name).mkString("[", ", ", "]")}") + } + + val alreadyAligned = partitionColumns.zip(partitionAttrs).forall { + case (partition, attr) => resolver(partition, attr.name) + } + // Avoid adding a redundant Project when partition columns are already in the table-defined order. + if (alreadyAligned) { + query + } else { + val orderedPartitionAttrs = partitionColumns.map { partition => + partitionAttrs.find(attr => resolver(partition, attr.name)).getOrElse { + throw new HoodieAnalysisException(s"Cannot resolve partition column $partition in CTAS query output") + } + } + Project(dataAttrs ++ orderedPartitionAttrs, query) + } + } + } } /** diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala index ead9943421620..88ec4b7fb4c36 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala @@ -416,6 +416,46 @@ class TestCreateTable extends HoodieSparkSqlTestBase { Seq(1, "a1", 10, "2021-04-01") ) + // Create table with multi-level partition + val tableNameMultiLevelPartition = generateTableName + spark.sql( + s""" + | create table $tableNameMultiLevelPartition using hudi + | partitioned by (year, month, day) + | tblproperties( + | primaryKey = 'id', + | type = '$tableType' + | ) + | location '${tmp.getCanonicalPath}/$tableNameMultiLevelPartition' + | AS + | select 1 as id, 'a1' as name, 10 as price, '2021' as year, '04' as month, '01' as day + """.stripMargin + ) + + checkAnswer(s"select id, name, price, year, month, day from $tableNameMultiLevelPartition")( + Seq(1, "a1", 10, "2021", "04", "01") + ) + + // Create table with multi-level partition and out-of-order partition columns + val tableNameMultiLevelPartitionDisorder = generateTableName + spark.sql( + s""" + | create table $tableNameMultiLevelPartitionDisorder using hudi + | partitioned by (year, month, day) + | tblproperties( + | primaryKey = 'id', + | type = '$tableType' + | ) + | location '${tmp.getCanonicalPath}/$tableNameMultiLevelPartitionDisorder' + | AS + | select 1 as id, 'a1' as name, 10 as price, '04' as month, '01' as day, '2021' as year + """.stripMargin + ) + + checkAnswer(s"select id, name, price, year, month, day from $tableNameMultiLevelPartitionDisorder")( + Seq(1, "a1", 10, "2021", "04", "01") + ) + // Create Partitioned table with timestamp data type val tableName3 = generateTableName // CTAS failed with null primaryKey From 7c8020b6aa313130c35a4d55d4019f8690e416a2 Mon Sep 17 00:00:00 2001 From: Geser Dugarov Date: Wed, 3 Jun 2026 17:52:10 +0700 Subject: [PATCH 043/255] fix(ci): Flink version corresponds to used in Docker image, resolution for Everit JSON schema (#18905) (cherry picked from commit 8ac3092dc4e3dff54235547bc094d621fdafd488) --- packaging/bundle-validation/run_docker_java17.sh | 2 +- pom.xml | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packaging/bundle-validation/run_docker_java17.sh b/packaging/bundle-validation/run_docker_java17.sh index a380319a210ab..56d19bb147704 100755 --- a/packaging/bundle-validation/run_docker_java17.sh +++ b/packaging/bundle-validation/run_docker_java17.sh @@ -77,7 +77,7 @@ elif [[ ${SPARK_RUNTIME} == 'spark4.0.0' && ${SCALA_PROFILE} == 'scala-2.13' ]]; HADOOP_VERSION=3.4.0 HIVE_VERSION=3.1.3 DERBY_VERSION=10.14.1.0 - FLINK_VERSION=1.18.0 + FLINK_VERSION=1.20.0 SPARK_VERSION=4.0.0 SPARK_HADOOP_VERSION=3 CONFLUENT_VERSION=5.5.12 diff --git a/pom.xml b/pom.xml index 0e556cc76c1cd..a7b6e797efec1 100644 --- a/pom.xml +++ b/pom.xml @@ -1977,6 +1977,16 @@ false + + confluent + https://packages.confluent.io/maven/ + + + + jitpack.io + https://jitpack.io + cloudera-repo-releases https://repository.cloudera.com/artifactory/public/ @@ -1987,10 +1997,6 @@ false - - confluent - https://packages.confluent.io/maven/ - From 920d63ccb55c2dba07b69c659b6fca8215e80bb9 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 3 Jun 2026 19:42:35 +0800 Subject: [PATCH 044/255] refactor: Add Lombok annotations to hudi-utilities (Part 3) (#17877) (cherry picked from commit 508e2955ef8be9d0062b5ea46baf3f183c5a411e) --- .../utilities/streamer/BootstrapExecutor.java | 16 +-- .../streamer/DefaultStreamContext.java | 20 +--- .../hudi/utilities/streamer/ErrorEvent.java | 37 +----- .../streamer/HoodieMultiTableStreamer.java | 34 ++---- .../utilities/streamer/HoodieStreamer.java | 62 +++++----- .../streamer/HoodieStreamerUtils.java | 8 +- .../NoNewDataTerminationStrategy.java | 8 +- .../streamer/SchedulerConfGenerator.java | 12 +- .../streamer/SourceFormatAdapter.java | 23 +--- .../streamer/SparkSampleWritesUtils.java | 24 ++-- .../hudi/utilities/streamer/StreamSync.java | 106 +++++++----------- .../streamer/StreamerCheckpointUtils.java | 9 +- .../streamer/TableExecutionContext.java | 58 ++-------- .../TestHoodieMetadataTableValidator.java | 24 +--- .../hudi/utilities/TestHoodieRepairTool.java | 8 +- .../HoodieDeltaStreamerTestBase.java | 34 +++--- .../MockConfigurationHotUpdateStrategy.java | 7 +- .../TestHoodieDeltaStreamer.java | 68 ++++++----- ...estHoodieDeltaStreamerWithMultiWriter.java | 18 ++- .../TestHoodieMultiTableDeltaStreamer.java | 12 +- .../functional/TestHiveSchemaProvider.java | 10 +- .../TestJdbcbasedSchemaProvider.java | 7 +- .../TestHoodieMultiTableServicesMain.java | 14 +-- .../sources/BaseTestKafkaSource.java | 22 +--- .../S3EventsHoodieIncrSourceHarness.java | 23 ++-- .../utilities/sources/TestDataSource.java | 11 +- .../TestGcsEventsHoodieIncrSource.java | 6 +- .../sources/TestHoodieIncrSource.java | 39 ++----- .../utilities/testutils/JdbcTestUtils.java | 18 ++- .../testutils/UtilitiesTestBase.java | 7 +- .../sources/AbstractBaseTestSource.java | 18 ++- .../sources/DistributedTestDataSource.java | 10 +- 32 files changed, 273 insertions(+), 500 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BootstrapExecutor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BootstrapExecutor.java index c9328cfc3bffe..24ce507b34a7c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BootstrapExecutor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BootstrapExecutor.java @@ -44,12 +44,12 @@ import org.apache.hudi.utilities.UtilHelpers; import org.apache.hudi.utilities.schema.SchemaProvider; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -71,10 +71,9 @@ /** * Performs bootstrap from a non-hudi source. */ +@Slf4j public class BootstrapExecutor implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(BootstrapExecutor.class); - /** * Config. */ @@ -103,6 +102,7 @@ public class BootstrapExecutor implements Serializable { /** * Bootstrap Configuration. */ + @Getter private final HoodieWriteConfig bootstrapConfig; /** @@ -146,7 +146,7 @@ public BootstrapExecutor(HoodieStreamer.Config cfg, JavaSparkContext jssc, FileS builder = builder.withSchema(schemaProvider.getTargetHoodieSchema().toString()); } this.bootstrapConfig = builder.build(); - LOG.info("Created bootstrap executor with configs : " + bootstrapConfig.getProps()); + log.info("Created bootstrap executor with configs: {}", bootstrapConfig.getProps()); } /** @@ -189,7 +189,7 @@ private void initializeTable() throws IOException { Path basePath = new Path(cfg.targetBasePath); if (fs.exists(basePath)) { if (cfg.bootstrapOverwrite) { - LOG.info("Target base path already exists, overwrite it"); + log.info("Target base path already exists, overwrite it"); fs.delete(basePath, true); } else { throw new HoodieException("target base path already exists at " + cfg.targetBasePath @@ -244,8 +244,4 @@ private void initializeTable() throws IOException { builder.initTable(HadoopFSUtils.getStorageConfWithCopy(jssc.hadoopConfiguration()), cfg.targetBasePath); } - - public HoodieWriteConfig getBootstrapConfig() { - return bootstrapConfig; - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/DefaultStreamContext.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/DefaultStreamContext.java index f8dabeb89c96c..46972fe353cf7 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/DefaultStreamContext.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/DefaultStreamContext.java @@ -21,28 +21,18 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.schema.SchemaProvider; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * The default implementation for the StreamContext interface, * composes SchemaProvider and SourceProfileSupplier currently, * can be extended for other arguments in the future. */ +@AllArgsConstructor +@Getter public class DefaultStreamContext implements StreamContext { private final SchemaProvider schemaProvider; private final Option sourceProfileSupplier; - - public DefaultStreamContext(SchemaProvider schemaProvider, Option sourceProfileSupplier) { - this.schemaProvider = schemaProvider; - this.sourceProfileSupplier = sourceProfileSupplier; - } - - @Override - public SchemaProvider getSchemaProvider() { - return schemaProvider; - } - - @Override - public Option getSourceProfileSupplier() { - return sourceProfileSupplier; - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorEvent.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorEvent.java index a2f1cb277ec60..5bbba78e3921c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorEvent.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorEvent.java @@ -19,45 +19,16 @@ package org.apache.hudi.utilities.streamer; -import java.util.Objects; +import lombok.Value; /** * Error event is an event triggered during write or processing failure of a record. */ +@Value public class ErrorEvent { - private final ErrorReason reason; - private final T payload; - - public ErrorEvent(T payload, ErrorReason reason) { - this.payload = payload; - this.reason = reason; - } - - public T getPayload() { - return payload; - } - - public ErrorReason getReason() { - return reason; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ErrorEvent that = (ErrorEvent) o; - return reason == that.reason && Objects.equals(payload, that.payload); - } - - @Override - public int hashCode() { - return Objects.hash(reason, payload); - } + T payload; + ErrorReason reason; /** * The reason behind write or processing failure of a record diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java index 90f8f6558ded9..56101e9ae93ad 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java @@ -39,12 +39,13 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -69,11 +70,12 @@ * Helps with ingesting incremental data into hoodie datasets for multiple tables. * Supports COPY_ON_WRITE and MERGE_ON_READ storage types. */ +@Getter +@Slf4j public class HoodieMultiTableStreamer { - private static final Logger LOG = LoggerFactory.getLogger(HoodieMultiTableStreamer.class); - private final List tableExecutionContexts; + @Getter(AccessLevel.NONE) private transient JavaSparkContext jssc; private final Set successTables; private final Set failedTables; @@ -120,7 +122,7 @@ private void checkIfTableConfigFileExists(String configFolder, FileSystem fs, St //commonProps are passed as parameter which contain table to config file mapping private void populateTableExecutionContextList(TypedProperties properties, String configFolder, FileSystem fs, Config config) throws IOException { List tablesToBeIngested = getTablesToBeIngested(properties); - LOG.info("tables to be ingested via MultiTableDeltaStreamer : " + tablesToBeIngested); + log.info("tables to be ingested via MultiTableDeltaStreamer : {}", tablesToBeIngested); TableExecutionContext executionContext; for (String table : tablesToBeIngested) { String[] tableWithDatabase = table.split("\\."); @@ -271,11 +273,11 @@ public static void main(String[] args) throws IOException { } if (config.enableHiveSync) { - LOG.warn("--enable-hive-sync will be deprecated in a future release; please use --enable-sync instead for Hive syncing"); + log.warn("--enable-hive-sync will be deprecated in a future release; please use --enable-sync instead for Hive syncing"); } if (config.targetTableName != null) { - LOG.warn("--target-table is deprecated and will be removed in a future release due to it's useless;" + log.warn("--target-table is deprecated and will be removed in a future release due to it's useless;" + " please use {} to configure multiple target tables", HoodieStreamerConfig.TABLES_TO_BE_INGESTED.key()); } @@ -469,7 +471,7 @@ public void sync() { successTables.add(Helpers.getTableWithDatabase(context)); streamer.shutdownGracefully(); } catch (Exception e) { - LOG.error("error while running MultiTableDeltaStreamer for table: " + context.getTableName(), e); + log.error("error while running MultiTableDeltaStreamer for table: {}", context.getTableName(), e); failedTables.add(Helpers.getTableWithDatabase(context)); } finally { if (streamer != null) { @@ -478,9 +480,9 @@ public void sync() { } } - LOG.info("Ingestion was successful for topics: " + successTables); + log.info("Ingestion was successful for topics: {}", successTables); if (!failedTables.isEmpty()) { - LOG.info("Ingestion failed for topics: " + failedTables); + log.info("Ingestion failed for topics: {}", failedTables); } } @@ -496,16 +498,4 @@ public static class Constants { private static final String UNDERSCORE = "_"; private static final String COMMA_SEPARATOR = ","; } - - public Set getSuccessTables() { - return successTables; - } - - public Set getFailedTables() { - return failedTables; - } - - public List getTableExecutionContexts() { - return this.tableExecutionContexts; - } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java index 84929ca695569..0186e5fd05840 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java @@ -74,14 +74,14 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -96,7 +96,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static java.lang.String.format; import static org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1.STREAMER_CHECKPOINT_RESET_KEY_V1; import static org.apache.hudi.common.util.ValidationUtils.checkArgument; import static org.apache.hudi.utilities.UtilHelpers.buildProperties; @@ -111,10 +110,10 @@ * write-to-sink (c) Schedule Compactions if needed (d) Conditionally Sync to Hive each cycle. For MOR table with * continuous mode enabled, a separate compactor thread is allocated to execute compactions */ +@Slf4j public class HoodieStreamer implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieStreamer.class); private static final List DEFAULT_SENSITIVE_CONFIG_KEYS = Arrays.asList( HoodieWriteConfig.SENSITIVE_CONFIG_KEYS_FILTER.defaultValue().split(",")); private static final String SENSITIVE_VALUES_MASKED = "SENSITIVE_INFO_MASKED"; @@ -214,9 +213,9 @@ public static TypedProperties combineProperties(Config cfg, Option { - LOG.info("Shutting down DeltaStreamer"); + log.info("Shutting down DeltaStreamer"); ds.shutdown(false); - LOG.info("Async service shutdown complete. Closing DeltaSync "); + log.info("Async service shutdown complete. Closing DeltaSync "); ds.close(); }); } @@ -226,7 +225,7 @@ public void shutdownGracefully() { */ public void sync() throws Exception { if (bootstrapExecutor.isPresent()) { - LOG.info("Performing bootstrap. Source=" + bootstrapExecutor.get().getBootstrapConfig().getBootstrapSourceBasePath()); + log.info("Performing bootstrap. Source={}", bootstrapExecutor.get().getBootstrapConfig().getBootstrapSourceBasePath()); bootstrapExecutor.get().execute(); } else { ingestionService.ifPresent(HoodieIngestionService::startIngestion); @@ -608,7 +607,7 @@ public static String toSortedTruncatedString(TypedProperties props) { for (String key : allKeys) { String value = Option.ofNullable(props.get(key)).orElse("").toString(); // Truncate too long values. - if (value.length() > 255 && !LOG.isDebugEnabled()) { + if (value.length() > 255 && !log.isDebugEnabled()) { value = value.substring(0, 128) + "[...]"; } @@ -648,7 +647,7 @@ public static void main(String[] args) throws Exception { jssc = UtilHelpers.buildSparkContext(sparkAppName, cfg.sparkMaster, cfg.enableHiveSupport, additionalSparkConfigs); } if (cfg.enableHiveSync) { - LOG.warn("--enable-hive-sync will be deprecated in a future release; please use --enable-sync instead for Hive syncing"); + log.warn("--enable-hive-sync will be deprecated in a future release; please use --enable-sync instead for Hive syncing"); } int exitCode = 0; @@ -676,11 +675,13 @@ public static class StreamSyncService extends HoodieIngestionService { /** * Schema provider that supplies the command for reading the input and writing out the target table. */ + @Getter private transient SchemaProvider schemaProvider; /** * Spark Session. */ + @Getter private transient SparkSession sparkSession; /** @@ -696,6 +697,7 @@ public static class StreamSyncService extends HoodieIngestionService { /** * Bag of properties with source, hoodie client, key generator etc. */ + @Getter TypedProperties props; /** @@ -757,7 +759,7 @@ public StreamSyncService(Config cfg, HoodieSparkEngineContext hoodieSparkContext properties.get().forEach((k, v) -> propsToValidate.put(k.toString(), v.toString())); HoodieWriterUtils.validateTableConfig(this.sparkSession, org.apache.hudi.HoodieConversionUtils.mapAsScalaImmutableMap(propsToValidate), meta.getTableConfig()); } catch (HoodieIOException e) { - LOG.warn("Full exception msg {}, msg {}", e.getLocalizedMessage(), e.getMessage()); + log.warn("Full exception msg {}, msg {}", e.getLocalizedMessage(), e.getMessage()); if (e.getMessage().contains("Could not load Hoodie properties") && e.getMessage().contains(HoodieTableConfig.HOODIE_PROPERTIES_FILE)) { initializeTableTypeAndBaseFileFormat(); } else { @@ -772,7 +774,7 @@ public StreamSyncService(Config cfg, HoodieSparkEngineContext hoodieSparkContext "'--filter-dupes' needs to be disabled when '--op' is 'UPSERT' to ensure updates are not missed."); this.props = properties.get(); - LOG.info(toSortedTruncatedString(props)); + log.info(toSortedTruncatedString(props)); this.schemaProvider = UtilHelpers.wrapSchemaProviderWithPostProcessor( UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, hoodieSparkContext.jsc()), @@ -817,7 +819,7 @@ protected Pair startService() { boolean error = false; if (cfg.isAsyncCompactionEnabled()) { // set Scheduler Pool. - LOG.info("Setting Spark Pool name for delta-sync to " + STREAMSYNC_POOL_NAME); + log.info("Setting Spark Pool name for delta-sync to {}", STREAMSYNC_POOL_NAME); hoodieSparkContext.setProperty(EngineProperty.DELTASYNC_POOL_NAME, STREAMSYNC_POOL_NAME); } @@ -834,14 +836,14 @@ protected Pair startService() { if (newProps.isPresent()) { this.props = newProps.get(); // reinit the DeltaSync only when the props updated - LOG.info("Re-init delta sync with new config properties:"); - LOG.info(toSortedTruncatedString(props)); + log.info("Re-init delta sync with new config properties:"); + log.info(toSortedTruncatedString(props)); reInitDeltaSync(); } } Option, JavaRDD>> scheduledCompactionInstantAndRDD = Option.ofNullable(streamSync.syncOnce()); if (scheduledCompactionInstantAndRDD.isPresent() && scheduledCompactionInstantAndRDD.get().getLeft().isPresent()) { - LOG.info("Enqueuing new pending compaction instant (" + scheduledCompactionInstantAndRDD.get().getLeft() + ")"); + log.info("Enqueuing new pending compaction instant ({})", scheduledCompactionInstantAndRDD.get().getLeft()); asyncCompactService.get().enqueuePendingAsyncServiceInstant(scheduledCompactionInstantAndRDD.get().getLeft().get()); asyncCompactService.get().waitTillPendingAsyncServiceInstantsReducesTo(cfg.maxPendingCompactions); if (asyncCompactService.get().hasError()) { @@ -852,7 +854,7 @@ protected Pair startService() { if (clusteringConfig.isAsyncClusteringEnabled()) { Option clusteringInstant = streamSync.getClusteringInstantOpt(); if (clusteringInstant.isPresent()) { - LOG.info("Scheduled async clustering for instant: " + clusteringInstant.get()); + log.info("Scheduled async clustering for instant: {}", clusteringInstant.get()); asyncClusteringService.get().enqueuePendingAsyncServiceInstant(clusteringInstant.get()); asyncClusteringService.get().waitTillPendingAsyncServiceInstantsReducesTo(cfg.maxPendingClustering); if (asyncClusteringService.get().hasError()) { @@ -865,7 +867,7 @@ protected Pair startService() { Option> lastWriteStatuses = Option.ofNullable( scheduledCompactionInstantAndRDD.isPresent() ? HoodieJavaRDD.of(scheduledCompactionInstantAndRDD.get().getRight()) : null); if (requestShutdownIfNeeded(lastWriteStatuses)) { - LOG.info("Closing and shutting down ingestion service"); + log.info("Closing and shutting down ingestion service"); error = true; onIngestionCompletes(false); shutdown(true); @@ -875,7 +877,7 @@ protected Pair startService() { } catch (HoodieUpsertException ue) { handleUpsertException(ue); } catch (Exception e) { - LOG.error("Shutting down delta-sync due to exception", e); + log.error("Shutting down delta-sync due to exception", e); error = true; throw new HoodieException(e.getMessage(), e); } @@ -890,7 +892,7 @@ protected Pair startService() { private void handleUpsertException(HoodieUpsertException ue) { if (ue.getCause() instanceof HoodieClusteringUpdateException) { - LOG.warn("Write rejected due to conflicts with pending clustering operation. Going to retry after 1 min with the hope " + log.warn("Write rejected due to conflicts with pending clustering operation. Going to retry after 1 min with the hope " + "that clustering will complete by then.", ue); try { Thread.sleep(60000); // Intentionally not using cfg.minSyncIntervalSeconds, since it could be too high or it could be 0. @@ -907,13 +909,13 @@ private void handleUpsertException(HoodieUpsertException ue) { * Shutdown async services like compaction/clustering as DeltaSync is shutdown. */ private void shutdownAsyncServices(boolean error) { - LOG.info("Delta Sync shutdown. Error ?{}", error); + log.info("Delta Sync shutdown. Error ?{}", error); if (asyncCompactService.isPresent()) { - LOG.info("Gracefully shutting down compactor"); + log.info("Gracefully shutting down compactor"); asyncCompactService.get().shutdown(false); } if (asyncClusteringService.isPresent()) { - LOG.info("Gracefully shutting down clustering service"); + log.info("Gracefully shutting down clustering service"); asyncClusteringService.get().shutdown(false); } } @@ -979,7 +981,7 @@ protected Boolean onInitializingWriteClient(SparkRDDWriteClient writeClient) { .setBasePath(cfg.targetBasePath) .setLoadActiveTimelineOnLoad(true).build(); List pending = ClusteringUtils.getPendingClusteringInstantTimes(meta); - LOG.info(format("Found %d pending clustering instants ", pending.size())); + log.info("Found {} pending clustering instants ", pending.size()); pending.forEach(hoodieInstant -> asyncClusteringService.get().enqueuePendingAsyncServiceInstant(hoodieInstant.requestedTime())); asyncClusteringService.get().start(error -> true); try { @@ -997,7 +999,7 @@ protected Boolean onInitializingWriteClient(SparkRDDWriteClient writeClient) { @Override protected boolean onIngestionCompletes(boolean hasError) { - LOG.info("Ingestion completed. Has error: " + hasError); + log.info("Ingestion completed. Has error: {}", hasError); close(); return true; } @@ -1014,18 +1016,6 @@ public void close() { } } - public SchemaProvider getSchemaProvider() { - return schemaProvider; - } - - public SparkSession getSparkSession() { - return sparkSession; - } - - public TypedProperties getProps() { - return props; - } - @VisibleForTesting public HoodieSparkEngineContext getHoodieSparkContext() { return hoodieSparkContext; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java index 12e620156c494..9e455795e5923 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java @@ -54,6 +54,7 @@ import org.apache.hudi.util.SparkKeyGenUtils; import org.apache.hudi.utilities.schema.SchemaProvider; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -64,8 +65,6 @@ import org.apache.spark.sql.avro.HoodieAvroDeserializer; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.Iterator; @@ -79,10 +78,9 @@ /** * Util class for HoodieStreamer. */ +@Slf4j public class HoodieStreamerUtils { - private static final Logger LOG = LoggerFactory.getLogger(HoodieStreamerUtils.class); - /** * Generates HoodieRecords for the avro data read from source. * Takes care of dropping columns, precombine, auto key generation. @@ -111,7 +109,7 @@ public static Option> createHoodieRecords(HoodieStreamer.C records = avroRDD.mapPartitions( (FlatMapFunction, Either>) genericRecordIterator -> { TaskContext taskContext = TaskContext.get(); - LOG.info("Creating HoodieRecords with stageId : {}, stage attempt no: {}, taskId : {}, task attempt no : {}, task attempt id : {} ", + log.info("Creating HoodieRecords with stageId : {}, stage attempt no: {}, taskId : {}, task attempt no : {}, task attempt id : {} ", taskContext.stageId(), taskContext.stageAttemptNumber(), taskContext.partitionId(), taskContext.attemptNumber(), taskContext.taskAttemptId()); if (autoGenerateRecordKeys) { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/NoNewDataTerminationStrategy.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/NoNewDataTerminationStrategy.java index 686bdb52e3c7b..08e4cf25dcbce 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/NoNewDataTerminationStrategy.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/NoNewDataTerminationStrategy.java @@ -23,17 +23,15 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.Option; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Post writer termination strategy for deltastreamer in continuous mode. This strategy is based on no new data for consecutive number of times. */ +@Slf4j public class NoNewDataTerminationStrategy implements PostWriteTerminationStrategy { - private static final Logger LOG = LoggerFactory.getLogger(NoNewDataTerminationStrategy.class); - public static final String MAX_ROUNDS_WITHOUT_NEW_DATA_TO_SHUTDOWN = "max.rounds.without.new.data.to.shutdown"; public static final int DEFAULT_MAX_ROUNDS_WITHOUT_NEW_DATA_TO_SHUTDOWN = 3; @@ -48,7 +46,7 @@ public NoNewDataTerminationStrategy(TypedProperties properties) { public boolean shouldShutdown(Option> writeStatuses) { numTimesNoNewData = writeStatuses.isPresent() ? 0 : numTimesNoNewData + 1; if (numTimesNoNewData >= numTimesNoNewDataToShutdown) { - LOG.info("Shutting down on continuous mode as there is no new data for " + numTimesNoNewData); + log.info("Shutting down on continuous mode as there is no new data for {}", numTimesNoNewData); return true; } return false; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SchedulerConfGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SchedulerConfGenerator.java index 19df192aad8ef..5e1c6ff2692e4 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SchedulerConfGenerator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SchedulerConfGenerator.java @@ -24,9 +24,8 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.util.Option; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.BufferedWriter; import java.io.File; @@ -42,10 +41,9 @@ * Utility Class to generate Spark Scheduling allocation file. This kicks in only when user sets * spark.scheduler.mode=FAIR at spark-submit time */ +@Slf4j public class SchedulerConfGenerator { - private static final Logger LOG = LoggerFactory.getLogger(SchedulerConfGenerator.class); - public static final String DELTASYNC_POOL_NAME = HoodieStreamer.STREAMSYNC_POOL_NAME; public static final String COMPACT_POOL_NAME = AsyncCompactService.COMPACT_POOL_NAME; public static final String SPARK_SCHEDULER_MODE_KEY = "spark.scheduler.mode"; @@ -106,10 +104,10 @@ public static Map getSparkSchedulingConfigs(HoodieStreamer.Confi String sparkSchedulingConfFile = generateAndStoreConfig(cfg.deltaSyncSchedulingWeight, cfg.compactSchedulingWeight, cfg.deltaSyncSchedulingMinShare, cfg.compactSchedulingMinShare, cfg.clusterSchedulingWeight, cfg.clusterSchedulingMinShare); - LOG.info("Spark scheduling config file {}", sparkSchedulingConfFile); + log.info("Spark scheduling config file {}", sparkSchedulingConfFile); additionalSparkConfigs.put(SparkConfigs.SPARK_SCHEDULER_ALLOCATION_FILE_KEY(), sparkSchedulingConfFile); } else { - LOG.warn("Job Scheduling Configs will not be in effect as spark.scheduler.mode " + log.warn("Job Scheduling Configs will not be in effect as spark.scheduler.mode " + "is not set to FAIR at instantiation time. Continuing without scheduling configs"); } return additionalSparkConfigs; @@ -135,7 +133,7 @@ private static String generateAndStoreConfig(Integer deltaSyncWeight, Integer co } // SPARK-35083 introduces remote scheduler pool files, so the file must include scheme since Spark 3.2 String path = tempConfigFile.toURI().toString(); - LOG.info("Configs written to file " + path); + log.info("Configs written to file {}", path); return path; } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java index 763caf8f54fec..0d63ffd6c0f45 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java @@ -43,6 +43,8 @@ import org.apache.hudi.utilities.sources.helpers.SanitizationUtils; import com.google.protobuf.Message; +import lombok.AccessLevel; +import lombok.Getter; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Column; @@ -73,12 +75,15 @@ */ public class SourceFormatAdapter implements Closeable { + @Getter private final Source source; private boolean shouldSanitize = SANITIZE_SCHEMA_FIELD_NAMES.defaultValue(); private boolean wrapWithException = ROW_THROW_EXPLICIT_EXCEPTIONS.defaultValue(); + @Getter(AccessLevel.PRIVATE) private String invalidCharMask = SCHEMA_FIELD_NAME_INVALID_CHAR_MASK.defaultValue(); + @Getter(AccessLevel.PRIVATE) private boolean useJava8api = (boolean) SQLConf.DATETIME_JAVA8API_ENABLED().defaultValue().get(); @@ -110,18 +115,6 @@ private boolean isFieldNameSanitizingEnabled() { return shouldSanitize; } - /** - * Replacement mask for invalid characters encountered in avro names. - * @return sanitized value. - */ - private String getInvalidCharMask() { - return invalidCharMask; - } - - private boolean getUseJava8api() { - return useJava8api; - } - /** * transform input rdd of json string to generic records with support for adding error events to error table * @param inputBatch @@ -144,7 +137,7 @@ private JavaRDD transformJsonToGenericRdd(InputBatch transformJsonToRowRdd(InputBatch> inputBatch) { MercifulJsonConverter.clearCache(inputBatch.getSchemaProvider().getSourceHoodieSchema().getFullName()); - RowConverter convertor = new RowConverter(inputBatch.getSchemaProvider().getSourceHoodieSchema(), isFieldNameSanitizingEnabled(), getInvalidCharMask(), getUseJava8api()); + RowConverter convertor = new RowConverter(inputBatch.getSchemaProvider().getSourceHoodieSchema(), isFieldNameSanitizingEnabled(), getInvalidCharMask(), isUseJava8api()); return inputBatch.getBatch().map(rdd -> { if (errorTableWriter.isPresent()) { JavaRDD> javaRDD = rdd.map(convertor::fromJsonToRowWithError); @@ -323,10 +316,6 @@ public InputBatch> fetchNewDataInRowFormat(Option lastC } } - public Source getSource() { - return source; - } - @Override public void close() { source.releaseResources(); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java index 05aeca28067b0..c50ff3484cad2 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java @@ -36,11 +36,10 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -58,31 +57,30 @@ *

    * TODO handle sample_writes sub-path clean-up w.r.t. rollback and insert overwrite. (HUDI-6044) */ +@Slf4j public class SparkSampleWritesUtils { - private static final Logger LOG = LoggerFactory.getLogger(SparkSampleWritesUtils.class); - public static Option getWriteConfigWithRecordSizeEstimate(JavaSparkContext jsc, Option> recordsOpt, HoodieWriteConfig writeConfig) { if (!writeConfig.getBoolean(SAMPLE_WRITES_ENABLED)) { - LOG.debug("Skip overwriting record size estimate as it's disabled."); + log.debug("Skip overwriting record size estimate as it's disabled."); return Option.empty(); } HoodieTableMetaClient metaClient = getMetaClient(jsc, writeConfig.getBasePath()); if (metaClient.isTimelineNonEmpty()) { - LOG.info("Skip overwriting record size estimate due to timeline is non-empty."); + log.info("Skip overwriting record size estimate due to timeline is non-empty."); return Option.empty(); } try { Pair result = doSampleWrites(jsc, recordsOpt, writeConfig); if (result.getLeft()) { long avgSize = getAvgSizeFromSampleWrites(jsc, result.getRight()); - LOG.info("Overwriting record size estimate to {}", avgSize); + log.info("Overwriting record size estimate to {}", avgSize); TypedProperties props = writeConfig.getProps(); props.put(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(avgSize)); return Option.of(HoodieWriteConfig.newBuilder().withProperties(props).build()); } } catch (IOException e) { - LOG.error(String.format("Not overwriting record size estimate for table %s due to error when doing sample writes.", writeConfig.getTableName()), e); + log.error("Not overwriting record size estimate for table {} due to error when doing sample writes.", writeConfig.getTableName(), e); } return Option.empty(); } @@ -117,13 +115,13 @@ private static Pair doSampleWrites(JavaSparkContext jsc, Option String instantTime = sampleWriteClient.startCommit(); JavaRDD writeStatusRDD = sampleWriteClient.bulkInsert(jsc.parallelize(samples, 1), instantTime); if (writeStatusRDD.filter(WriteStatus::hasErrors).count() > 0) { - LOG.error("sample writes for table {} failed with errors.", writeConfig.getTableName()); - if (LOG.isTraceEnabled()) { - LOG.trace("Printing out the top 100 errors"); + log.error("sample writes for table {} failed with errors.", writeConfig.getTableName()); + if (log.isTraceEnabled()) { + log.trace("Printing out the top 100 errors"); writeStatusRDD.filter(WriteStatus::hasErrors).take(100).forEach(ws -> { - LOG.trace("Global error :", ws.getGlobalError()); + log.trace("Global error :", ws.getGlobalError()); ws.getErrors().forEach((key, throwable) -> - LOG.trace(String.format("Error for key: %s", key), throwable)); + log.trace("Error for key: {}", key, throwable)); }); } return emptyRes; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 9811fe9b01784..1af9c503b6130 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -112,11 +112,13 @@ import org.apache.hudi.utilities.schema.SimpleSchemaProvider; import org.apache.hudi.utilities.sources.InputBatch; import org.apache.hudi.utilities.sources.Source; -import org.apache.hudi.utilities.streamer.HoodieStreamer.Config; import org.apache.hudi.utilities.streamer.validator.SparkStreamerValidatorUtils; import org.apache.hudi.utilities.transform.Transformer; import com.codahale.metrics.Timer; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -128,8 +130,6 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; import org.apache.spark.storage.StorageLevel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; @@ -177,16 +177,17 @@ /** * Sync's one batch of data to hoodie table. */ +@Slf4j public class StreamSync implements Serializable, Closeable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(StreamSync.class); private static final String NULL_PLACEHOLDER = "[null]"; public static final String CHECKPOINT_IGNORE_KEY = "deltastreamer.checkpoint.ignore_key"; /** * Delta Sync Config. */ + @Getter private final HoodieStreamer.Config cfg; /** @@ -214,6 +215,7 @@ public class StreamSync implements Serializable, Closeable { /** * Filesystem used. */ + @Getter private transient HoodieStorage storage; /** @@ -236,6 +238,7 @@ public class StreamSync implements Serializable, Closeable { * * NOTE: These properties are already consolidated w/ CLI provided config-overrides */ + @Getter private final TypedProperties props; /** @@ -246,6 +249,7 @@ public class StreamSync implements Serializable, Closeable { /** * Timeline with completed commits, including both .commit and .deltacommit. */ + @Getter private transient Option commitsTimelineOpt; // all commits timeline, including all (commits, delta commits, compaction, clean, savepoint, rollback, replace commits, index) @@ -270,6 +274,7 @@ public class StreamSync implements Serializable, Closeable { private Option errorTableWriter = Option.empty(); private HoodieErrorTableConfig.ErrorWriteFailureStrategy errorWriteFailureStrategy; + @Getter private transient HoodieIngestionMetrics metrics; private transient HoodieMetrics hoodieMetrics; @@ -394,7 +399,7 @@ private HoodieTableMetaClient initializeMetaClient(boolean refreshTimeline) thro } return metaClient; } catch (HoodieIOException e) { - LOG.warn("Full exception msg {}", e.getMessage()); + log.warn("Full exception msg {}", e.getMessage()); if (e.getMessage().contains("Could not load Hoodie properties") && e.getMessage().contains(HoodieTableConfig.HOODIE_PROPERTIES_FILE)) { String basePathWithForwardSlash = cfg.targetBasePath.endsWith("/") ? cfg.targetBasePath : String.format("%s/", cfg.targetBasePath); @@ -409,7 +414,7 @@ private HoodieTableMetaClient initializeMetaClient(boolean refreshTimeline) thro && storage.exists(new StoragePath(pathToHoodiePropsBackup)); if (!hoodiePropertiesExists) { - LOG.warn("Base path exists, but table is not fully initialized. Re-initializing again"); + log.warn("Base path exists, but table is not fully initialized. Re-initializing again"); HoodieTableMetaClient metaClientToValidate = initializeEmptyTable(); // reload the timeline from metaClient and validate that its empty table. If there are any instants found, then we should fail the pipeline, bcoz hoodie.properties got deleted by mistake. if (metaClientToValidate.reloadActiveTimeline().countInstants() > 0) { @@ -552,7 +557,7 @@ private void initializeWriteClientAndRetryTableServices(InputBatch inputBatch, H || (newTargetSchema != null && !processedSchema.isSchemaPresent(newTargetSchema))) { String sourceStr = newSourceSchema == null ? NULL_PLACEHOLDER : newSourceSchema.toString(true); String targetStr = newTargetSchema == null ? NULL_PLACEHOLDER : newTargetSchema.toString(true); - LOG.info("Seeing new schema. Source: {}, Target: {}", sourceStr, targetStr); + log.info("Seeing new schema. Source: {}, Target: {}", sourceStr, targetStr); // We need to recreate write client with new schema and register them. reInitWriteClient(newSourceSchema, newTargetSchema, inputBatch.getBatch(), metaClient); if (newSourceSchema != null) { @@ -607,7 +612,7 @@ private Option getLastPendingCompactionInstant(Option co public Pair readFromSource(HoodieTableMetaClient metaClient) throws IOException { // Retrieve the previous round checkpoints, if any Option checkpointToResume = StreamerCheckpointUtils.resolveCheckpointToResumeFrom(commitsTimelineOpt, cfg, props, metaClient); - LOG.info("Checkpoint to resume from : {}", checkpointToResume); + log.info("Checkpoint to resume from : {}", checkpointToResume); int maxRetryCount = cfg.retryOnSourceFailures ? cfg.maxRetryCount : 1; int curRetryCount = 0; @@ -620,11 +625,11 @@ public Pair readFromSource(HoodieTableMetaClient metaClient throw e; } try { - LOG.error("Exception thrown while fetching data from source. Msg : " + e.getMessage() + ", class : " + e.getClass() + ", cause : " + e.getCause()); - LOG.error("Sleeping for " + (cfg.retryIntervalSecs) + " before retrying again. Current retry count " + curRetryCount + ", max retry count " + cfg.maxRetryCount); + log.error("Exception thrown while fetching data from source. Msg : {}, class : {}, cause : {}", e.getMessage(), e.getClass(), e.getCause()); + log.error("Sleeping for {} before retrying again. Current retry count {}, max retry count {}", cfg.retryIntervalSecs, curRetryCount, cfg.maxRetryCount); Thread.sleep(cfg.retryIntervalSecs * 1000); } catch (InterruptedException ex) { - LOG.error("Ignoring InterruptedException while waiting to retry on source failure " + e.getMessage()); + log.error("Ignoring InterruptedException while waiting to retry on source failure {}", e.getMessage()); } } } @@ -648,8 +653,8 @@ private Pair fetchFromSourceAndPrepareRecords(Option fetchNextBatchFromSource(Option resumeChec inputBatchForWriter = new InputBatch<>(inputBatchNeedsDeduceSchema.getBatch(), inputBatchNeedsDeduceSchema.getCheckpointForNextBatch(), getDeducedSchemaProvider(inputBatchNeedsDeduceSchema.getSchemaProvider().getTargetHoodieSchema(), inputBatchNeedsDeduceSchema.getSchemaProvider(), metaClient)); } else { - LOG.warn("Row-writer is enabled but cannot be used due to the target schema"); + log.warn("Row-writer is enabled but cannot be used due to the target schema"); } } // if row writer was enabled but the target schema prevents us from using it, do not use the row writer @@ -913,7 +918,7 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood writeClient.rollback(instantTime); throw new HoodieStreamerWriteException("Error table commit failed for instant " + instantTime); case LOG_ERROR: - LOG.error("Error table write failed for instant {}", instantTime); + log.error("Error table write failed for instant {}", instantTime); break; default: throw new HoodieStreamerWriteException("Write failure strategy not implemented for " + errorWriteFailureStrategy); @@ -933,7 +938,7 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood SparkStreamerValidatorUtils.runValidators(props, instantTime, writeStatuses, checkpointCommitMetadata, metaClient); } catch (HoodieValidationException e) { - LOG.error("Pre-commit validators failed for instant {}", instantTime, e); + log.error("Pre-commit validators failed for instant {}", instantTime, e); writeClient.rollback(instantTime); throw new HoodieStreamerWriteException("Pre-commit validators failed for instant " + instantTime, e); } @@ -943,11 +948,11 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood SuccessfulRecordCounter.Counts counts = SuccessfulRecordCounter.compute( writeStatuses, errorTableWriteStatusRDDOpt, isErrorTableWriteUnificationEnabled); totalSuccessfulRecords.set(counts.getTotalSuccessfulRecords()); - LOG.info("instantTime={}, totalRecords={}, totalErrorRecords={}, totalSuccessfulRecords={}", + log.info("instantTime={}, totalRecords={}, totalErrorRecords={}, totalSuccessfulRecords={}", instantTime, counts.getTotalRecords(), counts.getTotalErrorRecords(), counts.getTotalSuccessfulRecords()); if (counts.getTotalRecords() == 0) { - LOG.info("No new data, perform empty commit."); + log.info("No new data, perform empty commit."); } // Step 4: Apply the legacy HSWSV write-error gate. @@ -960,10 +965,10 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood // failure-policy story. if (counts.hasErrors()) { if (cfg.commitOnErrors) { - LOG.warn("Some records failed to be merged but forcing commit since commitOnErrors set. Errors/Total={}/{}", + log.warn("Some records failed to be merged but forcing commit since commitOnErrors set. Errors/Total={}/{}", counts.getTotalErrorRecords(), counts.getTotalRecords()); } else { - LOG.error("Delta Sync found errors when writing. Errors/Total={}/{}", + log.error("Delta Sync found errors when writing. Errors/Total={}/{}", counts.getTotalErrorRecords(), counts.getTotalRecords()); WriteErrorReporter.logTopErrors(writeStatuses); // Roll back the inflight data-table instant so it doesn't leak under LAZY @@ -983,7 +988,7 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood } releaseResourcesInvoked = true; if (success) { - LOG.info("Commit " + instantTime + " successful!"); + log.info("Commit {} successful!", instantTime); this.formatAdapter.getSource().onCommit(inputBatch.getCheckpointForNextBatch() != null ? inputBatch.getCheckpointForNextBatch().getCheckpointKey() : null); // Schedule compaction if needed @@ -994,10 +999,10 @@ private Pair, JavaRDD> writeToSinkAndDoMetaSync(Hood if ((totalSuccessfulRecords.get() > 0) || cfg.forceEmptyMetaSync) { runMetaSync(); } else { - LOG.info(String.format("Not running metaSync totalSuccessfulRecords=%d", totalSuccessfulRecords.get())); + log.info("Not running metaSync totalSuccessfulRecords={}", totalSuccessfulRecords.get()); } } else { - LOG.info("Commit " + instantTime + " failed!"); + log.info("Commit {} failed!", instantTime); throw new HoodieStreamerWriteException("Commit " + instantTime + " failed!"); } @@ -1051,7 +1056,7 @@ private String startCommit(HoodieTableMetaClient metaClient, boolean retryEnable if (!retryEnabled) { throw ie; } - LOG.error("Got error trying to start a new commit. Retrying after sleeping for a sec", ie); + log.error("Got error trying to start a new commit. Retrying after sleeping for a sec", ie); retryNum++; try { Thread.sleep(1000); @@ -1131,10 +1136,10 @@ public void runMetaSync() { if (cfg.enableHiveSync) { cfg.enableMetaSync = true; syncClientToolClasses.add(HiveSyncTool.class.getName()); - LOG.info("When set --enable-hive-sync will use HiveSyncTool for backward compatibility"); + log.info("When set --enable-hive-sync will use HiveSyncTool for backward compatibility"); } if (cfg.enableMetaSync && !syncClientToolClasses.isEmpty()) { - LOG.debug("[MetaSync] Starting sync"); + log.debug("[MetaSync] Starting sync"); HoodieTableMetaClient metaClient; try { metaClient = initializeMetaClient(); @@ -1158,7 +1163,7 @@ public void runMetaSync() { Map failedMetaSyncs = new HashMap<>(); for (String impl : syncClientToolClasses) { if (impl.trim().isEmpty()) { - LOG.warn("Cannot run MetaSync with empty class name"); + log.warn("Cannot run MetaSync with empty class name"); continue; } @@ -1183,10 +1188,10 @@ private void logMetaSync(String impl, Timer.Context syncContext, Map> recordsOpt, HoodieTa } private void reInitWriteClient(HoodieSchema sourceSchema, HoodieSchema targetSchema, Option> recordsOpt, HoodieTableMetaClient metaClient) throws IOException { - LOG.info("Setting up new Hoodie Write Client"); + log.info("Setting up new Hoodie Write Client"); if (HoodieStreamerUtils.isDropPartitionColumns(props)) { targetSchema = org.apache.hudi.common.schema.HoodieSchemaUtils.removeFields(targetSchema, HoodieStreamerUtils.getPartitionColumns(props)); } @@ -1350,7 +1355,7 @@ private HoodieSchema getSchemaForWriteConfig(HoodieSchema targetSchema, HoodieTa if (tableSchema.isPresent()) { newWriteSchema = tableSchema.get(); } else { - LOG.warn("Could not fetch schema from table. Falling back to using target schema from schema provider"); + log.warn("Could not fetch schema from table. Falling back to using target schema from schema provider"); } } } @@ -1376,7 +1381,7 @@ private void registerAvroSchemas(HoodieSchema sourceSchema, HoodieSchema targetS schemas.add(targetSchema); } if (!schemas.isEmpty()) { - LOG.debug("Registering Schema: {}", schemas); + log.debug("Registering Schema: {}", schemas); // Use the underlying spark context in case the java context is changed during runtime hoodieSparkContext.getJavaSparkContext().sc().getConf() .registerAvroSchemas(JavaScalaConverters.convertJavaListToScalaList(schemas.stream().map(HoodieSchema::toAvroSchema).collect(Collectors.toList())).toList()); @@ -1397,7 +1402,7 @@ public void close() { formatAdapter.close(); } - LOG.info("Shutting down embedded timeline server"); + log.info("Shutting down embedded timeline server"); if (embeddedTimelineService.isPresent()) { embeddedTimelineService.get().stopForBasePath(cfg.targetBasePath); } @@ -1408,26 +1413,6 @@ public void close() { } - public HoodieStorage getStorage() { - return storage; - } - - public TypedProperties getProps() { - return props; - } - - public Config getCfg() { - return cfg; - } - - public Option getCommitsTimelineOpt() { - return commitsTimelineOpt; - } - - public HoodieIngestionMetrics getMetrics() { - return metrics; - } - /** * Schedule clustering. * Called from {@link HoodieStreamer} when async clustering is enabled. @@ -1454,25 +1439,16 @@ private Option getLatestCommittedInstant() { } } + @Getter class WriteClientWriteResult { + + @Setter private Map> partitionToReplacedFileIds = Collections.emptyMap(); private final JavaRDD writeStatusRDD; public WriteClientWriteResult(JavaRDD writeStatusRDD) { this.writeStatusRDD = writeStatusRDD; } - - public Map> getPartitionToReplacedFileIds() { - return partitionToReplacedFileIds; - } - - public void setPartitionToReplacedFileIds(Map> partitionToReplacedFileIds) { - this.partitionToReplacedFileIds = partitionToReplacedFileIds; - } - - public JavaRDD getWriteStatusRDD() { - return writeStatusRDD; - } } /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java index 560d9fd2172a8..323a2b95149de 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java @@ -40,8 +40,7 @@ import org.apache.hudi.utilities.config.KafkaSourceConfig; import org.apache.hudi.utilities.exception.HoodieStreamerException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; @@ -54,8 +53,8 @@ import static org.apache.hudi.common.util.ConfigUtils.removeConfigFromProps; import static org.apache.hudi.table.upgrade.UpgradeDowngrade.needsUpgradeOrDowngrade; +@Slf4j public class StreamerCheckpointUtils { - private static final Logger LOG = LoggerFactory.getLogger(StreamerCheckpointUtils.class); /** * The first phase of checkpoint resolution - read the checkpoint configs from 2 sources and resolve @@ -116,7 +115,7 @@ static void assertNoCheckpointOverrideDuringUpgradeForHoodieIncSource(HoodieTabl private static Option useCkpFromOverrideConfigIfAny( HoodieStreamer.Config streamerConfig, TypedProperties props, Option checkpoint) { - LOG.debug("Checkpoint from config: {}", streamerConfig.checkpoint); + log.debug("Checkpoint from config: {}", streamerConfig.checkpoint); if (!checkpoint.isPresent() && streamerConfig.checkpoint != null) { int writeTableVersion = ConfigUtils.getIntWithAltKeys(props, HoodieWriteConfig.WRITE_TABLE_VERSION); checkpoint = Option.of(buildCheckpointFromConfigOverride(streamerConfig.sourceClassName, writeTableVersion, streamerConfig.checkpoint)); @@ -157,7 +156,7 @@ static Option resolveCheckpointBetweenConfigAndPrevCommit(HoodieTime if (commitMetadataOption.isPresent()) { HoodieCommitMetadata commitMetadata = commitMetadataOption.get(); Checkpoint checkpointFromCommit = CheckpointUtils.getCheckpoint(commitMetadata); - LOG.debug("Checkpoint reset from metadata: {}", checkpointFromCommit.getCheckpointResetKey()); + log.debug("Checkpoint reset from metadata: {}", checkpointFromCommit.getCheckpointResetKey()); if (ignoreCkpCfgPrevailsOverCkpFromPrevCommit(streamerConfig, checkpointFromCommit)) { // we ignore any existing checkpoint and start ingesting afresh resumeCheckpoint = Option.empty(); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/TableExecutionContext.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/TableExecutionContext.java index f5f523a5c2135..e24269a6afa4b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/TableExecutionContext.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/TableExecutionContext.java @@ -21,66 +21,22 @@ import org.apache.hudi.common.config.TypedProperties; -import java.util.Objects; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; /** * Wrapper over TableConfig objects. * Useful for incrementally syncing multiple tables one by one via HoodieMultiTableStreamer.java class. */ +@Getter +@Setter +@EqualsAndHashCode public class TableExecutionContext { private TypedProperties properties; + @EqualsAndHashCode.Exclude private HoodieStreamer.Config config; private String database; private String tableName; - - public HoodieStreamer.Config getConfig() { - return config; - } - - public void setConfig(HoodieStreamer.Config config) { - this.config = config; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public TypedProperties getProperties() { - return properties; - } - - public void setProperties(TypedProperties properties) { - this.properties = properties; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - TableExecutionContext that = (TableExecutionContext) o; - return Objects.equals(properties, that.properties) && Objects.equals(database, that.database) && Objects.equals(tableName, that.tableName); - } - - @Override - public int hashCode() { - return Objects.hash(properties, database, tableName); - } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java index 9aebbfd4dc3f1..b9fbe74b6d029 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java @@ -68,6 +68,8 @@ import org.apache.hudi.testutils.HoodieSparkClientTestBase; import org.apache.hudi.testutils.SparkRDDValidationUtils; +import lombok.AccessLevel; +import lombok.Setter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -1345,6 +1347,7 @@ public void testRecordIndexMismatch(boolean ignoreFailed) throws IOException { } } + @Setter(AccessLevel.PACKAGE) class MockHoodieMetadataTableValidator extends HoodieMetadataTableValidator { private List metadataPartitionsToReturn; @@ -1355,18 +1358,6 @@ public MockHoodieMetadataTableValidator(JavaSparkContext jsc, Config cfg) { super(jsc, cfg); } - void setMetadataPartitionsToReturn(List metadataPartitionsToReturn) { - this.metadataPartitionsToReturn = metadataPartitionsToReturn; - } - - void setFsPartitionsToReturn(List fsPartitionsToReturn) { - this.fsPartitionsToReturn = fsPartitionsToReturn; - } - - void setPartitionCreationTime(Option partitionCreationTime) { - this.partitionCreationTime = partitionCreationTime; - } - @Override List getPartitionsFromFileSystem(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, HoodieTimeline completedTimeline) { return fsPartitionsToReturn; @@ -1449,6 +1440,7 @@ public void testRliValidationFalsePositiveCase() throws Exception { /** * Class to assist with testing a false positive case with RLI validation. */ + @Setter static class MockHoodieMetadataTableValidatorForRli extends HoodieMetadataTableValidator { private String destFilePath; @@ -1466,14 +1458,6 @@ JavaPairRDD> getRecordLocationsFromRLI(HoodieSparkE new File(destFilePath).renameTo(new File(originalFilePath)); return super.getRecordLocationsFromRLI(sparkEngineContext, basePath, latestCompletedCommit); } - - public void setDestFilePath(String destFilePath) { - this.destFilePath = destFilePath; - } - - public void setOriginalFilePath(String originalFilePath) { - this.originalFilePath = originalFilePath; - } } private String getTempLocation() { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieRepairTool.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieRepairTool.java index 0541b429a27f5..a2e9b0e15e01a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieRepairTool.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieRepairTool.java @@ -33,6 +33,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.testutils.providers.SparkProvider; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.HoodieSparkKryoRegistrar$; import org.apache.spark.SparkConf; @@ -45,8 +46,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.provider.Arguments; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.security.SecureRandom; @@ -67,8 +66,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j public class TestHoodieRepairTool extends HoodieCommonTestHarness implements SparkProvider { - private static final Logger LOG = LoggerFactory.getLogger(TestHoodieRepairTool.class); + // Instant time -> List> private static final Map>> BASE_FILE_INFO = new HashMap<>(); private static final Map>> LOG_FILE_INFO = new HashMap<>(); @@ -385,7 +385,7 @@ private List createDanglingDataFilesInFS(String parentPath) { storage.create(path, false); } } catch (IOException e) { - LOG.error("Error creating file: " + path); + log.error("Error creating file: {}", path); } return path.toString(); }) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java index 7ea5a39af3fa7..f38f8db4f48b8 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java @@ -57,6 +57,7 @@ import org.apache.hudi.utilities.testutils.KafkaTestUtils; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -67,8 +68,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -91,10 +90,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase { - private static final Logger LOG = LoggerFactory.getLogger(HoodieDeltaStreamerTestBase.class); - static final Random RANDOM = new Random(); static final String PROPS_FILENAME_TEST_SOURCE = "test-source.properties"; static final String PROPS_FILENAME_TEST_SOURCE1 = "test-source1.properties"; @@ -706,7 +704,7 @@ static HoodieDeltaStreamer.Config makeConfigForHudiIncrSrc(String srcBasePath, S static void assertAtleastNCompactionCommits(int minExpected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCommitAndReplaceTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCompactionCommits = timeline.countInstants(); assertTrue(minExpected <= numCompactionCommits, "Got=" + numCompactionCommits + ", exp >=" + minExpected); } @@ -714,7 +712,7 @@ static void assertAtleastNCompactionCommits(int minExpected, String tablePath) { static void assertAtleastNDeltaCommits(int minExpected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertTrue(minExpected <= numDeltaCommits, "Got=" + numDeltaCommits + ", exp >=" + minExpected); } @@ -722,7 +720,7 @@ static void assertAtleastNDeltaCommits(int minExpected, String tablePath) { static void assertAtleastNCompactionCommitsAfterCommit(int minExpected, String lastSuccessfulCommit, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCommitAndReplaceTimeline().findInstantsAfter(lastSuccessfulCommit).filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCompactionCommits = timeline.countInstants(); assertTrue(minExpected <= numCompactionCommits, "Got=" + numCompactionCommits + ", exp >=" + minExpected); } @@ -730,7 +728,7 @@ static void assertAtleastNCompactionCommitsAfterCommit(int minExpected, String l static void assertAtleastNDeltaCommitsAfterCommit(int minExpected, String lastSuccessfulCommit, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.reloadActiveTimeline().getDeltaCommitTimeline().findInstantsAfter(lastSuccessfulCommit).filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertTrue(minExpected <= numDeltaCommits, "Got=" + numDeltaCommits + ", exp >=" + minExpected); } @@ -757,9 +755,9 @@ static void waitTillCondition(Function condition, Future dsFut try { Thread.sleep(2000); ret = condition.apply(true); - LOG.info("Condition completed successfully"); + log.info("Condition completed successfully"); } catch (Throwable error) { - LOG.debug("Got error waiting for condition", error); + log.debug("Got error waiting for condition", error); ret = false; } } @@ -777,7 +775,7 @@ static void waitFor(BooleanSupplier booleanSupplier) { try { Thread.sleep(5); } catch (Throwable error) { - LOG.debug("Got error waiting for condition", error); + log.debug("Got error waiting for condition", error); } } } @@ -785,7 +783,7 @@ static void waitFor(BooleanSupplier booleanSupplier) { static void assertAtLeastNCommits(int minExpected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertTrue(minExpected <= numDeltaCommits, "Got=" + numDeltaCommits + ", exp >=" + minExpected); } @@ -793,7 +791,7 @@ static void assertAtLeastNCommits(int minExpected, String tablePath) { static void assertAtLeastNReplaceCommits(int minExpected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCompletedReplaceTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertTrue(minExpected <= numDeltaCommits, "Got=" + numDeltaCommits + ", exp >=" + minExpected); } @@ -801,7 +799,7 @@ static void assertAtLeastNReplaceCommits(int minExpected, String tablePath) { static void assertPendingIndexCommit(String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.reloadActiveTimeline().getAllCommitsTimeline().filterPendingIndexTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numIndexCommits = timeline.countInstants(); assertEquals(1, numIndexCommits, "Got=" + numIndexCommits + ", exp=1"); } @@ -809,7 +807,7 @@ static void assertPendingIndexCommit(String tablePath) { static void assertCompletedIndexCommit(String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.reloadActiveTimeline().getAllCommitsTimeline().filterCompletedIndexTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numIndexCommits = timeline.countInstants(); assertEquals(1, numIndexCommits, "Got=" + numIndexCommits + ", exp=1"); } @@ -817,7 +815,7 @@ static void assertCompletedIndexCommit(String tablePath) { static void assertNoReplaceCommits(String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCompletedReplaceTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertEquals(0, numDeltaCommits, "Got=" + numDeltaCommits + ", exp =" + 0); } @@ -825,7 +823,7 @@ static void assertNoReplaceCommits(String tablePath) { static void assertAtLeastNClusterRequests(int minExpected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().filterPendingClusteringTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numDeltaCommits = timeline.countInstants(); assertTrue(minExpected <= numDeltaCommits, "Got=" + numDeltaCommits + ", exp >=" + minExpected); } @@ -833,7 +831,7 @@ static void assertAtLeastNClusterRequests(int minExpected, String tablePath) { static void assertAtLeastNCommitsAfterRollback(int minExpectedRollback, int minExpectedCommits, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getRollbackTimeline().filterCompletedInstants(); - LOG.info("Rollback Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Rollback Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numRollbackCommits = timeline.countInstants(); assertTrue(minExpectedRollback <= numRollbackCommits, "Got=" + numRollbackCommits + ", exp >=" + minExpectedRollback); HoodieInstant firstRollback = timeline.getInstants().get(0); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/MockConfigurationHotUpdateStrategy.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/MockConfigurationHotUpdateStrategy.java index ac2a9a4c56044..9ef10f76ffc1e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/MockConfigurationHotUpdateStrategy.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/MockConfigurationHotUpdateStrategy.java @@ -23,8 +23,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.streamer.ConfigurationHotUpdateStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; @@ -33,8 +32,8 @@ /** * ConfigurationHotUpdateStrategy for test purpose. */ +@Slf4j public class MockConfigurationHotUpdateStrategy extends ConfigurationHotUpdateStrategy implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(MockConfigurationHotUpdateStrategy.class); public MockConfigurationHotUpdateStrategy(HoodieDeltaStreamer.Config cfg, TypedProperties properties) { super(cfg, properties); @@ -46,7 +45,7 @@ public Option updateProperties(TypedProperties currentProps) { long upsertShuffleParallelism = currentProps.getLong(UPSERT_PARALLELISM_VALUE.key()); TypedProperties newProps = TypedProperties.copy(currentProps); newProps.setProperty(UPSERT_PARALLELISM_VALUE.key(), String.valueOf(upsertShuffleParallelism + 5)); - LOG.info("update {} from [{}] to [{}]", UPSERT_PARALLELISM_VALUE.key(), upsertShuffleParallelism, upsertShuffleParallelism + 5); + log.info("update {} from [{}] to [{}]", UPSERT_PARALLELISM_VALUE.key(), upsertShuffleParallelism, upsertShuffleParallelism + 5); return Option.of(newProps); } return Option.empty(); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 2145d9c726209..96bb18044a45b 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -128,6 +128,7 @@ import org.apache.hudi.utilities.transform.Transformer; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -157,8 +158,6 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; @@ -207,10 +206,9 @@ /** * Basic tests against {@link HoodieDeltaStreamer}, by issuing bulk_inserts, upserts, inserts. Check counts at the end. */ +@Slf4j public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { - private static final Logger LOG = LoggerFactory.getLogger(TestHoodieDeltaStreamer.class); - private void addRecordMerger(HoodieRecordType type, List hoodieConfig) { if (type == HoodieRecordType.SPARK) { Map opts = new HashMap<>(); @@ -434,7 +432,7 @@ public void testPropsWithInvalidKeyGenerator() { deltaStreamer.sync(); }, "Should error out when setting the key generator class property to an invalid value"); // expected - LOG.warn("Expected error during getting the key generator", e); + log.warn("Expected error during getting the key generator", e); assertTrue(e.getMessage().contains("Unable to load class")); } @@ -484,7 +482,7 @@ public void testTableCreation() throws Exception { syncOnce(TestHelpers.makeConfig(basePath + "/not_a_table", WriteOperationType.BULK_INSERT)); }, "Should error out when pointed out at a dir thats not a table"); // expected - LOG.debug("Expected error during table creation", e); + log.debug("Expected error during table creation", e); } @ParameterizedTest @@ -561,7 +559,7 @@ public void testBulkInsertsAndUpsertsWithBootstrap(HoodieRecordType recordType) cfg.targetBasePath = newDatasetBasePath; syncOnce(cfg); Dataset res = sqlContext.read().format("org.apache.hudi").load(newDatasetBasePath); - LOG.info("Schema : {}", res.schema()); + log.info("Schema : {}", res.schema()); assertRecordCount(1950, newDatasetBasePath, sqlContext); res.registerTempTable("bootstrapped"); @@ -1568,7 +1566,7 @@ static void deltaStreamerTestRunner(HoodieDeltaStreamer ds, HoodieDeltaStreamer. try { ds.sync(); } catch (Exception ex) { - LOG.warn("DS continuous job failed, hence not proceeding with condition check for " + jobId); + log.warn("DS continuous job failed, hence not proceeding with condition check for {}", jobId); throw new RuntimeException(ex.getMessage(), ex); } }); @@ -1966,19 +1964,19 @@ public void testHoodieIndexer(HoodieRecordType recordType) throws Exception { buildIndexerConfig(tableBasePath, ds.getConfig().targetTableName, null, UtilHelpers.SCHEDULE, "COLUMN_STATS")); scheduleIndexInstantTime = scheduleIndexingJob.doSchedule(); } catch (Exception e) { - LOG.info("Schedule indexing failed", e); + log.info("Schedule indexing failed", e); return false; } if (scheduleIndexInstantTime.isPresent()) { TestHelpers.assertPendingIndexCommit(tableBasePath); - LOG.info("Schedule indexing success, now build index with instant time " + scheduleIndexInstantTime.get()); + log.info("Schedule indexing success, now build index with instant time {}", scheduleIndexInstantTime.get()); HoodieIndexer runIndexingJob = new HoodieIndexer(jsc, buildIndexerConfig(tableBasePath, ds.getConfig().targetTableName, scheduleIndexInstantTime.get(), UtilHelpers.EXECUTE, "COLUMN_STATS")); runIndexingJob.start(0); - LOG.info("Metadata indexing success"); + log.info("Metadata indexing success"); TestHelpers.assertCompletedIndexCommit(tableBasePath); } else { - LOG.warn("Metadata indexing failed"); + log.warn("Metadata indexing failed"); } return true; }); @@ -2006,7 +2004,7 @@ public void testHoodieIndexerExecutionAfterCommit() throws Exception { Arrays.asList(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=true", HoodieWriteConfig.MARKERS_TYPE.key() + "=DIRECT"))); scheduleIndexInstantTime = scheduleIndexingJob.doSchedule(); TestHelpers.assertPendingIndexCommit(tableBasePath); - LOG.info("Schedule indexing success, now build index with instant time " + scheduleIndexInstantTime.get()); + log.info("Schedule indexing success, now build index with instant time {}", scheduleIndexInstantTime.get()); // Wait for a pending commit before starting execution phase for the executor. This ensures that indexer waits for the commit to complete. TestHelpers.waitFor(() -> { HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(storage.getConf(), tableBasePath); @@ -2017,7 +2015,7 @@ public void testHoodieIndexerExecutionAfterCommit() throws Exception { buildIndexerConfig(tableBasePath, ds.getConfig().targetTableName, scheduleIndexInstantTime.get(), UtilHelpers.EXECUTE, "RECORD_INDEX", Arrays.asList(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=true", HoodieWriteConfig.MARKERS_TYPE.key() + "=DIRECT"))); runIndexingJob.start(0); - LOG.info("Metadata indexing success"); + log.info("Metadata indexing success"); TestHelpers.assertCompletedIndexCommit(tableBasePath); // Assert no pending commits before indexing instant HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(storage.getConf(), tableBasePath); @@ -2066,7 +2064,7 @@ public void testHoodieIndexerExecutionAfterClustering(HoodieRecordType recordTyp buildIndexerConfig(tableBasePath, ds.getConfig().targetTableName, null, UtilHelpers.SCHEDULE, "RECORD_INDEX")); scheduleIndexInstantTime = scheduleIndexingJob.doSchedule(); TestHelpers.assertPendingIndexCommit(tableBasePath); - LOG.info("Schedule indexing success, now build index with instant time " + scheduleIndexInstantTime.get()); + log.info("Schedule indexing success, now build index with instant time {}", scheduleIndexInstantTime.get()); // Wait for clustering instant to be scheduled before starting execution phase of the executor TestHelpers.waitFor(() -> { HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(storage, tableBasePath); @@ -2083,7 +2081,7 @@ public void testHoodieIndexerExecutionAfterClustering(HoodieRecordType recordTyp boolean res = JavaTestUtils.checkNestedExceptionContains(t, "Index catchup failed"); assertTrue(res, "Indexing catchup task should have timed out"); } - LOG.info("Metadata indexing timed out"); + log.info("Metadata indexing timed out"); } catch (Exception e) { fail("Indexing job should not have failed", e); } @@ -2113,19 +2111,19 @@ public void testHoodieAsyncClusteringJob(boolean shouldPassInClusteringInstantTi initialHoodieClusteringJob(tableBasePath, null, true, null); scheduleClusteringInstantTime = scheduleClusteringJob.doSchedule(); } catch (Exception e) { - LOG.warn("Schedule clustering failed", e); + log.warn("Schedule clustering failed", e); Assertions.fail("Schedule clustering failed", e); } if (scheduleClusteringInstantTime.isPresent()) { - LOG.info("Schedule clustering success, now cluster with instant time " + scheduleClusteringInstantTime.get()); + log.info("Schedule clustering success, now cluster with instant time {}", scheduleClusteringInstantTime.get()); HoodieClusteringJob.Config clusterClusteringConfig = buildHoodieClusteringUtilConfig(tableBasePath, shouldPassInClusteringInstantTime ? scheduleClusteringInstantTime.get() : null, false); HoodieClusteringJob clusterClusteringJob = new HoodieClusteringJob(jsc, clusterClusteringConfig); clusterClusteringJob.cluster(clusterClusteringConfig.retry); TestHelpers.assertAtLeastNReplaceCommits(1, tableBasePath); - LOG.info("Cluster success"); + log.info("Cluster success"); } else { - LOG.warn("Clustering execution failed"); + log.warn("Clustering execution failed"); Assertions.fail("Clustering execution failed"); } } else { @@ -2295,15 +2293,15 @@ public void testHoodieAsyncClusteringJobWithScheduleAndExecute(String runningMod try { int result = scheduleClusteringJob.cluster(0); if (result == 0) { - LOG.info("Cluster success"); + log.info("Cluster success"); } else { - LOG.warn("Cluster failed"); + log.warn("Cluster failed"); if (!runningMode.toLowerCase().equals(UtilHelpers.EXECUTE)) { return false; } } } catch (Exception e) { - LOG.warn("ScheduleAndExecute clustering failed", e); + log.warn("ScheduleAndExecute clustering failed", e); exception = e; if (!runningMode.equalsIgnoreCase(UtilHelpers.EXECUTE)) { return false; @@ -2463,13 +2461,13 @@ private void testBulkInsertRowWriterContinuousMode(boolean useSchemaProvider, Li try { int counter = 2; while (counter < 100) { // lets keep going. if the test times out, we will cancel the future within finally. So, safe to generate 100 batches. - LOG.info("Generating data for batch {}", counter); + log.info("Generating data for batch {}", counter); prepareParquetDFSFiles(100, PARQUET_SOURCE_ROOT, Integer.toString(counter) + ".parquet", false, null, null, makeDatesAmbiguous); counter++; Thread.sleep(2000); } } catch (Exception ex) { - LOG.warn("Input data generation failed", ex); + log.warn("Input data generation failed", ex); throw new RuntimeException(ex.getMessage(), ex); } }); @@ -2613,7 +2611,7 @@ public void testNullSchemaProvider() { Exception e = assertThrows(HoodieException.class, () -> { syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf())); }, "Should error out when schema provider is not provided"); - LOG.debug("Expected error during reading data from source ", e); + log.debug("Expected error during reading data from source ", e); assertTrue(e.getMessage().contains("Schema provider is required for this operation and for the source of interest. " + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. " + "Based on the schema provider class chosen, additional configs might be required. " @@ -2845,7 +2843,7 @@ public void testFilterDupes() throws Exception { // Ensure it is empty HoodieCommitMetadata commitMetadata = mClient.getActiveTimeline().readCommitMetadata(newLastFinished); - LOG.info("New Commit Metadata={}", commitMetadata); + log.info("New Commit Metadata={}", commitMetadata); assertTrue(commitMetadata.getPartitionToWriteStats().isEmpty()); // Try UPSERT with filterDupes true. Expect exception @@ -3361,7 +3359,7 @@ private void testDeltaStreamerRestartAfterMissingHoodieProps(boolean testInitFai try { fs.delete(entry.getPath()); } catch (IOException e) { - LOG.warn("Failed to delete " + entry.getPath().toString(), e); + log.warn("Failed to delete: {}", entry.getPath().toString(), e); } }); } @@ -3537,7 +3535,7 @@ public void testCsvDFSSourceNoHeaderWithoutSchemaProviderAndWithTransformer() th Exception e = assertThrows(AnalysisException.class, () -> { testCsvDFSSource(false, '\t', false, Collections.singletonList(TripsWithDistanceTransformer.class.getName())); }, "Should error out when doing the transformation."); - LOG.debug("Expected error during transformation", e); + log.debug("Expected error during transformation", e); // First message for Spark 3.4 and above, second message for Spark 3.3, third message for Spark 3.2 and below assertTrue( e.getMessage().contains("[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column or function parameter " @@ -3904,9 +3902,9 @@ public void testResumeCheckpointAfterChangingCOW2MOR() throws Exception { .build(); Properties hoodieProps = new Properties(); hoodieProps.load(fs.open(new Path(cfg.targetBasePath + "/.hoodie/hoodie.properties"))); - LOG.info("old props: {}", hoodieProps); + log.info("old props: {}", hoodieProps); hoodieProps.put("hoodie.table.type", HoodieTableType.MERGE_ON_READ.name()); - LOG.info("new props: {}", hoodieProps); + log.info("new props: {}", hoodieProps); StoragePath metaPathDir = new StoragePath(metaClient.getBasePath(), HoodieTableMetaClient.METAFOLDER_NAME); HoodieTableConfig.create(metaClient.getStorage(), metaPathDir, hoodieProps); @@ -3975,9 +3973,9 @@ public void testResumeCheckpointAfterChangingMOR2COW() throws Exception { .build(); Properties hoodieProps = new Properties(); hoodieProps.load(fs.open(new Path(cfg.targetBasePath + "/.hoodie/hoodie.properties"))); - LOG.info("old props: " + hoodieProps); + log.info("Old props: {}", hoodieProps); hoodieProps.put("hoodie.table.type", HoodieTableType.COPY_ON_WRITE.name()); - LOG.info("new props: " + hoodieProps); + log.info("New props: {}", hoodieProps); StoragePath metaPathDir = new StoragePath(metaClient.getBasePath(), ".hoodie"); HoodieTableConfig.create(metaClient.getStorage(), metaPathDir, hoodieProps); @@ -4271,13 +4269,13 @@ public DummyAvroPayload(GenericRecord gr, Comparable orderingVal) { /** * Return empty table. */ + @Slf4j public static class DropAllTransformer implements Transformer { - private static final Logger LOG = LoggerFactory.getLogger(DropAllTransformer.class); @Override public Dataset apply(JavaSparkContext jsc, SparkSession sparkSession, Dataset rowDataset, TypedProperties properties) { - LOG.info("DropAllTransformer called !!"); + log.info("DropAllTransformer called !!"); return sparkSession.createDataFrame(jsc.emptyRDD(), rowDataset.schema()); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java index 5e09ac9237941..bfe5dbda81904 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java @@ -38,13 +38,12 @@ import org.apache.hudi.utilities.sources.TestDataSource; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -68,10 +67,9 @@ import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.CHECKPOINT_KEY; import static org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner; +@Slf4j public class TestHoodieDeltaStreamerWithMultiWriter extends HoodieDeltaStreamerTestBase { - private static final Logger LOG = LoggerFactory.getLogger(TestHoodieDeltaStreamerWithMultiWriter.class); - String basePath; String propsFilePath; String tableBasePath; @@ -413,7 +411,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, deltaStreamerTestRunner(ingestionJob, cfgIngestionJob, conditionForRegularIngestion, jobId); } catch (Throwable ex) { continuousFailed.set(true); - LOG.error("Continuous job failed " + ex.getMessage()); + log.error("Continuous job failed {}", ex.getMessage()); throw new RuntimeException(ex); } }); @@ -424,7 +422,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, awaitCondition(new GetCommitsAfterInstant(tableBasePath, lastSuccessfulCommit)); backfillJob.sync(); } catch (Throwable ex) { - LOG.error("Backfilling job failed " + ex.getMessage()); + log.error("Backfilling job failed {}", ex.getMessage()); backfillFailed.set(true); throw new RuntimeException(ex); } @@ -443,7 +441,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, // expected ConcurrentModificationException since ingestion & backfill will have overlapping writes if (!continuousFailed.get()) { // if backfill job failed, shutdown the continuous job. - LOG.warn("Calling shutdown on ingestion job since the backfill job has failed for " + jobId); + log.warn("Calling shutdown on ingestion job since the backfill job has failed for {}", jobId); ingestionJob.shutdownGracefully(); } else { // both backfill and ingestion job cannot fail. @@ -452,14 +450,14 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, } else if (expectConflict && continuousFailed.get() && e.getCause().getMessage().contains("Ingestion service was shut down with exception")) { // incase of regular ingestion job failing, ConcurrentModificationException is not throw all the way. if (!backfillFailed.get()) { - LOG.warn("Calling shutdown on backfill job since the ingstion/continuous job has failed for " + jobId); + log.warn("Calling shutdown on backfill job since the ingstion/continuous job has failed for {}", jobId); backfillJob.shutdownGracefully(); } else { // both backfill and ingestion job cannot fail. throw new HoodieException("Both backfilling and ingestion job failed ", e); } } else { - LOG.error("Conflict happened, but not expected " + e.getCause().getMessage()); + log.error("Conflict happened, but not expected {}", e.getCause().getMessage()); throw e; } } finally { @@ -495,7 +493,7 @@ private static void awaitCondition(GetCommitsAfterInstant callback) throws Inter soFar += 500; } } - LOG.warn("Awaiting completed in " + (System.currentTimeMillis() - startTime)); + log.warn("Awaiting completed in {}", System.currentTimeMillis() - startTime); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieMultiTableDeltaStreamer.java index fc75e76275af4..b1e15b494c7e3 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieMultiTableDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieMultiTableDeltaStreamer.java @@ -34,9 +34,8 @@ import org.apache.hudi.utilities.streamer.TableExecutionContext; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; @@ -49,10 +48,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j public class TestHoodieMultiTableDeltaStreamer extends HoodieDeltaStreamerTestBase { - private static final Logger LOG = LoggerFactory.getLogger(TestHoodieMultiTableDeltaStreamer.class); - static class TestHelpers { static HoodieMultiTableDeltaStreamer.Config getConfig(String fileName, String configFolder, String sourceClassName, boolean enableHiveSync, boolean enableMetaSync, @@ -104,7 +102,7 @@ public void testInvalidHiveSyncProps() throws IOException { Exception e = assertThrows(HoodieException.class, () -> { new HoodieMultiTableDeltaStreamer(cfg, jsc); }, "Should fail when hive sync table not provided with enableHiveSync flag"); - LOG.debug("Expected error when creating table execution objects", e); + log.debug("Expected error when creating table execution objects", e); assertTrue(e.getMessage().contains("Meta sync table field not provided!")); } @@ -114,7 +112,7 @@ public void testInvalidPropsFilePath() throws IOException { Exception e = assertThrows(IllegalArgumentException.class, () -> { new HoodieMultiTableDeltaStreamer(cfg, jsc); }, "Should fail when invalid props file is provided"); - LOG.debug("Expected error when creating table execution objects", e); + log.debug("Expected error when creating table execution objects", e); assertTrue(e.getMessage().contains("Please provide valid common config file path!")); } @@ -124,7 +122,7 @@ public void testInvalidTableConfigFilePath() throws IOException { Exception e = assertThrows(IllegalArgumentException.class, () -> { new HoodieMultiTableDeltaStreamer(cfg, jsc); }, "Should fail when invalid table config props file path is provided"); - LOG.debug("Expected error when creating table execution objects", e); + log.debug("Expected error when creating table execution objects", e); assertTrue(e.getMessage().contains("Please provide valid table config file path!")); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHiveSchemaProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHiveSchemaProvider.java index bbd9eae152dc5..028e67c291b7a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHiveSchemaProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHiveSchemaProvider.java @@ -29,6 +29,7 @@ import org.apache.hudi.utilities.testutils.SparkClientFunctionalTestHarnessWithHiveSupport; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.junit.jupiter.api.Assertions; @@ -36,8 +37,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; @@ -46,9 +45,10 @@ /** * Basic tests against {@link HiveSchemaProvider}. */ +@Slf4j @Tag("functional") public class TestHiveSchemaProvider extends SparkClientFunctionalTestHarnessWithHiveSupport { - private static final Logger LOG = LoggerFactory.getLogger(TestHiveSchemaProvider.class); + private static final TypedProperties PROPS = new TypedProperties(); private static final String SOURCE_SCHEMA_TABLE_NAME = "schema_registry.source_schema_tab"; private static final String TARGET_SCHEMA_TABLE_NAME = "schema_registry.target_schema_tab"; @@ -75,7 +75,7 @@ public void testSourceSchema() throws Exception { assertNotNull(originalField); } } catch (HoodieException e) { - LOG.error("Failed to get source schema. ", e); + log.error("Failed to get source schema. ", e); throw e; } } @@ -97,7 +97,7 @@ public void testTargetSchema() throws Exception { assertNotNull(originalField); } } catch (HoodieException e) { - LOG.error("Failed to get source/target schema. ", e); + log.error("Failed to get source/target schema. ", e); throw e; } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java index a13a270b4c02d..fc8755ff878cf 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java @@ -26,11 +26,10 @@ import org.apache.hudi.utilities.schema.JdbcbasedSchemaProvider; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; @@ -43,10 +42,10 @@ import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_USER; import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j @Tag("functional") public class TestJdbcbasedSchemaProvider extends SparkClientFunctionalTestHarness { - private static final Logger LOG = LoggerFactory.getLogger(TestJdbcbasedSchemaProvider.class); private static final TypedProperties PROPS = new TypedProperties(); @BeforeAll @@ -67,7 +66,7 @@ public void testJdbcbasedSchemaProvider() throws Exception { HoodieSchema sourceSchema = UtilHelpers.createSchemaProvider(JdbcbasedSchemaProvider.class.getName(), PROPS, jsc()).getSourceHoodieSchema(); assertEquals(sourceSchema.toString().toUpperCase(), HoodieSchema.parse(UtilitiesTestBase.Helpers.readFile("streamer-config/source-jdbc.avsc")).toString().toUpperCase()); } catch (HoodieException e) { - LOG.error("Failed to get connection through jdbc. ", e); + log.error("Failed to get connection through jdbc. ", e); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/multitable/TestHoodieMultiTableServicesMain.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/multitable/TestHoodieMultiTableServicesMain.java index b93b70342b1cc..39e30475cbb02 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/multitable/TestHoodieMultiTableServicesMain.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/multitable/TestHoodieMultiTableServicesMain.java @@ -50,6 +50,7 @@ import org.apache.hudi.testutils.providers.SparkProvider; import org.apache.hudi.utilities.HoodieCompactor; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.spark.HoodieSparkKryoRegistrar$; import org.apache.spark.SparkConf; @@ -61,8 +62,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -79,10 +78,9 @@ * Tests for HoodieMultiTableServicesMain * @see HoodieMultiTableServicesMain */ +@Slf4j class TestHoodieMultiTableServicesMain extends HoodieCommonTestHarness implements SparkProvider { - private static final Logger LOG = LoggerFactory.getLogger(TestHoodieMultiTableServicesMain.class); - protected boolean initialized = false; private static SparkSession spark; @@ -155,10 +153,10 @@ public void testStreamRunAllServices() throws IOException, ExecutionException, I new Thread(() -> { try { Thread.sleep(10000); - LOG.info("Shutdown the table services"); + log.info("Shutdown the table services"); main.cancel(); } catch (InterruptedException e) { - LOG.warn("InterruptedException: ", e); + log.warn("InterruptedException: ", e); } }).start(); main.startServices(); @@ -193,10 +191,10 @@ public void testRunMultiTableServicesWithOneWrongPath() throws IOException { new Thread(() -> { try { Thread.sleep(10000); - LOG.info("Shutdown the table services"); + log.info("Shutdown the table services"); main.cancel(); } catch (InterruptedException e) { - LOG.warn("InterruptedException: ", e); + log.warn("InterruptedException: ", e); } }).start(); try { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/BaseTestKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/BaseTestKafkaSource.java index 243ad570ad75c..5c836a95c8f49 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/BaseTestKafkaSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/BaseTestKafkaSource.java @@ -33,6 +33,9 @@ import org.apache.hudi.utilities.streamer.SourceProfileSupplier; import org.apache.hudi.utilities.testutils.KafkaTestUtils; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -323,28 +326,15 @@ public void testKafkaSourceWithOffsetsFromSourceProfile() { verify(metrics, times(2)).updateStreamerSourceBytesToBeIngestedInSyncRound(Long.MAX_VALUE); } + @AllArgsConstructor + @Getter static class TestSourceProfile implements SourceProfile { private final long maxSourceBytes; private final int sourcePartitions; + @Getter(AccessLevel.NONE) private final long numEvents; - public TestSourceProfile(long maxSourceBytes, int sourcePartitions, long numEvents) { - this.maxSourceBytes = maxSourceBytes; - this.sourcePartitions = sourcePartitions; - this.numEvents = numEvents; - } - - @Override - public long getMaxSourceBytes() { - return maxSourceBytes; - } - - @Override - public int getSourcePartitions() { - return sourcePartitions; - } - @Override public Long getSourceSpecificContext() { return numEvents; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/S3EventsHoodieIncrSourceHarness.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/S3EventsHoodieIncrSourceHarness.java index 5e43daaa2a9c2..ecf34920218d7 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/S3EventsHoodieIncrSourceHarness.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/S3EventsHoodieIncrSourceHarness.java @@ -56,6 +56,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; @@ -279,27 +282,15 @@ protected void readAndAssert(IncrSourceHelper.MissingCheckpointStrategy missingC readAndAssert(missingCheckpointStrategy, checkpointToPull, sourceLimit, expectedCheckpoint, typedProperties); } + @AllArgsConstructor + @Getter static class TestSourceProfile implements SourceProfile { + private final long maxSourceBytes; private final int sourcePartitions; + @Getter(AccessLevel.NONE) private final long bytesPerPartition; - public TestSourceProfile(long maxSourceBytes, int sourcePartitions, long bytesPerPartition) { - this.maxSourceBytes = maxSourceBytes; - this.sourcePartitions = sourcePartitions; - this.bytesPerPartition = bytesPerPartition; - } - - @Override - public long getMaxSourceBytes() { - return maxSourceBytes; - } - - @Override - public int getSourcePartitions() { - return sourcePartitions; - } - @Override public Long getSourceSpecificContext() { return bytesPerPartition; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java index 923aba49f5e9b..1ec7842fe4e12 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java @@ -24,12 +24,11 @@ import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.testutils.sources.AbstractBaseTestSource; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; import java.util.stream.Collectors; @@ -37,9 +36,9 @@ /** * An implementation of {@link Source}, that emits test upserts. */ +@Slf4j public class TestDataSource extends AbstractBaseTestSource { - private static final Logger LOG = LoggerFactory.getLogger(TestDataSource.class); public static boolean returnEmptyBatch = false; public static Option recordInstantTime = Option.empty(); private static int counter = 0; @@ -56,14 +55,14 @@ protected InputBatch> readFromCheckpoint(Option Integer.parseInt(s.getCheckpointKey()) + 1).orElse(0); String instantTime = String.format("%05d", nextCommitNum); - LOG.info("Source Limit is set to " + sourceLimit); + log.info("Source Limit is set to {}", sourceLimit); // No new data. if (sourceLimit <= 0 || returnEmptyBatch) { - LOG.warn("Return no new data from Test Data source " + counter + ", source limit " + sourceLimit); + log.warn("Return no new data from Test Data source {}, source limit {}", counter, sourceLimit); return new InputBatch<>(Option.empty(), lastCheckpoint.orElse(null)); } else { - LOG.warn("Returning valid data from Test Data source " + counter + ", source limit " + sourceLimit); + log.warn("Returning valid data from Test Data source {}, source limit {}", counter, sourceLimit); } counter++; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java index 9e95d3e6d4f18..746760a86307d 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java @@ -57,6 +57,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; @@ -73,8 +74,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -96,6 +95,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +@Slf4j public class TestGcsEventsHoodieIncrSource extends SparkClientFunctionalTestHarness { private static final HoodieSchema GCS_METADATA_SCHEMA = SchemaTestUtil.getSchemaFromResource( @@ -122,8 +122,6 @@ public class TestGcsEventsHoodieIncrSource extends SparkClientFunctionalTestHarn private HoodieTableMetaClient metaClient; private JavaSparkContext jsc; - private static final Logger LOG = LoggerFactory.getLogger(TestGcsEventsHoodieIncrSource.class); - @BeforeEach public void setUp() throws IOException { metaClient = getHoodieMetaClient(storageConf(), basePath()); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java index 6011339a48a76..8e6bbd4f024a3 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java @@ -67,6 +67,10 @@ import org.apache.hudi.utilities.streamer.SourceProfile; import org.apache.hudi.utilities.streamer.SourceProfileSupplier; +import com.codahale.metrics.MetricRegistry; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.apache.spark.SparkConf; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -1092,47 +1096,28 @@ private static Stream getArgsForLogicalPlanSizeValidation() { ); } + @AllArgsConstructor + @Getter static class TestSourceProfile implements SourceProfile { private final long maxSourceBytes; private final int sourcePartitions; + @Getter(AccessLevel.NONE) private final int numInstantsPerFetch; - public TestSourceProfile(long maxSourceBytes, int sourcePartitions, int numInstantsPerFetch) { - this.maxSourceBytes = maxSourceBytes; - this.sourcePartitions = sourcePartitions; - this.numInstantsPerFetch = numInstantsPerFetch; - } - - @Override - public long getMaxSourceBytes() { - return maxSourceBytes; - } - - @Override - public int getSourcePartitions() { - return sourcePartitions; - } - @Override public Integer getSourceSpecificContext() { return numInstantsPerFetch; } } + @AllArgsConstructor + @Getter static class WriteResult { + private HoodieInstant instant; private List records; - WriteResult(HoodieInstant instant, List records) { - this.instant = instant; - this.records = records; - } - - public HoodieInstant getInstant() { - return instant; - } - public String getInstantTime() { return instant.requestedTime(); } @@ -1140,9 +1125,5 @@ public String getInstantTime() { public String getCompletionTime() { return instant.getCompletionTime(); } - - public List getRecords() { - return records; - } } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java index 7f4f264b69c23..4cd96d24301c9 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java @@ -23,9 +23,8 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.sql.Connection; @@ -39,10 +38,9 @@ /** * Helper class used in testing {@link org.apache.hudi.utilities.sources.JdbcSource}. */ +@Slf4j public class JdbcTestUtils { - private static final Logger LOG = LoggerFactory.getLogger(JdbcTestUtils.class); - public static final String JDBC_URL = "jdbc:h2:mem:test_mem"; public static final String JDBC_DRIVER = "org.h2.Driver"; public static final String JDBC_USER = "test"; @@ -99,7 +97,7 @@ public static List insert(String commitTime, int numRecords, Conne insertStatement.setDouble(9, Double.parseDouble(((GenericRecord) record.get("fare")).get("amount").toString())); insertStatement.addBatch(); } catch (SQLException e) { - LOG.warn(e.getMessage()); + log.warn(e.getMessage()); } }); insertStatement.executeBatch(); @@ -137,7 +135,7 @@ public static List update(String commitTime, List in updateStatement.setString(10, r.get("_row_key").toString()); updateStatement.addBatch(); } catch (SQLException e) { - LOG.warn(e.getMessage()); + log.warn(e.getMessage()); } }); updateStatement.executeBatch(); @@ -149,7 +147,7 @@ private static void execute(Connection connection, String query, String message) try (Statement statement = connection.createStatement()) { statement.executeUpdate(query); } catch (SQLException e) { - LOG.error(message); + log.error(message); } } @@ -159,7 +157,7 @@ private static void close(Statement statement) { statement.close(); } } catch (SQLException e) { - LOG.error("Error while closing statement. " + e.getMessage()); + log.error("Error while closing statement. {}", e.getMessage()); } } @@ -169,7 +167,7 @@ public static void close(Connection connection) { connection.close(); } } catch (SQLException e) { - LOG.error("Error while closing connection. " + e.getMessage()); + log.error("Error while closing connection. {}", e.getMessage()); } } @@ -179,7 +177,7 @@ public static int count(Connection connection, String tableName) { rs.next(); return rs.getInt(1); } catch (SQLException e) { - LOG.warn("Error while counting records. " + e.getMessage()); + log.warn("Error while counting records. {}", e.getMessage()); return 0; } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index 4b5269e0a6dd1..7f7b30b27e3e4 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -53,6 +53,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import com.fasterxml.jackson.dataformat.csv.CsvSchema.Builder; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericDatumWriter; @@ -79,8 +80,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.FileInputStream; @@ -110,8 +109,8 @@ * Abstract test that provides a dfs & spark contexts. * */ +@Slf4j public class UtilitiesTestBase { - private static final Logger LOG = LoggerFactory.getLogger(UtilitiesTestBase.class); @TempDir protected static java.nio.file.Path sharedTempDir; protected static FileSystem fs; @@ -256,7 +255,7 @@ public static void cleanUpUtilitiesTestServices() { } if (!failedReleases.isEmpty()) { - LOG.error("Exception happened during releasing: " + String.join(",", failedReleases)); + log.error("Exception happened during releasing: {}", String.join(",", failedReleases)); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java index e943545ddbc09..3036810d84756 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java @@ -30,13 +30,12 @@ import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.AvroSource; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Row; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -48,12 +47,11 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +@Slf4j public abstract class AbstractBaseTestSource extends AvroSource { public static String schemaStr = HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; - private static final Logger LOG = LoggerFactory.getLogger(AbstractBaseTestSource.class); - public static final int DEFAULT_PARTITION_NUM = 0; // Static instance, helps with reuse across a test. @@ -69,7 +67,7 @@ public static void initDataGen(TypedProperties props, int partition) { boolean useRocksForTestDataGenKeys = ConfigUtils.getBooleanWithAltKeys(props, SourceTestConfig.USE_ROCKSDB_FOR_TEST_DATAGEN_KEYS); String baseStoreDir = ConfigUtils.getStringWithAltKeys(props, SourceTestConfig.ROCKSDB_BASE_DIR_FOR_TEST_DATAGEN_KEYS, File.createTempFile("test_data_gen", ".keys").getParent()) + "/" + partition; - LOG.info("useRocksForTestDataGenKeys={}, BaseStoreDir={}", useRocksForTestDataGenKeys, baseStoreDir); + log.info("useRocksForTestDataGenKeys={}, BaseStoreDir={}", useRocksForTestDataGenKeys, baseStoreDir); dataGeneratorMap.put(partition, new HoodieTestDataGenerator(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, useRocksForTestDataGenKeys ? new RocksDBBasedMap<>(baseStoreDir) : new HashMap<>())); } catch (IOException e) { @@ -114,11 +112,11 @@ protected static Stream fetchNextBatch(TypedProperties props, int // generate `sourceLimit` number of upserts each time. int numExistingKeys = dataGenerator.getNumExistingKeys(schemaStr); - LOG.info("NumExistingKeys={}", numExistingKeys); + log.info("NumExistingKeys={}", numExistingKeys); int numUpdates = Math.min(numExistingKeys, sourceLimit / 2); int numInserts = sourceLimit - numUpdates; - LOG.info("Before adjustments => numInserts={}, numUpdates={}", numInserts, numUpdates); + log.info("Before adjustments => numInserts={}, numUpdates={}", numInserts, numUpdates); boolean reachedMax = false; if (numInserts + numExistingKeys > maxUniqueKeys) { @@ -135,16 +133,16 @@ protected static Stream fetchNextBatch(TypedProperties props, int Stream deleteStream = Stream.empty(); Stream updateStream; long memoryUsage1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - LOG.info("Before DataGen. Memory Usage={}, Total Memory={}, Free Memory={}", memoryUsage1, Runtime.getRuntime().totalMemory(), + log.info("Before DataGen. Memory Usage={}, Total Memory={}, Free Memory={}", memoryUsage1, Runtime.getRuntime().totalMemory(), Runtime.getRuntime().freeMemory()); if (!reachedMax && numUpdates >= 50) { - LOG.info("After adjustments => NumInserts={}, NumUpdates={}, NumDeletes=50, maxUniqueRecords={}", numInserts, (numUpdates - 50), maxUniqueKeys); + log.info("After adjustments => NumInserts={}, NumUpdates={}, NumDeletes=50, maxUniqueRecords={}", numInserts, (numUpdates - 50), maxUniqueKeys); // if we generate update followed by deletes -> some keys in update batch might be picked up for deletes. Hence generating delete batch followed by updates deleteStream = dataGenerator.generateUniqueDeleteRecordStream(instantTime, 50, false, schemaStr, 0L).map(AbstractBaseTestSource::toGenericRecord); updateStream = dataGenerator.generateUniqueUpdatesStream(instantTime, numUpdates - 50, schemaStr, 0L) .map(AbstractBaseTestSource::toGenericRecord); } else { - LOG.info("After adjustments => NumInserts={}, NumUpdates={}, maxUniqueRecords={}", numInserts, numUpdates, maxUniqueKeys); + log.info("After adjustments => NumInserts={}, NumUpdates={}, maxUniqueRecords={}", numInserts, numUpdates, maxUniqueKeys); updateStream = dataGenerator.generateUniqueUpdatesStream(instantTime, numUpdates, schemaStr, 0L) .map(AbstractBaseTestSource::toGenericRecord); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java index 224acd6016a9c..017507f2f9ece 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java @@ -26,12 +26,11 @@ import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.InputBatch; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -39,10 +38,9 @@ /** * A Test DataSource which scales test-data generation by using spark parallelism. */ +@Slf4j public class DistributedTestDataSource extends AbstractBaseTestSource { - private static final Logger LOG = LoggerFactory.getLogger(DistributedTestDataSource.class); - private final int numTestSourcePartitions; public DistributedTestDataSource(TypedProperties props, JavaSparkContext sparkContext, SparkSession sparkSession, @@ -55,7 +53,7 @@ public DistributedTestDataSource(TypedProperties props, JavaSparkContext sparkCo protected InputBatch> readFromCheckpoint(Option lastCheckpoint, long sourceLimit) { int nextCommitNum = lastCheckpoint.map(s -> Integer.parseInt(s.getCheckpointKey()) + 1).orElse(0); String instantTime = String.format("%05d", nextCommitNum); - LOG.info("Source Limit is set to {}", sourceLimit); + log.info("Source Limit is set to {}", sourceLimit); // No new data. if (sourceLimit <= 0) { @@ -73,7 +71,7 @@ protected InputBatch> readFromCheckpoint(Option avroRDD = sparkContext.parallelize(IntStream.range(0, numTestSourcePartitions).boxed().collect(Collectors.toList()), numTestSourcePartitions).mapPartitionsWithIndex((p, idx) -> { - LOG.info("Initializing source with newProps={}", newProps); + log.info("Initializing source with newProps={}", newProps); if (!dataGeneratorMap.containsKey(p)) { initDataGen(newProps, p); } From 7b457efbae56e9ab8bca7bab4a09958003262049 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 3 Jun 2026 19:43:00 +0800 Subject: [PATCH 045/255] refactor: Add Lombok annotations to hudi-common module (part 3) (#17825) * refactor: Add Lombok annotations to hudi-common module (part 3) * Address comments * Address bot comments (cherry picked from commit 438bbdfcaef43abc5d0d21f931a47c5af05a58a8) --- .../action/clean/CleanActionExecutor.java | 20 +- .../utils/TestMetadataConversionUtils.java | 14 +- .../functional/TestExternalPathHandling.java | 10 +- .../hudi/io/TestHoodieTimelineArchiver.java | 15 +- .../org/apache/hudi/table/TestCleaner.java | 24 +- .../hudi/testutils/HoodieCleanerTestBase.java | 31 +-- .../HoodieSparkClientTestHarness.java | 14 +- hudi-common/pom.xml | 1 + .../apache/hudi/common/HoodieCleanStat.java | 210 ++---------------- .../common/HoodiePendingRollbackInfo.java | 18 +- .../hudi/common/HoodieRollbackStat.java | 34 +-- .../bloom/InternalDynamicBloomFilter.java | 9 +- .../hudi/common/bloom/InternalFilter.java | 7 +- .../org/apache/hudi/common/bloom/Key.java | 8 +- .../index/hfile/HFileBootstrapIndex.java | 15 +- .../hfile/HFileBootstrapIndexReader.java | 22 +- .../hfile/HFileBootstrapIndexWriter.java | 17 +- .../hudi/common/config/ConfigGroups.java | 20 +- .../hudi/common/config/ConfigProperty.java | 66 ++---- .../hudi/common/config/HoodieConfig.java | 12 +- .../DirectMarkerBasedDetectionStrategy.java | 21 +- .../hudi/common/data/HoodieBaseListData.java | 16 +- .../common/engine/HoodieEngineContext.java | 18 +- .../common/engine/HoodieReaderContext.java | 128 +++-------- .../hudi/common/engine/RecordContext.java | 12 +- .../org/apache/hudi/common/fs/FSUtils.java | 9 +- .../common/fs/FailSafeConsistencyGuard.java | 16 +- .../common/fs/OptimisticConsistencyGuard.java | 10 +- .../heartbeat/HoodieHeartbeatUtils.java | 7 +- .../hudi/common/schema/HoodieSchema.java | 38 +--- .../schema/HoodieSchemaCompatibility.java | 8 +- .../hudi/common/schema/HoodieSchemaField.java | 14 +- .../table/view/TestIncrementalFSViewSync.java | 13 +- .../common/testutils/HoodieTestTable.java | 24 +- ...HoodieFileGroupReaderBasedFileFormat.scala | 2 +- 35 files changed, 272 insertions(+), 631 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java index 63a6e7d8f994d..2a636221ae60f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java @@ -171,20 +171,18 @@ List clean(HoodieEngineContext context, HoodieCleanerPlan clean ? partitionCleanStatsMap.get(partitionPath) : new PartitionCleanStat(partitionPath); HoodieActionInstant actionInstant = cleanerPlan.getEarliestInstantToRetain(); - return HoodieCleanStat.newBuilder().withPolicy(config.getCleanerPolicy()).withPartitionPath(partitionPath) - .withEarliestCommitRetained(Option.ofNullable( - actionInstant != null - ? instantGenerator.createNewInstant(HoodieInstant.State.valueOf(actionInstant.getState()), - actionInstant.getAction(), actionInstant.getTimestamp()) - : null)) + return HoodieCleanStat.builder() + .withPolicy(config.getCleanerPolicy()) + .withPartitionPath(partitionPath) + .withEarliestCommitToRetain(actionInstant != null ? actionInstant.getTimestamp() : "") .withLastCompletedCommitTimestamp(cleanerPlan.getLastCompletedCommitTimestamp()) - .withDeletePathPattern(partitionCleanStat.deletePathPatterns()) - .withSuccessfulDeletes(partitionCleanStat.successDeleteFiles()) - .withFailedDeletes(partitionCleanStat.failedDeleteFiles()) + .withDeletePathPatterns(partitionCleanStat.deletePathPatterns()) + .withSuccessDeleteFiles(partitionCleanStat.successDeleteFiles()) + .withFailedDeleteFiles(partitionCleanStat.failedDeleteFiles()) .withDeleteBootstrapBasePathPatterns(partitionCleanStat.getDeleteBootstrapBasePathPatterns()) - .withSuccessfulDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles()) + .withSuccessDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles()) .withFailedDeleteBootstrapBaseFiles(partitionCleanStat.getFailedDeleteBootstrapBaseFiles()) - .isPartitionDeleted(partitionsToBeDeleted.contains(partitionPath)) + .withPartitionDeleted(partitionsToBeDeleted.contains(partitionPath)) .build(); }).collect(Collectors.toList()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java index 01af9d3fc3ffd..c0374b25e1f10 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java @@ -405,14 +405,12 @@ private void createCleanMetadata(String instantTime) throws IOException { HoodieCleanFileInfo fileInfo = new HoodieCleanFileInfo("file1", false); HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new HoodieActionInstant("", "", ""), "", "", new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, Collections.singletonMap("key", Collections.singletonList(fileInfo)), new ArrayList<>(), Collections.EMPTY_MAP); - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java index ce94187aacbcf..5d7c4aac24165 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java @@ -339,8 +339,14 @@ private WriteStatus createWriteStatus(String commitTime, String partitionPath, S } private HoodieCleanStat createCleanStat(String partitionPath, List deletePaths, String earliestCommitToRetain, String lastCompletedCommitTimestamp) { - return new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, partitionPath, deletePaths, deletePaths, Collections.emptyList(), - earliestCommitToRetain, lastCompletedCommitTimestamp); + return HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(partitionPath) + .withDeletePathPatterns(deletePaths) + .withSuccessDeleteFiles(deletePaths) + .withEarliestCommitToRetain(earliestCommitToRetain) + .withLastCompletedCommitTimestamp(lastCompletedCommitTimestamp) + .build(); } private HoodieCleanerPlan cleanerPlan(HoodieActionInstant earliestInstantToRetain, String latestCommit, Map> filePathsToBeDeletedPerPartition) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java index 60db8b8a13b75..5725c571c8dc3 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java @@ -2356,15 +2356,12 @@ private HoodieWriteConfig buildECTRTestConfig(int minCommits, int maxCommits, bo private void addCleanCommitWithECTR(HoodieTestTable testTable, String cleanInstant, String ectr, String lastCompleted) throws Exception { List cleanStatsList = new ArrayList<>(); - cleanStatsList.add(new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_COMMITS, - "p1", - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - ectr, - lastCompleted - )); + cleanStatsList.add(HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath("p1") + .withEarliestCommitToRetain(ectr) + .withLastCompletedCommitTimestamp(lastCompleted) + .build()); HoodieCleanMetadata cleanMetadata = CleanerUtils.convertCleanMetadata(cleanInstant, Option.of(0L), cleanStatsList, Collections.emptyMap()); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java index f57ee82385f6c..dde8ac5729b39 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -724,18 +724,30 @@ public void testCleanMetadataUpgradeDowngrade() { List failedDeleteFiles1 = Collections.singletonList(filePath2); // create partition1 clean stat. - HoodieCleanStat cleanStat1 = new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - partition1, deletePathPatterns1, successDeleteFiles1, - failedDeleteFiles1, instantTime, ""); + HoodieCleanStat cleanStat1 = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(partition1) + .withDeletePathPatterns(deletePathPatterns1) + .withSuccessDeleteFiles(successDeleteFiles1) + .withFailedDeleteFiles(failedDeleteFiles1) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); List deletePathPatterns2 = new ArrayList<>(); List successDeleteFiles2 = new ArrayList<>(); List failedDeleteFiles2 = new ArrayList<>(); // create partition2 empty clean stat. - HoodieCleanStat cleanStat2 = new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, - partition2, deletePathPatterns2, successDeleteFiles2, - failedDeleteFiles2, instantTime, ""); + HoodieCleanStat cleanStat2 = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(partition2) + .withDeletePathPatterns(deletePathPatterns2) + .withSuccessDeleteFiles(successDeleteFiles2) + .withFailedDeleteFiles(failedDeleteFiles2) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); // map with absolute file path. Map oldExpected = new HashMap<>(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java index 66f2f595ef4da..5563f39c811b8 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java @@ -164,24 +164,27 @@ protected List runCleaner( } Map cleanStatMap = cleanMetadata1.getPartitionMetadata().values().stream() - .map(x -> new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()) - .withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles()) - .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns()) - .withEarliestCommitRetained(Option.ofNullable(cleanMetadata1.getEarliestCommitToRetain() != null - ? INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "000") - : null)) + .map(x -> HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())) + .withPartitionPath(x.getPartitionPath()) + .withDeletePathPatterns(x.getDeletePathPatterns()) + .withSuccessDeleteFiles(x.getSuccessDeleteFiles()) + .withFailedDeleteFiles(x.getFailedDeleteFiles()) + .withEarliestCommitToRetain(cleanMetadata1.getEarliestCommitToRetain() != null ? "000" : "") .build()) .collect(Collectors.toMap(HoodieCleanStat::getPartitionPath, x -> x)); cleanMetadata1.getBootstrapPartitionMetadata().values().forEach(x -> { - HoodieCleanStat s = cleanStatMap.get(x.getPartitionPath()); - cleanStatMap.put(x.getPartitionPath(), new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()) - .withFailedDeletes(s.getFailedDeleteFiles()).withSuccessfulDeletes(s.getSuccessDeleteFiles()) - .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(s.getDeletePathPatterns()) - .withEarliestCommitRetained(Option.ofNullable(s.getEarliestCommitToRetain()) - .map(y -> INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, y))) - .withSuccessfulDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles()) + cleanStatMap.compute(x.getPartitionPath(), (k, s) -> HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())) + .withPartitionPath(x.getPartitionPath()) + .withDeletePathPatterns(s.getDeletePathPatterns()) + .withSuccessDeleteFiles(s.getSuccessDeleteFiles()) + .withFailedDeleteFiles(s.getFailedDeleteFiles()) + .withEarliestCommitToRetain(s.getEarliestCommitToRetain() != null ? s.getEarliestCommitToRetain() : "") + .withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()) + .withSuccessDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles()) .withFailedDeleteBootstrapBaseFiles(x.getFailedDeleteFiles()) - .withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()).build()); + .build()); }); return new ArrayList<>(cleanStatMap.values()); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java index 97bd0e1175d76..296690556327e 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java @@ -676,14 +676,12 @@ public HoodieInstant createCleanMetadata(String instantTime, boolean inflightOnl if (inflightOnly) { HoodieTestTable.of(metaClient).addInflightClean(instantTime, cleanerPlan); } else { - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata, isEmptyForAll, isEmptyCompleted); } diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index f0416d3cd8179..7bb76c74ad5d9 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -128,6 +128,7 @@ org.projectlombok lombok + provided diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java index 498d430b5c3a6..0106dfc96200c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java @@ -19,9 +19,10 @@ package org.apache.hudi.common; import org.apache.hudi.common.model.HoodieCleaningPolicy; -import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.CollectionUtils; -import org.apache.hudi.common.util.Option; + +import lombok.Builder; +import lombok.Value; import java.io.Serializable; import java.util.List; @@ -29,197 +30,34 @@ /** * Collects stats about a single partition clean operation. */ +@Builder(setterPrefix = "with") +@Value public class HoodieCleanStat implements Serializable { // Policy used - private final HoodieCleaningPolicy policy; + HoodieCleaningPolicy policy; // Partition path cleaned - private final String partitionPath; + String partitionPath; // The patterns that were generated for the delete operation - private final List deletePathPatterns; - private final List successDeleteFiles; - // Files that could not be deleted - private final List failedDeleteFiles; - // Bootstrap Base Path patterns that were generated for the delete operation - private final List deleteBootstrapBasePathPatterns; - private final List successDeleteBootstrapBaseFiles; + @Builder.Default + List deletePathPatterns = CollectionUtils.createImmutableList(); + @Builder.Default + List successDeleteFiles = CollectionUtils.createImmutableList(); // Files that could not be deleted - private final List failedDeleteBootstrapBaseFiles; + @Builder.Default + List failedDeleteFiles = CollectionUtils.createImmutableList(); // Earliest commit that was retained in this clean - private final String earliestCommitToRetain; + String earliestCommitToRetain; // Last completed commit timestamp before clean - private final String lastCompletedCommitTimestamp; + String lastCompletedCommitTimestamp; + // Bootstrap Base Path patterns that were generated for the delete operation + @Builder.Default + List deleteBootstrapBasePathPatterns = CollectionUtils.createImmutableList(); + @Builder.Default + List successDeleteBootstrapBaseFiles = CollectionUtils.createImmutableList(); + // Files that could not be deleted + @Builder.Default + List failedDeleteBootstrapBaseFiles = CollectionUtils.createImmutableList(); // set to true if partition is deleted - private final boolean isPartitionDeleted; - - public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath, List deletePathPatterns, - List successDeleteFiles, List failedDeleteFiles, String earliestCommitToRetain,String lastCompletedCommitTimestamp) { - this(policy, partitionPath, deletePathPatterns, successDeleteFiles, failedDeleteFiles, earliestCommitToRetain, - lastCompletedCommitTimestamp, CollectionUtils.createImmutableList(), CollectionUtils.createImmutableList(), - CollectionUtils.createImmutableList(), false); - } - - public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath, List deletePathPatterns, - List successDeleteFiles, List failedDeleteFiles, - String earliestCommitToRetain,String lastCompletedCommitTimestamp, - List deleteBootstrapBasePathPatterns, - List successDeleteBootstrapBaseFiles, - List failedDeleteBootstrapBaseFiles, - boolean isPartitionDeleted) { - this.policy = policy; - this.partitionPath = partitionPath; - this.deletePathPatterns = deletePathPatterns; - this.successDeleteFiles = successDeleteFiles; - this.failedDeleteFiles = failedDeleteFiles; - this.earliestCommitToRetain = earliestCommitToRetain; - this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp; - this.deleteBootstrapBasePathPatterns = deleteBootstrapBasePathPatterns; - this.successDeleteBootstrapBaseFiles = successDeleteBootstrapBaseFiles; - this.failedDeleteBootstrapBaseFiles = failedDeleteBootstrapBaseFiles; - this.isPartitionDeleted = isPartitionDeleted; - } - - public HoodieCleaningPolicy getPolicy() { - return policy; - } - - public String getPartitionPath() { - return partitionPath; - } - - public List getDeletePathPatterns() { - return deletePathPatterns; - } - - public List getSuccessDeleteFiles() { - return successDeleteFiles; - } - - public List getFailedDeleteFiles() { - return failedDeleteFiles; - } - - public List getDeleteBootstrapBasePathPatterns() { - return deleteBootstrapBasePathPatterns; - } - - public List getSuccessDeleteBootstrapBaseFiles() { - return successDeleteBootstrapBaseFiles; - } - - public List getFailedDeleteBootstrapBaseFiles() { - return failedDeleteBootstrapBaseFiles; - } - - public String getEarliestCommitToRetain() { - return earliestCommitToRetain; - } - - public String getLastCompletedCommitTimestamp() { - return lastCompletedCommitTimestamp; - } - - public boolean isPartitionDeleted() { - return isPartitionDeleted; - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** - * A builder used to build {@link HoodieCleanStat}. - */ - public static class Builder { - - private HoodieCleaningPolicy policy; - private List deletePathPatterns; - private List successDeleteFiles; - private List failedDeleteFiles; - private String partitionPath; - private String earliestCommitToRetain; - private String lastCompletedCommitTimestamp; - private List deleteBootstrapBasePathPatterns; - private List successDeleteBootstrapBaseFiles; - private List failedDeleteBootstrapBaseFiles; - private boolean isPartitionDeleted; - - public Builder withPolicy(HoodieCleaningPolicy policy) { - this.policy = policy; - return this; - } - - public Builder withDeletePathPattern(List deletePathPatterns) { - this.deletePathPatterns = deletePathPatterns; - return this; - } - - public Builder withSuccessfulDeletes(List successDeleteFiles) { - this.successDeleteFiles = successDeleteFiles; - return this; - } - - public Builder withFailedDeletes(List failedDeleteFiles) { - this.failedDeleteFiles = failedDeleteFiles; - return this; - } - - public Builder withDeleteBootstrapBasePathPatterns(List deletePathPatterns) { - this.deleteBootstrapBasePathPatterns = deletePathPatterns; - return this; - } - - public Builder withSuccessfulDeleteBootstrapBaseFiles(List successDeleteFiles) { - this.successDeleteBootstrapBaseFiles = successDeleteFiles; - return this; - } - - public Builder withFailedDeleteBootstrapBaseFiles(List failedDeleteFiles) { - this.failedDeleteBootstrapBaseFiles = failedDeleteFiles; - return this; - } - - public Builder withPartitionPath(String partitionPath) { - this.partitionPath = partitionPath; - return this; - } - - public Builder withEarliestCommitRetained(Option earliestCommitToRetain) { - this.earliestCommitToRetain = - (earliestCommitToRetain.isPresent()) ? earliestCommitToRetain.get().requestedTime() : ""; - return this; - } - - public Builder withLastCompletedCommitTimestamp(String lastCompletedCommitTimestamp) { - this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp; - return this; - } - - public Builder isPartitionDeleted(boolean isPartitionDeleted) { - this.isPartitionDeleted = isPartitionDeleted; - return this; - } - - public HoodieCleanStat build() { - return new HoodieCleanStat(policy, partitionPath, deletePathPatterns, successDeleteFiles, failedDeleteFiles, - earliestCommitToRetain, lastCompletedCommitTimestamp, deleteBootstrapBasePathPatterns, - successDeleteBootstrapBaseFiles, failedDeleteBootstrapBaseFiles, isPartitionDeleted); - } - } - - @Override - public String toString() { - return "HoodieCleanStat{" - + "policy=" + policy - + ", partitionPath='" + partitionPath + '\'' - + ", deletePathPatterns=" + deletePathPatterns - + ", successDeleteFiles=" + successDeleteFiles - + ", failedDeleteFiles=" + failedDeleteFiles - + ", earliestCommitToRetain='" + earliestCommitToRetain - + ", deleteBootstrapBasePathPatterns=" + deleteBootstrapBasePathPatterns - + ", successDeleteBootstrapBaseFiles=" + successDeleteBootstrapBaseFiles - + ", failedDeleteBootstrapBaseFiles=" + failedDeleteBootstrapBaseFiles - + ", isPartitionDeleted=" + isPartitionDeleted + '\'' - + '}'; - } + boolean partitionDeleted; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java index c53babf350102..44e61f651b8ce 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java @@ -21,24 +21,16 @@ import org.apache.hudi.avro.model.HoodieRollbackPlan; import org.apache.hudi.common.table.timeline.HoodieInstant; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * Holds rollback instant and rollback plan for a pending rollback. */ +@AllArgsConstructor +@Getter public class HoodiePendingRollbackInfo { private final HoodieInstant rollbackInstant; private final HoodieRollbackPlan rollbackPlan; - - public HoodiePendingRollbackInfo(HoodieInstant rollbackInstant, HoodieRollbackPlan rollbackPlan) { - this.rollbackInstant = rollbackInstant; - this.rollbackPlan = rollbackPlan; - } - - public HoodieInstant getRollbackInstant() { - return rollbackInstant; - } - - public HoodieRollbackPlan getRollbackPlan() { - return rollbackPlan; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java index 59308a43325c2..c9be79687854b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java @@ -20,6 +20,9 @@ import org.apache.hudi.storage.StoragePathInfo; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.io.Serializable; import java.util.Collections; import java.util.List; @@ -29,6 +32,8 @@ /** * Collects stats about a single partition clean operation. */ +@AllArgsConstructor +@Getter public class HoodieRollbackStat implements Serializable { // Partition path @@ -41,35 +46,6 @@ public class HoodieRollbackStat implements Serializable { private final Map logFilesFromFailedCommit; - public HoodieRollbackStat(String partitionPath, List successDeleteFiles, List failedDeleteFiles, - Map commandBlocksCount, Map logFilesFromFailedCommit) { - this.partitionPath = partitionPath; - this.successDeleteFiles = successDeleteFiles; - this.failedDeleteFiles = failedDeleteFiles; - this.commandBlocksCount = commandBlocksCount; - this.logFilesFromFailedCommit = logFilesFromFailedCommit; - } - - public Map getCommandBlocksCount() { - return commandBlocksCount; - } - - public String getPartitionPath() { - return partitionPath; - } - - public List getSuccessDeleteFiles() { - return successDeleteFiles; - } - - public List getFailedDeleteFiles() { - return failedDeleteFiles; - } - - public Map getLogFilesFromFailedCommit() { - return logFilesFromFailedCommit; - } - public static HoodieRollbackStat.Builder newBuilder() { return new Builder(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java index bf35aeaee61de..fa14e2976f43c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.bloom; +import lombok.NoArgsConstructor; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -27,6 +29,7 @@ * with bounds on maximum number of entries. Once the max entries is reached, false positive guarantees are not * honored. */ +@NoArgsConstructor class InternalDynamicBloomFilter extends InternalFilter { /** @@ -47,12 +50,6 @@ class InternalDynamicBloomFilter extends InternalFilter { */ private InternalBloomFilter[] matrix; - /** - * Zero-args constructor for the serialization. - */ - public InternalDynamicBloomFilter() { - } - /** * Constructor. *

    diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java index e23255bb4b616..c5b45e7877a8d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java @@ -20,6 +20,9 @@ import org.apache.hudi.common.util.hash.Hash; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -42,6 +45,7 @@ * @see Key The general behavior of a key * @see HashFunction A hash function */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) abstract class InternalFilter { private static final int VERSION = -1; // negative to accommodate for old format /** @@ -64,9 +68,6 @@ abstract class InternalFilter { */ protected int hashType; - protected InternalFilter() { - } - /** * Constructor. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java index 013c0f08ea46f..ed8096edea346 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java @@ -20,6 +20,7 @@ package org.apache.hudi.common.bloom; import lombok.Getter; +import lombok.NoArgsConstructor; import java.io.DataInput; import java.io.DataOutput; @@ -33,6 +34,7 @@ * @see InternalBloomFilter The general behavior of a bloom filter and how the key is used. */ @Getter +@NoArgsConstructor public class Key implements Comparable { /** * Byte value of key @@ -47,12 +49,6 @@ public class Key implements Comparable { */ double weight; - /** - * default constructor - use with readFields - */ - public Key() { - } - /** * Constructor. *

    diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java index fd342288cf011..79feafbc47847 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java @@ -30,8 +30,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; @@ -49,12 +49,11 @@ * on these index files to manage multiple file-groups. */ +@Slf4j public class HFileBootstrapIndex extends BootstrapIndex { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndex.class); - public static final String BOOTSTRAP_INDEX_FILE_ID = "00000000-0000-0000-0000-000000000000-0"; private static final String PARTITION_KEY_PREFIX = "part"; @@ -68,6 +67,7 @@ public class HFileBootstrapIndex extends BootstrapIndex { public static final String INDEX_INFO_KEY_STRING = "INDEX_INFO"; public static final byte[] INDEX_INFO_KEY = getUTF8Bytes(INDEX_INFO_KEY_STRING); + @Getter private final boolean isPresent; public HFileBootstrapIndex(HoodieTableMetaClient metaClient) { @@ -152,7 +152,7 @@ public void dropIndex() { StoragePath[] indexPaths = new StoragePath[] {partitionIndexPath(metaClient), fileIdIndexPath(metaClient)}; for (StoragePath indexPath : indexPaths) { if (metaClient.getStorage().exists(indexPath)) { - LOG.info("Dropping bootstrap index. Deleting file: {}", indexPath); + log.info("Dropping bootstrap index. Deleting file: {}", indexPath); metaClient.getStorage().deleteDirectory(indexPath); } } @@ -160,9 +160,4 @@ public void dropIndex() { throw new HoodieIOException(ioe.getMessage(), ioe); } } - - @Override - public boolean isPresent() { - return isPresent; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java index 53debdb71ba06..320b365e35190 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java @@ -38,8 +38,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayList; @@ -59,10 +59,11 @@ /** * HFile Based Index Reader. */ +@Slf4j public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndexReader.class); // Base Path of external files. + @Getter private final String bootstrapBasePath; // Well Known Paths for indices private final String indexByPartitionPath; @@ -83,7 +84,7 @@ public HFileBootstrapIndexReader(HoodieTableMetaClient metaClient) { this.indexByFileIdPath = indexByFilePath.toString(); initIndexInfo(); this.bootstrapBasePath = bootstrapIndexInfo.getBootstrapBasePath(); - LOG.info("Loaded HFileBasedBootstrapIndex with source base path :" + bootstrapBasePath); + log.info("Loaded HFileBasedBootstrapIndex with source base path :{}", bootstrapBasePath); } /** @@ -93,7 +94,7 @@ public HFileBootstrapIndexReader(HoodieTableMetaClient metaClient) { * @param storage {@link HoodieStorage} instance. */ private static HFileReader createReader(String hFilePath, HoodieStorage storage) throws IOException { - LOG.info("Opening HFile for reading :" + hFilePath); + log.info("Opening HFile for reading :{}", hFilePath); StoragePath path = new StoragePath(hFilePath); long fileSize = storage.getPathInfo(path).getLength(); SeekableDataInputStream stream = storage.openSeekable(path, false); @@ -118,7 +119,7 @@ private HoodieBootstrapIndexInfo fetchBootstrapIndexInfo() throws IOException { private synchronized HFileReader partitionIndexReader() throws IOException { if (indexByPartitionReader == null) { - LOG.info("Opening partition index :" + indexByPartitionPath); + log.info("Opening partition index :{}", indexByPartitionPath); this.indexByPartitionReader = createReader(indexByPartitionPath, metaClient.getStorage()); } return indexByPartitionReader; @@ -126,7 +127,7 @@ private synchronized HFileReader partitionIndexReader() throws IOException { private synchronized HFileReader fileIdIndexReader() throws IOException { if (indexByFileIdReader == null) { - LOG.info("Opening fileId index :" + indexByFileIdPath); + log.info("Opening fileId index :{}", indexByFileIdPath); this.indexByFileIdReader = createReader(indexByFileIdPath, metaClient.getStorage()); } return indexByFileIdReader; @@ -181,7 +182,7 @@ public List getSourceFileMappingForPartition(String partit .map(e -> new BootstrapFileMapping(bootstrapBasePath, metadata.getBootstrapPartitionPath(), e.getValue(), partition, e.getKey())).collect(Collectors.toList()); } else { - LOG.warn("No value found for partition key ({})", partition); + log.warn("No value found for partition key ({})", partition); return new ArrayList<>(); } } catch (IOException ioe) { @@ -189,11 +190,6 @@ public List getSourceFileMappingForPartition(String partit } } - @Override - public String getBootstrapBasePath() { - return bootstrapBasePath; - } - @Override public Map getSourceFileMappingForFileIds( List ids) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java index bcd063ff5d0d0..8fd232a87402a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java @@ -35,8 +35,7 @@ import org.apache.hudi.io.hfile.HFileWriterImpl; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.OutputStream; @@ -53,8 +52,8 @@ import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.getPartitionKey; import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.partitionIndexPath; +@Slf4j public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndexWriter.class); private final String bootstrapBasePath; private final StoragePath indexByPartitionPath; @@ -80,7 +79,7 @@ public HFileBootstrapIndexWriter(String bootstrapBasePath, HoodieTableMetaClient || metaClient.getStorage().exists(indexByFileIdPath)) { String errMsg = "Previous version of bootstrap index exists. Partition Index Path :" + indexByPartitionPath + ", FileId index Path :" + indexByFileIdPath; - LOG.info(errMsg); + log.info(errMsg); throw new HoodieException(errMsg); } } catch (IOException ioe) { @@ -97,9 +96,9 @@ public HFileBootstrapIndexWriter(String bootstrapBasePath, HoodieTableMetaClient private void writeNextPartition(String partitionPath, String bootstrapPartitionPath, List bootstrapFileMappings) { try { - LOG.info("Adding bootstrap partition Index entry for partition :" + partitionPath - + ", bootstrap Partition :" + bootstrapPartitionPath + ", Num Entries :" + bootstrapFileMappings.size()); - LOG.info("ADDING entries :" + bootstrapFileMappings); + log.info("Adding bootstrap partition Index entry for partition :{}, bootstrap Partition :{}, Num Entries :{}", + partitionPath, bootstrapPartitionPath, bootstrapFileMappings.size()); + log.info("ADDING entries :{}", bootstrapFileMappings); HoodieBootstrapPartitionMetadata bootstrapPartitionMetadata = new HoodieBootstrapPartitionMetadata(); bootstrapPartitionMetadata.setBootstrapPartitionPath(bootstrapPartitionPath); bootstrapPartitionMetadata.setPartitionPath(partitionPath); @@ -148,14 +147,14 @@ private void commit() { .setNumKeys(numPartitionKeysAdded) .setBootstrapBasePath(bootstrapBasePath) .build(); - LOG.info("Adding Partition FileInfo :" + partitionIndexInfo); + log.info("Adding Partition FileInfo :{}", partitionIndexInfo); HoodieBootstrapIndexInfo fileIdIndexInfo = HoodieBootstrapIndexInfo.newBuilder() .setCreatedTimestamp(new Date().getTime()) .setNumKeys(numFileIdKeysAdded) .setBootstrapBasePath(bootstrapBasePath) .build(); - LOG.info("Appending FileId FileInfo :" + fileIdIndexInfo); + log.info("Appending FileId FileInfo :{}", fileIdIndexInfo); indexByPartitionWriter.appendFileInfo( INDEX_INFO_KEY_STRING, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java index f4b6bffd80807..2048b21c7e261 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.config; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * In Hudi, we have multiple superclasses, aka Config Classes of {@link HoodieConfig} that maintain * several configs. This class group one or more of these superclasses into higher @@ -29,6 +33,7 @@ public class ConfigGroups { * Config group names. Please add the description of each group in * {@link ConfigGroups#getDescription}. */ + @AllArgsConstructor(access = AccessLevel.PACKAGE) public enum Names { TABLE_CONFIG("Hudi Table Config"), ENVIRONMENT_CONFIG("Environment Config"), @@ -44,12 +49,9 @@ public enum Names { HUDI_STREAMER("Hudi Streamer Configs"); public final String name; - - Names(String name) { - this.name = name; - } } + @AllArgsConstructor(access = AccessLevel.PACKAGE) public enum SubGroupNames { INDEX( "Index Configs", @@ -80,16 +82,8 @@ public enum SubGroupNames { "No subgroup. This description should be hidden."); public final String name; + @Getter private final String description; - - SubGroupNames(String name, String description) { - this.name = name; - this.description = description; - } - - public String getDescription() { - return description; - } } public static String getDescription(Names names) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java index 56a087f1e7ea1..b93d980eafc12 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java @@ -22,6 +22,12 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieException; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; +import lombok.experimental.Accessors; + import java.io.Serializable; import java.lang.reflect.Field; import java.util.Arrays; @@ -41,14 +47,21 @@ * * @param The type of the default value. */ +@AllArgsConstructor(access = AccessLevel.PACKAGE) +@Getter public class ConfigProperty implements Serializable { + @NonNull + @Accessors(fluent = true) // Required so that #key() is generated instead of #getKey() by Lombok private final String key; + @Getter(AccessLevel.NONE) private final T defaultValue; + @Getter(AccessLevel.NONE) private final String docOnDefaultValue; + @Getter(AccessLevel.NONE) private final String doc; private final Option sinceVersion; @@ -57,37 +70,16 @@ public class ConfigProperty implements Serializable { private final List supportedVersions; + // provide the ability to infer config value based on other configs + private final Option>> inferFunction; + + @Getter(AccessLevel.NONE) private final Set validValues; private final boolean advanced; private final String[] alternatives; - // provide the ability to infer config value based on other configs - private final Option>> inferFunction; - - ConfigProperty(String key, T defaultValue, String docOnDefaultValue, String doc, - Option sinceVersion, Option deprecatedVersion, - List supportedVersions, - Option>> inferFunc, Set validValues, - boolean advanced, String... alternatives) { - this.key = Objects.requireNonNull(key); - this.defaultValue = defaultValue; - this.docOnDefaultValue = docOnDefaultValue; - this.doc = doc; - this.sinceVersion = sinceVersion; - this.deprecatedVersion = deprecatedVersion; - this.supportedVersions = supportedVersions; - this.inferFunction = inferFunc; - this.validValues = validValues; - this.advanced = advanced; - this.alternatives = alternatives; - } - - public String key() { - return key; - } - public T defaultValue() { if (defaultValue == null) { throw new HoodieException(String.format("There's no default value for this config: %s", key)); @@ -108,26 +100,10 @@ public String doc() { return StringUtils.isNullOrEmpty(doc) ? StringUtils.EMPTY_STRING : doc; } - public Option getSinceVersion() { - return sinceVersion; - } - - public Option getDeprecatedVersion() { - return deprecatedVersion; - } - - public List getSupportedVersions() { - return supportedVersions; - } - public boolean hasInferFunction() { return getInferFunction().isPresent(); } - public Option>> getInferFunction() { - return inferFunction; - } - public void checkValues(String value) { if (!isValid(value)) { throw new IllegalArgumentException( @@ -144,10 +120,6 @@ public List getAlternatives() { return Arrays.asList(alternatives); } - public boolean isAdvanced() { - return advanced; - } - public ConfigProperty withDocumentation(String doc) { Objects.requireNonNull(doc); return new ConfigProperty<>(key, defaultValue, docOnDefaultValue, doc, sinceVersion, deprecatedVersion, supportedVersions, inferFunction, validValues, advanced, alternatives); @@ -271,7 +243,7 @@ public ConfigProperty defaultValue(T value) { public ConfigProperty defaultValue(T value, String docOnDefaultValue) { Objects.requireNonNull(docOnDefaultValue); return new ConfigProperty<>(key, value, docOnDefaultValue, "", Option.empty(), - Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false); + Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false, new String[0]); } public ConfigProperty noDefaultValue() { @@ -280,7 +252,7 @@ public ConfigProperty noDefaultValue() { public ConfigProperty noDefaultValue(String docOnDefaultValue) { return new ConfigProperty<>(key, null, docOnDefaultValue, "", Option.empty(), - Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false); + Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false, new String[0]); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java index 498de3821666c..6e6668b0284c2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java @@ -24,8 +24,8 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.lang.reflect.Modifier; @@ -39,16 +39,16 @@ /** * This class deals with {@link ConfigProperty} and provides get/set functionalities. */ +@Slf4j public class HoodieConfig implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieConfig.class); - protected static final String CONFIG_VALUES_DELIMITER = ","; // Number of retries while reading the properties file to deal with parallel updates protected static final int MAX_READ_RETRIES = 5; // Delay between retries while reading the properties file protected static final int READ_RETRY_DELAY_MSEC = 1000; + @Getter protected TypedProperties props; public HoodieConfig() { @@ -246,10 +246,6 @@ public String getStringOrDefault(String key, String defaultVal) { return Option.ofNullable(props.getProperty(key)).orElse(defaultVal); } - public TypedProperties getProps() { - return props; - } - public TypedProperties getProps(boolean includeGlobalProps) { if (includeGlobalProps) { TypedProperties mergedProps = loadGlobalProperties(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java index 4b11a348bb059..466b8c99769e3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java @@ -30,8 +30,8 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; @@ -42,10 +42,10 @@ * This abstract strategy is used for direct marker writers, trying to do early conflict detection. */ @PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING) +@AllArgsConstructor +@Slf4j public abstract class DirectMarkerBasedDetectionStrategy implements EarlyConflictDetectionStrategy { - private static final Logger LOG = LoggerFactory.getLogger(DirectMarkerBasedDetectionStrategy.class); - protected final HoodieStorage storage; protected final String partitionPath; protected final String fileId; @@ -53,17 +53,6 @@ public abstract class DirectMarkerBasedDetectionStrategy implements EarlyConflic protected final HoodieActiveTimeline activeTimeline; protected final HoodieConfig config; - public DirectMarkerBasedDetectionStrategy(HoodieStorage storage, String partitionPath, String fileId, - String instantTime, - HoodieActiveTimeline activeTimeline, HoodieConfig config) { - this.storage = storage; - this.partitionPath = partitionPath; - this.fileId = fileId; - this.instantTime = instantTime; - this.activeTimeline = activeTimeline; - this.config = config; - } - /** * We need to do list operation here. * In order to reduce the list pressure as much as possible, first we build path prefix in advance: @@ -106,7 +95,7 @@ public boolean checkMarkerConflict(String basePath, long maxAllowableHeartbeatIn }).count(); if (res != 0L) { - LOG.warn("Detected conflict marker files: {}/{} for {}", partitionPath, fileId, instantTime); + log.warn("Detected conflict marker files: {}/{} for {}", partitionPath, fileId, instantTime); return true; } return false; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java index b603b99d93022..f31a0072ab9de 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java @@ -21,8 +21,9 @@ import org.apache.hudi.common.util.Either; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.util.Iterator; import java.util.List; @@ -34,6 +35,7 @@ * * @param Object value type. */ +@Slf4j public abstract class HoodieBaseListData { protected final Either, List> data; @@ -86,13 +88,11 @@ protected List collectAsList() { } } + @AllArgsConstructor(access = AccessLevel.PACKAGE) + @Slf4j static class IteratorCloser implements Runnable { - private static final Logger LOG = LoggerFactory.getLogger(IteratorCloser.class); - private final Iterator iterator; - IteratorCloser(Iterator iterator) { - this.iterator = iterator; - } + private final Iterator iterator; @Override public void run() { @@ -100,7 +100,7 @@ public void run() { try { ((AutoCloseable) iterator).close(); } catch (Exception ex) { - LOG.warn("Failed to properly close iterator", ex); + log.warn("Failed to properly close iterator", ex); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java index cf2893e1249c0..b4663509002d1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java @@ -41,6 +41,9 @@ import org.apache.hudi.keygen.KeyGenerator; import org.apache.hudi.storage.StorageConfiguration; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -51,6 +54,8 @@ * Base class contains the context information needed by the engine at runtime. It will be extended by different * engine implementation if needed. */ +@AllArgsConstructor +@Getter public abstract class HoodieEngineContext { /** @@ -60,19 +65,6 @@ public abstract class HoodieEngineContext { protected TaskContextSupplier taskContextSupplier; - public HoodieEngineContext(StorageConfiguration storageConf, TaskContextSupplier taskContextSupplier) { - this.storageConf = storageConf; - this.taskContextSupplier = taskContextSupplier; - } - - public StorageConfiguration getStorageConf() { - return storageConf; - } - - public TaskContextSupplier getTaskContextSupplier() { - return taskContextSupplier; - } - public abstract HoodieAccumulator newAccumulator(); public abstract HoodieData emptyHoodieData(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java index 953014802cf22..14e412ffcb184 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java @@ -53,6 +53,10 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -77,28 +81,53 @@ * and {@code RowData} in Flink. */ public abstract class HoodieReaderContext { + + @Getter private final StorageConfiguration storageConfiguration; protected final HoodieFileFormat baseFileFormat; // For general predicate pushdown. + @Getter protected final Option keyFilterOpt; protected final HoodieTableConfig tableConfig; + @Setter private String tablePath = null; + @Getter + @Setter private String latestCommitTime = null; + @Getter + @Setter private Option recordMerger = null; + @Getter + @Setter private Boolean hasLogFiles = null; + @Getter + @Setter private Boolean hasBootstrapBaseFile = null; + @Getter + @Setter private Boolean needsBootstrapMerge = null; + @Getter + @Setter // should we do position based merging for mor private Boolean shouldMergeUseRecordPosition = null; protected Option instantRangeOpt; + @Getter private RecordMergeMode mergeMode; + @Getter protected RecordContext recordContext; + @Getter + @Setter private FileGroupReaderSchemaHandler schemaHandler = null; // the default iterator mode is engine-specific record mode + @Setter private IteratorMode iteratorMode = IteratorMode.ENGINE_RECORD; + @Getter protected final HoodieConfig hoodieReaderConfig; - private boolean enableLogicalTimestampFieldRepair = true; + @Getter + @Setter + @Accessors(fluent = true) + private Boolean enableLogicalTimestampFieldRepair = true; protected HoodieReaderContext(StorageConfiguration storageConfiguration, HoodieTableConfig tableConfig, @@ -123,19 +152,6 @@ protected HoodieReaderContext(StorageConfiguration storageConfiguration, this.hoodieReaderConfig = hoodieReaderConfig; } - // Getter and Setter for schemaHandler - public FileGroupReaderSchemaHandler getSchemaHandler() { - return schemaHandler; - } - - public void setSchemaHandler(FileGroupReaderSchemaHandler schemaHandler) { - this.schemaHandler = schemaHandler; - } - - public void setIteratorMode(IteratorMode iteratorMode) { - this.iteratorMode = iteratorMode; - } - public IteratorMode getIteratorMode() { ValidationUtils.checkArgument(iteratorMode != null, "iterator mode should not be null!"); return this.iteratorMode; @@ -148,82 +164,10 @@ public String getTablePath() { return tablePath; } - public void setEnableLogicalTimestampFieldRepair(boolean enableLogicalTimestampFieldRepair) { - this.enableLogicalTimestampFieldRepair = enableLogicalTimestampFieldRepair; - } - - public void setTablePath(String tablePath) { - this.tablePath = tablePath; - } - - public String getLatestCommitTime() { - return latestCommitTime; - } - - public void setLatestCommitTime(String latestCommitTime) { - this.latestCommitTime = latestCommitTime; - } - - public Option getRecordMerger() { - return recordMerger; - } - - public void setRecordMerger(Option recordMerger) { - this.recordMerger = recordMerger; - } - - // Getter and Setter for hasLogFiles - public boolean getHasLogFiles() { - return hasLogFiles; - } - - public void setHasLogFiles(boolean hasLogFiles) { - this.hasLogFiles = hasLogFiles; - } - - // Getter and Setter for hasBootstrapBaseFile - public boolean getHasBootstrapBaseFile() { - return hasBootstrapBaseFile; - } - - public void setHasBootstrapBaseFile(boolean hasBootstrapBaseFile) { - this.hasBootstrapBaseFile = hasBootstrapBaseFile; - } - - // Getter and Setter for needsBootstrapMerge - public boolean getNeedsBootstrapMerge() { - return needsBootstrapMerge; - } - - public boolean enableLogicalTimestampFieldRepair() { - return enableLogicalTimestampFieldRepair; - } - - public void setNeedsBootstrapMerge(boolean needsBootstrapMerge) { - this.needsBootstrapMerge = needsBootstrapMerge; - } - - // Getter and Setter for useRecordPosition - public boolean getShouldMergeUseRecordPosition() { - return shouldMergeUseRecordPosition; - } - - public void setShouldMergeUseRecordPosition(boolean shouldMergeUseRecordPosition) { - this.shouldMergeUseRecordPosition = shouldMergeUseRecordPosition; - } - - public StorageConfiguration getStorageConfiguration() { - return storageConfiguration; - } - public TypedProperties getMergeProps(TypedProperties props) { return ConfigUtils.getMergeProps(props, this.tableConfig); } - public Option getKeyFilterOpt() { - return keyFilterOpt; - } - public SizeEstimator> getRecordSizeEstimator() { return new HoodieRecordSizeEstimator<>(getSchemaHandler().getSchemaForUpdates()); } @@ -232,14 +176,6 @@ public CustomSerializer> getRecordSerializer() { return new DefaultSerializer<>(); } - public RecordContext getRecordContext() { - return recordContext; - } - - public HoodieConfig getHoodieReaderConfig() { - return hoodieReaderConfig; - } - /** * Gets the record iterator based on the type of engine-specific record representation from the * file. @@ -335,10 +271,6 @@ private void initRecordMerger(TypedProperties properties, boolean isIngestion) { properties.getString(RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY, ""))); } - public RecordMergeMode getMergeMode() { - return mergeMode; - } - /** * Get the {@link InstantRange} filter. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java index b1fa5d97892ed..d63c9bd63408b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java @@ -35,6 +35,8 @@ import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.KeyGenerator; +import lombok.Getter; +import lombok.Setter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -65,7 +67,9 @@ public abstract class RecordContext implements Serializable { // for encoding and decoding schemas to the spillable map private final LocalHoodieSchemaCache localSchemaCache = LocalHoodieSchemaCache.getInstance(); + @Getter protected final JavaTypeConverter typeConverter; + @Setter protected String partitionPath; protected RecordContext(HoodieTableConfig tableConfig, JavaTypeConverter typeConverter) { @@ -84,10 +88,6 @@ protected RecordContext(JavaTypeConverter typeConverter) { this.typeConverter = typeConverter; } - public void setPartitionPath(String partitionPath) { - this.partitionPath = partitionPath; - } - public T extractDataFromRecord(HoodieRecord record, HoodieSchema schema, Properties properties) { return (T) record.getData(); } @@ -170,10 +170,6 @@ public abstract T mergeWithEngineRecord(HoodieSchema schema, */ public abstract T constructEngineRecord(HoodieSchema recordSchema, Object[] fieldValues); - public JavaTypeConverter getTypeConverter() { - return typeConverter; - } - /** * Gets the record key in String. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 56c0cc3a5b460..1b5f3ea62e787 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -45,8 +45,7 @@ import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.inline.InLineFSUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.FileNotFoundException; @@ -72,9 +71,9 @@ /** * Utility functions related to accessing the file storage. */ +@Slf4j public class FSUtils { - private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class); // Log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1 // Archive log files are of this pattern - .commits_.archive.1_1-0-1 public static final String PATH_SEPARATOR = "/"; @@ -624,7 +623,7 @@ public static boolean deleteDir( pairOfSubPathAndConf.getKey(), pairOfSubPathAndConf.getValue(), true) ); boolean result = storage.deleteDirectory(dirPath); - LOG.info("Removed directory at {}", dirPath); + log.info("Removed directory at {}", dirPath); return result; } } catch (IOException ioe) { @@ -803,7 +802,7 @@ public static Map deleteFilesParallelize( if (!ignoreFailed) { throw new HoodieIOException("Failed to delete : " + file, e); } else { - LOG.info("Ignore failed deleting : {}", file); + log.info("Ignore failed deleting : {}", file); return true; } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java index ed82343ee3be7..ada10aaa2fd03 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java @@ -23,8 +23,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.FileNotFoundException; import java.io.IOException; @@ -36,10 +35,9 @@ /** * A consistency checker that fails if it is unable to meet the required condition within a specified timeout. */ +@Slf4j public class FailSafeConsistencyGuard implements ConsistencyGuard { - private static final Logger LOG = LoggerFactory.getLogger(FailSafeConsistencyGuard.class); - protected final HoodieStorage storage; protected final ConsistencyGuardConfig consistencyGuardConfig; @@ -129,7 +127,7 @@ private void waitForFileVisibility(StoragePath filePath, FileVisibility visibili return; } } catch (IOException ioe) { - LOG.warn("Got IOException waiting for file visibility. Retrying", ioe); + log.warn("Got IOException waiting for file visibility. Retrying", ioe); } sleepSafe(waitMs); @@ -152,7 +150,7 @@ private void retryTillSuccess(StoragePath dir, List files, FileVisibilit throws TimeoutException { long waitMs = consistencyGuardConfig.getInitialConsistencyCheckIntervalMs(); int attempt = 0; - LOG.info("Max Attempts=" + consistencyGuardConfig.getMaxConsistencyChecks()); + log.info("Max Attempts={}", consistencyGuardConfig.getMaxConsistencyChecks()); while (attempt < consistencyGuardConfig.getMaxConsistencyChecks()) { boolean success = checkFilesVisibility(attempt, dir, files, event); if (success) { @@ -178,7 +176,7 @@ private void retryTillSuccess(StoragePath dir, List files, FileVisibilit protected boolean checkFilesVisibility(int retryNum, StoragePath dir, List files, FileVisibility event) { try { - LOG.info("Trying " + retryNum); + log.info("Trying {}", retryNum); List entries = storage.listDirectEntries(dir); List gotFiles = entries.stream() .map(e -> e.getPath().getPathWithoutSchemeAndAuthority()) @@ -188,7 +186,7 @@ protected boolean checkFilesVisibility(int retryNum, StoragePath dir, List files) throws Ti Thread.sleep(consistencyGuardConfig.getOptimisticConsistencyGuardSleepTimeMs()); } } catch (InterruptedException ie) { - LOG.warn("Got InterruptedException waiting for file visibility. Ignoring", ie); + log.warn("Got InterruptedException waiting for file visibility. Ignoring", ie); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java index a51914554d715..fab4511c703f2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java @@ -23,16 +23,15 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; /** * Common utils for Hudi heartbeat */ +@Slf4j public class HoodieHeartbeatUtils { - private static final Logger LOG = LoggerFactory.getLogger(HoodieHeartbeatUtils.class); /** * Use modification time as last heart beat time. @@ -72,7 +71,7 @@ public static boolean isHeartbeatExpired(String instantTime, Long currentTime = System.currentTimeMillis(); Long lastHeartbeatTime = getLastHeartbeatTime(storage, basePath, instantTime); if (currentTime - lastHeartbeatTime > maxAllowableHeartbeatIntervalInMs) { - LOG.warn("Heartbeat expired, for instant: {}", instantTime); + log.warn("Heartbeat expired, for instant: {}", instantTime); return true; } return false; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java index dff06a4c0e0f0..0b0978d552a23 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java @@ -89,6 +89,7 @@ * * @since 1.2.0 */ +@Getter public class HoodieSchema implements Serializable { private static final long serialVersionUID = 1L; @@ -1567,18 +1568,6 @@ private static int getNextOffset(String path, int offset, String component) { return (path.charAt(next) == '.') ? next + 1 : -1; } - /** - * Returns the underlying Avro schema for compatibility purposes. - * - *

    This method is provided for gradual migration and should be used - * sparingly. New code should prefer the HoodieSchema API.

    - * - * @return the wrapped Avro Schema - */ - public Schema getAvroSchema() { - return avroSchema; - } - /** * Converts this HoodieSchema to an Avro Schema. * This is an alias for getAvroSchema() provided for API consistency. @@ -1909,7 +1898,9 @@ public HoodieSchema build() { } public static class Decimal extends HoodieSchema { + @Getter private final int precision; + @Getter private final int scale; private final Option fixedSize; @@ -1934,14 +1925,6 @@ private Decimal(Schema avroSchema) { } } - public int getPrecision() { - return precision; - } - - public int getScale() { - return scale; - } - @Override public String getName() { return String.format("decimal(%d,%d)", precision, scale); @@ -2224,7 +2207,9 @@ public int hashCode() { } public static class Timestamp extends HoodieSchema { + @Getter private final boolean isUtcAdjusted; + @Getter private final TimePrecision precision; /** @@ -2255,14 +2240,6 @@ private Timestamp(Schema avroSchema) { } } - public TimePrecision getPrecision() { - return precision; - } - - public boolean isUtcAdjusted() { - return isUtcAdjusted; - } - @Override public String getName() { if (isUtcAdjusted) { @@ -2299,6 +2276,7 @@ public int hashCode() { } public static class Time extends HoodieSchema { + @Getter private final TimePrecision precision; /** @@ -2321,10 +2299,6 @@ private Time(Schema avroSchema) { } } - public TimePrecision getPrecision() { - return precision; - } - @Override public String getName() { if (precision == TimePrecision.MILLIS) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java index 9842cdd098c87..0fd96e55233ec 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java @@ -24,6 +24,9 @@ import org.apache.hudi.exception.SchemaBackwardsCompatibilityException; import org.apache.hudi.internal.schema.HoodieSchemaException; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -45,12 +48,9 @@ *
  • Metadata field handling during schema checks
    • * */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) public final class HoodieSchemaCompatibility { - // Prevent instantiation - private HoodieSchemaCompatibility() { - } - public static boolean areSchemasCompatible(HoodieSchema tableSchema, HoodieSchema writerSchema) { return HoodieSchemaCompatibilityChecker.checkReaderWriterCompatibility(tableSchema, writerSchema, false).getType() == HoodieSchemaCompatibilityChecker.SchemaCompatibilityType.COMPATIBLE; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java index 190e180b4968b..9b1d16952192b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; +import lombok.Getter; import org.apache.avro.Schema; import java.io.Serializable; @@ -55,6 +56,7 @@ public class HoodieSchemaField implements Serializable { private static final long serialVersionUID = 1L; + @Getter private final Schema.Field avroField; private final HoodieSchema fieldSchema; @@ -251,18 +253,6 @@ public Set aliases() { return avroField.aliases(); } - /** - * Returns the underlying Avro field for compatibility purposes. - * - *

    This method is provided for gradual migration and should be used - * sparingly. New code should prefer the HoodieSchemaField API.

    - * - * @return the wrapped Avro Schema.Field - */ - public Schema.Field getAvroField() { - return avroField; - } - /** * Creates a copy of this field with a new name. * diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index d01669b507f32..6535f6be700e1 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -629,10 +629,15 @@ private void performClean(String instant, List files, String cleanInstan throws IOException { Map> partitionToFiles = deleteFiles(files); List cleanStats = partitionToFiles.entrySet().stream().map(e -> - new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, e.getKey(), e.getValue(), e.getValue(), - new ArrayList<>(), - instant.length() < 3 ? String.valueOf(Integer.parseInt(instant) + 1) : HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1), - "")).collect(Collectors.toList()); + HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(e.getKey()) + .withDeletePathPatterns(e.getValue()) + .withSuccessDeleteFiles(e.getValue()) + .withFailedDeleteFiles(Collections.emptyList()) + .withEarliestCommitToRetain(instant.length() < 3 ? String.valueOf(Integer.parseInt(instant) + 1) : HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1)) + .withLastCompletedCommitTimestamp("") + .build()).collect(Collectors.toList()); HoodieInstant cleanInflightInstant = INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.CLEAN_ACTION, cleanInstant); metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index d0ec62e0abb25..46b5248e6f337 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -451,14 +451,12 @@ public HoodieTestTable addClean( public HoodieTestTable addClean(String instantTime) throws IOException { HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new HoodieActionInstant(EMPTY_STRING, EMPTY_STRING, EMPTY_STRING), EMPTY_STRING, EMPTY_STRING, new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(), Collections.EMPTY_MAP); - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); return HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata); } @@ -468,8 +466,14 @@ public Pair getHoodieCleanMetadata(Strin EMPTY_STRING, EMPTY_STRING, new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(), Collections.EMPTY_MAP); List cleanStats = new ArrayList<>(); for (Map.Entry> entry : testTableState.getPartitionToFileIdMapForCleaner(commitTime).entrySet()) { - cleanStats.add(new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - entry.getKey(), entry.getValue(), entry.getValue(), Collections.emptyList(), commitTime, "")); + cleanStats.add(HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(entry.getKey()) + .withDeletePathPatterns(entry.getValue()) + .withSuccessDeleteFiles(entry.getValue()) + .withEarliestCommitToRetain(commitTime) + .withLastCompletedCommitTimestamp("") + .build()); } return Pair.of(cleanerPlan, convertCleanMetadata(commitTime, Option.of(0L), cleanStats, Collections.EMPTY_MAP)); } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala index da5bef9f785ca..3fe8c6ff62f71 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala @@ -304,7 +304,7 @@ class HoodieFileGroupReaderBasedFileFormat(tablePath: String, val readerContext = new SparkFileFormatInternalRowReaderContext( fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig, sparkRequiredSchema = Some(requiredSchema)) - readerContext.setEnableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) + readerContext.enableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) val props = metaClient.getTableConfig.getProps options.foreach(kv => props.setProperty(kv._1, kv._2)) props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)) From 32c564aea5efd052c55daa79f3024a5975985dff Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 3 Jun 2026 20:06:12 +0800 Subject: [PATCH 046/255] refactor: Add Lombok Builder annotation to HoodieLogFormat (#17785) * refactor: Add Lombok Builder annotation to HoodieLogFormat * Addressed comments * Addressed comments - Restore fixed DEFAULT_SIZE_THRESHOLD as the HDFS block size in createNewFile, decoupling it from the user-configurable log rollover threshold - Fix stale 'file len' comment -> 'file size' and drop redundant '= 0L' initializer on Writer.fileSize (cherry picked from commit 163a15a3c899130745071396171fe451e6e62ab1) --- .../commands/TestHoodieLogFileCommand.java | 34 +- .../versioning/v1/TimelineArchiverV1.java | 11 +- .../apache/hudi/io/HoodieAppendHandle.java | 4 +- .../org/apache/hudi/io/HoodieWriteHandle.java | 13 +- .../HoodieBackedTableMetadataWriter.java | 7 +- .../action/rollback/RollbackHelperV1.java | 7 +- .../TestLegacyArchivedMetaEntryReader.java | 12 +- .../testutils/HoodieWriteableTestTable.java | 12 +- .../HoodieFlinkWriteableTestTable.java | 14 +- .../common/table/log/HoodieLogFormat.java | 323 ++++++-------- .../table/log/HoodieLogFormatWriter.java | 114 ++--- .../functional/TestHoodieLogFormat.java | 402 ++++++++++++------ .../TestHoodieLogFormatAppendFailure.java | 22 +- .../common/table/TestTableSchemaResolver.java | 10 +- ... => TestHoodieLogFormatWriterBuilder.java} | 13 +- .../timeline/TestArchivedTimelineV1.java | 20 +- .../testutils/HoodieCommonTestHarness.java | 14 +- .../reader/HoodieFileSliceTestUtils.java | 7 +- .../hadoop/testutils/InputFormatTestUtil.java | 25 +- ...arkMergeOnReadTableInsertUpdateDelete.java | 7 +- .../TestShowTimelineTableProcedure.scala | 8 +- .../hudi/hive/testutils/HiveTestUtil.java | 18 +- .../TestHoodieMetadataTableValidator.java | 7 +- 23 files changed, 610 insertions(+), 494 deletions(-) rename hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/{TestHoodieLogWriterBuilder.java => TestHoodieLogFormatWriterBuilder.java} (87%) rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java (98%) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index 51bd2c2843f37..7ba69b323acad 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -36,6 +36,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; @@ -109,11 +110,14 @@ public void init() throws IOException, InterruptedException, URISyntaxException Files.createDirectories(Paths.get(partitionPath)); storage = HoodieStorageUtils.getStorage(tablePath, storageConf()); - try (HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder() - .onParentPath(new StoragePath(partitionPath)) + try (HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(partitionPath)) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-log-fileid1").withInstantTime("100").withStorage(storage) - .withSizeThreshold(1).build()) { + .withLogFileId("test-log-fileid1") + .withInstantTime("100") + .withStorage(storage) + .withSizeThreshold(1L) + .build()) { // write data to file List records = SchemaTestUtil.generateTestRecords(0, 100).stream().map(HoodieAvroIndexedRecord::new).collect(Collectors.toList()); @@ -203,16 +207,14 @@ public void testShowLogFileRecordsWithMerge() throws IOException, InterruptedExc partitionPath = tablePath + StoragePath.SEPARATOR + HoodieTestCommitMetadataGenerator.DEFAULT_SECOND_PARTITION_PATH; Files.createDirectories(Paths.get(partitionPath)); - HoodieLogFormat.Writer writer = null; - try { - // set little threshold to split file. - writer = - HoodieLogFormat.newWriterBuilder().onParentPath(new StoragePath(partitionPath)) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-log-fileid1").withInstantTime(INSTANT_TIME).withStorage( - storage) - .withSizeThreshold(500).build(); - + try (HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(partitionPath)) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-log-fileid1") + .withInstantTime(INSTANT_TIME) + .withStorage(storage) + .withSizeThreshold(500L) // set little threshold to split file. + .build()) { SchemaTestUtil testUtil = new SchemaTestUtil(); List records1 = testUtil.generateHoodieTestRecords(0, 100).stream().map(HoodieAvroIndexedRecord::new).collect(Collectors.toList()); Map header = new HashMap<>(); @@ -220,10 +222,6 @@ public void testShowLogFileRecordsWithMerge() throws IOException, InterruptedExc header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records1, header, HoodieRecord.RECORD_KEY_METADATA_FIELD); writer.appendBlock(dataBlock); - } finally { - if (writer != null) { - writer.close(); - } } Object result = shell.evaluate(() -> "show logfile records --logFilePathPattern " diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java index d518ac5525dd6..bab78324d7ded 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java @@ -31,8 +31,8 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; @@ -116,9 +116,12 @@ public TimelineArchiverV1(HoodieWriteConfig config, HoodieTable tabl private Writer openWriter(StoragePath archivePath) { try { if (this.writer == null) { - return HoodieLogFormat.newWriterBuilder().onParentPath(archivePath).withInstantTime("") - .withFileId(archiveFilePath.getName()).withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) - .withStorage(metaClient.getStorage()).build(); + return HoodieLogFormatWriter.builder() + .withParentPath(archivePath).withInstantTime("") + .withLogFileId(archiveFilePath.getName()) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withStorage(metaClient.getStorage()) + .build(); } else { return this.writer; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index 5ea8ba460f873..3b825bbf1ade7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -39,7 +39,7 @@ import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.AppendResult; -import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieHFileDataBlock; @@ -105,7 +105,7 @@ public class HoodieAppendHandle extends HoodieWriteHandle> recordItr; // Writer to log into the file group's latest slice. - protected Writer writer; + protected HoodieLogFormat.Writer writer; protected final List statuses; // Total number of records written during appending diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index cd22186aa8e09..cc8d9c7e927a7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.LogFileCreationCallback; import org.apache.hudi.common.table.read.DeleteContext; import org.apache.hudi.common.util.ConfigUtils; @@ -288,9 +289,9 @@ protected HoodieLogFormat.Writer createLogWriter(String instantTime, Option fileSliceOpt) { try { if (config.getWriteVersion().greaterThanOrEquals(HoodieTableVersion.EIGHT)) { - return HoodieLogFormat.newWriterBuilder() - .onParentPath(FSUtils.constructAbsolutePath(hoodieTable.getMetaClient().getBasePath(), partitionPath)) - .withFileId(fileId) + return HoodieLogFormatWriter.builder() + .withParentPath(FSUtils.constructAbsolutePath(hoodieTable.getMetaClient().getBasePath(), partitionPath)) + .withLogFileId(fileId) .withInstantTime(instantTime) .withFileSize(0L) .withSizeThreshold(config.getLogFileMaxSize()) @@ -305,9 +306,9 @@ protected HoodieLogFormat.Writer createLogWriter(String instantTime, String file Option latestLogFile = fileSliceOpt.isPresent() ? fileSliceOpt.get().getLatestLogFile() : Option.empty(); - return HoodieLogFormat.newWriterBuilder() - .onParentPath(FSUtils.constructAbsolutePath(hoodieTable.getMetaClient().getBasePath(), partitionPath)) - .withFileId(fileId) + return HoodieLogFormatWriter.builder() + .withParentPath(FSUtils.constructAbsolutePath(hoodieTable.getMetaClient().getBasePath(), partitionPath)) + .withLogFileId(fileId) .withInstantTime(instantTime) .withLogVersion(latestLogFile.map(HoodieLogFile::getLogVersion).orElse(HoodieLogFile.LOGFILE_BASE_VERSION)) .withFileSize(latestLogFile.map(HoodieLogFile::getFileSize).orElse(0L)) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 9ed26c586ef13..85701fcb715fd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -60,6 +60,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.read.HoodieFileGroupReader; @@ -1197,9 +1198,9 @@ private void initializeFileGroups(HoodieTableMetaClient dataMetaClient, Metadata final HoodieDeleteBlock block = new HoodieDeleteBlock(Collections.emptyList(), blockHeader); - try (HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder() - .onParentPath(FSUtils.constructAbsolutePath(metadataWriteConfig.getBasePath(), relativePartitionPath)) - .withFileId(fileGroupFileId) + try (HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(FSUtils.constructAbsolutePath(metadataWriteConfig.getBasePath(), relativePartitionPath)) + .withLogFileId(fileGroupFileId) .withInstantTime(instantTime) .withLogVersion(HoodieLogFile.LOGFILE_BASE_VERSION) .withFileSize(0L) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java index 79a28421751ef..1b288a70e8ba8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.model.IOType; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.LogFileCreationCallback; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; @@ -311,9 +312,9 @@ List> maybeDeleteAndCollectStats(HoodieEngineCo // Let's emit markers for rollback as well. markers are emitted under rollback instant time. WriteMarkers writeMarkers = WriteMarkersFactory.get(config.getMarkersType(), table, instantTime); - HoodieLogFormat.WriterBuilder writerBuilder = HoodieLogFormat.newWriterBuilder() - .onParentPath(FSUtils.constructAbsolutePath(metaClient.getBasePath(), partitionPath)) - .withFileId(fileId) + HoodieLogFormatWriter.HoodieLogFormatWriterBuilder writerBuilder = HoodieLogFormatWriter.builder() + .withParentPath(FSUtils.constructAbsolutePath(metaClient.getBasePath(), partitionPath)) + .withLogFileId(fileId) .withLogWriteToken(CommonClientUtils.generateWriteToken(taskContextSupplier)) .withInstantTime(tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) ? instantToRollback.requestedTime() : rollbackRequest.getLatestBaseInstant() diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java index 5317583bbe9c2..ac6e7df560381 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.ActiveAction; @@ -101,10 +102,13 @@ private void prepareLegacyArchivedTimeline(HoodieTableMetaClient metaClient) thr private HoodieLogFormat.Writer openWriter(HoodieTableMetaClient metaClient) { try { - return HoodieLogFormat.newWriterBuilder() - .onParentPath(metaClient.getArchivePath()) - .withFileId("commits").withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) - .withStorage(metaClient.getStorage()).withInstantTime("").build(); + return HoodieLogFormatWriter.builder() + .withParentPath(metaClient.getArchivePath()) + .withLogFileId("commits") + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withStorage(metaClient.getStorage()) + .withInstantTime("") + .build(); } catch (IOException e) { throw new HoodieException("Unable to initialize HoodieLogFormat writer", e); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java index 71a7d423ced67..f9802b1634761 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java @@ -33,6 +33,7 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.testutils.FileCreateUtilsLegacy; @@ -168,10 +169,13 @@ public Map> withLogAppends(String partition, String } private Pair appendRecordsToLogFile(String partitionPath, String fileId, List records) throws Exception { - try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder() - .onParentPath(new StoragePath(basePath, partitionPath)) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(fileId) - .withInstantTime(currentInstantTime).withStorage(storage).build()) { + try (HoodieLogFormat.Writer logWriter = HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(basePath, partitionPath)) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(fileId) + .withInstantTime(currentInstantTime) + .withStorage(storage) + .build()) { Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, currentInstantTime); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java index 09e24fbbb0fe4..8d4395a50d7dd 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java @@ -32,6 +32,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.util.collection.Pair; @@ -136,10 +137,13 @@ public Map> withLogAppends(List record private Pair appendRecordsToLogFile(List groupedRecords) throws Exception { String partitionPath = groupedRecords.get(0).getPartitionPath(); HoodieRecordLocation location = groupedRecords.get(0).getCurrentLocation(); - try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder() - .onParentPath(new StoragePath(basePath, partitionPath)) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(location.getFileId()) - .withInstantTime(location.getInstantTime()).withStorage(storage).build()) { + try (HoodieLogFormat.Writer logWriter = HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(basePath, partitionPath)) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(location.getFileId()) + .withInstantTime(location.getInstantTime()) + .withStorage(storage) + .build()) { Map header = new java.util.HashMap<>(); header.put(HeaderMetadataType.INSTANT_TIME, location.getInstantTime()); header.put(HeaderMetadataType.SCHEMA, schema.toString()); @@ -150,7 +154,7 @@ private Pair appendRecordsToLogFile(List gr HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), ""); return (IndexedRecord) val; } catch (IOException e) { - log.warn("Failed to convert record " + r.toString(), e); + log.warn("Failed to convert record {}", r, e); return null; } }).map(HoodieAvroIndexedRecord::new).collect(Collectors.toList()), header, HoodieRecord.RECORD_KEY_METADATA_FIELD)); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index 2c536ec077222..a675e0d9da895 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -24,13 +24,13 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Closeable; import java.io.IOException; @@ -60,31 +60,140 @@ public interface HoodieLogFormat { String DEFAULT_WRITE_TOKEN = "0-0-0"; - String DEFAULT_LOG_FORMAT_WRITER = "org.apache.hudi.common.table.log.HoodieLogFormatWriter"; - /** - * Writer interface to allow appending block to this file format. + * Abstract base class for appending blocks to the Hoodie log format. + * Subclasses provide specific implementations for writing to different storage layers. */ - interface Writer extends Closeable { + @Getter + @Slf4j + abstract class Writer implements Closeable { + + // Default max log file size 512 MB + public static final long DEFAULT_SIZE_THRESHOLD = 512 * 1024 * 1024L; + + // Buffer size + protected Integer bufferSize; + // FileSystem + protected HoodieStorage storage; + // Size threshold for the log file. Useful when used with a rolling log appender + protected Long sizeThreshold; + // Log File extension. Could be .avro.delta or .avro.commits etc + protected String fileExtension; + // File Id + protected String logFileId; + // File Commit Time stamp + protected String instantTime; + // version number for this log file. If not specified, then the current version will be + // computed by inspecting the file system + protected Integer logVersion; + // file size of this log file + protected Long fileSize; + // Location of the directory containing the log + protected StoragePath parentPath; + // Log File Write Token + protected String logWriteToken; + // optional file suffix + protected String suffix; + // file creation hook + protected LogFileCreationCallback fileCreationCallback; + protected HoodieLogFile logFile; + + protected HoodieTableVersion tableVersion; /** - * @return the path to the current {@link HoodieLogFile} being written to. + * Base constructor that performs the core Hudi Log logic. */ - HoodieLogFile getLogFile(); + protected Writer( + Integer bufferSize, + HoodieStorage storage, + StoragePath parentPath, + String logFileId, + String fileExtension, + String instantTime, + Integer logVersion, + String logWriteToken, + String suffix, + Long fileSize, + Long sizeThreshold, + LogFileCreationCallback fileCreationCallback, + HoodieTableVersion tableVersion) throws IOException { + log.info("Building HoodieLogFormat.Writer"); + + // Validation + ValidationUtils.checkArgument(storage != null, "Storage is not specified"); + ValidationUtils.checkArgument(logFileId != null, "FileID is not specified"); + ValidationUtils.checkArgument(instantTime != null, "Instant time is not specified"); + ValidationUtils.checkArgument(fileExtension != null, "File extension is not specified"); + ValidationUtils.checkArgument(parentPath != null, "Log file parent location is not specified"); + + this.bufferSize = bufferSize != null ? bufferSize : storage.getDefaultBufferSize(); + this.storage = storage; + this.parentPath = parentPath; + this.logFileId = logFileId; + this.fileExtension = fileExtension; + this.instantTime = instantTime; + this.logVersion = logVersion; + this.logWriteToken = logWriteToken; + this.suffix = suffix; + + // Defaults and logic + this.fileSize = fileSize != null ? fileSize : 0L; + this.sizeThreshold = sizeThreshold != null ? sizeThreshold : DEFAULT_SIZE_THRESHOLD; + // Does nothing by default + this.fileCreationCallback = fileCreationCallback != null ? fileCreationCallback : new LogFileCreationCallback() {}; + this.tableVersion = tableVersion != null ? tableVersion : HoodieTableVersion.current(); + + // Log version computation + if (this.logVersion == null) { + log.info("Computing next log version for {} in {}", logFileId, parentPath); + boolean useBaseVersion = this.tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) && this.logWriteToken != null; + + if (useBaseVersion) { + this.logVersion = HoodieLogFile.LOGFILE_BASE_VERSION; + } else { + // Compute from storage (expensive) + Option> versionAndToken = FSUtils.getLatestLogVersion(this.storage, this.parentPath, this.logFileId, this.fileExtension, this.instantTime); + if (versionAndToken.isPresent()) { + this.logVersion = versionAndToken.get().getKey(); + this.logWriteToken = versionAndToken.get().getValue(); + } else { + this.logVersion = HoodieLogFile.LOGFILE_BASE_VERSION; + this.logWriteToken = UNKNOWN_WRITE_TOKEN; + } + } + } + + if (this.logWriteToken == null) { + this.logWriteToken = UNKNOWN_WRITE_TOKEN; + } + + if (this.suffix != null) { + // A little hacky to simplify the file name concatenation: + // patch the write token with an optional suffix + // instead of adding a new extension + this.logWriteToken = this.logWriteToken + this.suffix; + } + + // Initialise logFile + StoragePath logPath = new StoragePath(parentPath, + FSUtils.makeLogFileName(this.logFileId, this.fileExtension, this.instantTime, this.logVersion, this.logWriteToken)); + log.info("HoodieLogFile on path {}", logPath); + this.logFile = new HoodieLogFile(logPath, this.fileSize); + } /** * Append Block to a log file. * @return {@link AppendResult} containing result of the append. */ - AppendResult appendBlock(HoodieLogBlock block) throws IOException, InterruptedException; + public abstract AppendResult appendBlock(HoodieLogBlock block) throws IOException, InterruptedException; /** * Appends the list of blocks to a logfile. * @return {@link AppendResult} containing result of the append. */ - AppendResult appendBlocks(List blocks) throws IOException, InterruptedException; + public abstract AppendResult appendBlocks(List blocks) throws IOException, InterruptedException; - long getCurrentSize() throws IOException; + public abstract long getCurrentSize() throws IOException; /** * Force previously appended blocks to durable storage so that downstream @@ -95,7 +204,7 @@ interface Writer extends Closeable { * mainly for tests that assert per-append visibility on the underlying * file system. */ - void sync() throws IOException; + public abstract void sync() throws IOException; } /** @@ -110,204 +219,20 @@ interface Reader extends Closeable, Iterator { /** * Read log file in reverse order and check if prev block is present. - * + * * @return {@code true} if previous block is present, {@code false} otherwise. */ boolean hasPrev(); /** * Read log file in reverse order and return prev block if present. - * + * * @return {@link HoodieLogBlock} the previous block * @throws IOException */ HoodieLogBlock prev() throws IOException; } - /** - * Builder class to construct the default log format writer. - */ - class WriterBuilder { - - private static final Logger LOG = LoggerFactory.getLogger(WriterBuilder.class); - // Default max log file size 512 MB - public static final long DEFAULT_SIZE_THRESHOLD = 512 * 1024 * 1024L; - - // Buffer size - private Integer bufferSize; - // FileSystem - private HoodieStorage storage; - // Size threshold for the log file. Useful when used with a rolling log appender - private Long sizeThreshold; - // Log File extension. Could be .avro.delta or .avro.commits etc - private String fileExtension; - // File Id - private String logFileId; - // File Commit Time stamp - private String instantTime; - // version number for this log file. If not specified, then the current version will be - // computed by inspecting the file system - private Integer logVersion; - // file len of this log file - private Long fileLen = 0L; - // Location of the directory containing the log - private StoragePath parentPath; - // Log File Write Token - private String logWriteToken; - // optional file suffix - private String suffix; - // file creation hook - private LogFileCreationCallback fileCreationCallback; - - private HoodieTableVersion tableVersion; - - public WriterBuilder withBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - - public WriterBuilder withLogWriteToken(String logWriteToken) { - this.logWriteToken = logWriteToken; - return this; - } - - public WriterBuilder withSuffix(String suffix) { - this.suffix = suffix; - return this; - } - - public WriterBuilder withStorage(HoodieStorage storage) { - this.storage = storage; - return this; - } - - public WriterBuilder withSizeThreshold(long sizeThreshold) { - this.sizeThreshold = sizeThreshold; - return this; - } - - public WriterBuilder withFileExtension(String logFileExtension) { - this.fileExtension = logFileExtension; - return this; - } - - public WriterBuilder withFileId(String fileId) { - this.logFileId = fileId; - return this; - } - - public WriterBuilder withInstantTime(String instantTime) { - this.instantTime = instantTime; - return this; - } - - public WriterBuilder withLogVersion(int version) { - this.logVersion = version; - return this; - } - - public WriterBuilder withFileSize(long fileLen) { - this.fileLen = fileLen; - return this; - } - - public WriterBuilder onParentPath(StoragePath parentPath) { - this.parentPath = parentPath; - return this; - } - - public WriterBuilder withFileCreationCallback(LogFileCreationCallback fileCreationCallback) { - this.fileCreationCallback = fileCreationCallback; - return this; - } - - public WriterBuilder withTableVersion(HoodieTableVersion writeTableVersion) { - this.tableVersion = writeTableVersion; - return this; - } - - public Writer build() throws IOException { - LOG.info("Building HoodieLogFormat Writer"); - if (storage == null) { - throw new IllegalArgumentException("fs is not specified"); - } - if (logFileId == null) { - throw new IllegalArgumentException("FileID is not specified"); - } - if (instantTime == null) { - throw new IllegalArgumentException("Instant time is not specified"); - } - if (fileExtension == null) { - throw new IllegalArgumentException("File extension is not specified"); - } - if (parentPath == null) { - throw new IllegalArgumentException("Log file parent location is not specified"); - } - - if (fileCreationCallback == null) { - // by default does nothing. - fileCreationCallback = new LogFileCreationCallback() {}; - } - - if (tableVersion == null) { - tableVersion = HoodieTableVersion.current(); - } - - if (logVersion == null) { - LOG.info("Computing the next log version for {} in {}", logFileId, parentPath); - boolean useBaseVersion = tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) - && logWriteToken != null; - if (useBaseVersion) { - // the log format writer handles the existence check. - logVersion = HoodieLogFile.LOGFILE_BASE_VERSION; - } else { - // compute from storage (expensive) - Option> versionAndWriteToken = - FSUtils.getLatestLogVersion(storage, parentPath, logFileId, fileExtension, instantTime); - if (versionAndWriteToken.isPresent()) { - logVersion = versionAndWriteToken.get().getKey(); - logWriteToken = versionAndWriteToken.get().getValue(); - } else { - // this is the case where there is no existing log-file. - logVersion = HoodieLogFile.LOGFILE_BASE_VERSION; - logWriteToken = UNKNOWN_WRITE_TOKEN; - } - } - LOG.info("Computed the next log version for {} in {} as {} with write-token {}", logFileId, parentPath, logVersion, logWriteToken); - } - - if (logWriteToken == null) { - fileLen = 0L; - logWriteToken = UNKNOWN_WRITE_TOKEN; - } - - if (suffix != null) { - // A little hacky to simplify the file name concatenation: - // patch the write token with an optional suffix - // instead of adding a new extension - logWriteToken = logWriteToken + suffix; - } - - StoragePath logPath = new StoragePath(parentPath, - FSUtils.makeLogFileName(logFileId, fileExtension, instantTime, logVersion, logWriteToken)); - LOG.info("HoodieLogFile on path {}", logPath); - HoodieLogFile logFile = new HoodieLogFile(logPath, fileLen); - - if (sizeThreshold == null) { - sizeThreshold = DEFAULT_SIZE_THRESHOLD; - } - return (Writer) ReflectionUtils.loadClass( - DEFAULT_LOG_FORMAT_WRITER, - new Class[] {HoodieStorage.class, HoodieLogFile.class, Integer.class, Short.class, Long.class, String.class, LogFileCreationCallback.class}, - storage, logFile, bufferSize, null, sizeThreshold, logWriteToken, fileCreationCallback - ); - } - } - - static WriterBuilder newWriterBuilder() { - return new WriterBuilder(); - } - static HoodieLogFormat.Reader newReader(HoodieStorage storage, HoodieLogFile logFile, HoodieSchema readerSchema) throws IOException { return new HoodieLogFileReader(storage, logFile, readerSchema, HoodieLogFileReader.DEFAULT_BUFFER_SIZE); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 1d6acc8d85ef3..2aed1d7dd87a2 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -19,13 +19,14 @@ package org.apache.hudi.common.table.log; -import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.table.log.HoodieLogFormat.WriterBuilder; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FSDataOutputStream; @@ -42,38 +43,36 @@ /** * HoodieLogFormatWriter can be used to append blocks to a log file Use HoodieLogFormat.WriterBuilder to construct. */ +@Getter @Slf4j -public class HoodieLogFormatWriter implements HoodieLogFormat.Writer { +public class HoodieLogFormatWriter extends HoodieLogFormat.Writer { - @Getter - private HoodieLogFile logFile; - private FSDataOutputStream output; - - private final HoodieStorage storage; - @Getter - private final long sizeThreshold; - private final Integer bufferSize; private final Short replication; - private final String rolloverLogWriteToken; - private final LogFileCreationCallback fileCreationHook; + private FSDataOutputStream outputStream; private boolean closed = false; private transient Thread shutdownThread = null; - public HoodieLogFormatWriter( - HoodieStorage storage, - HoodieLogFile logFile, + @Builder(setterPrefix = "with") + private HoodieLogFormatWriter( Integer bufferSize, - Short replication, + HoodieStorage storage, + StoragePath parentPath, + String logFileId, + String fileExtension, + String instantTime, + Integer logVersion, + String logWriteToken, + String suffix, + Long fileSize, Long sizeThreshold, - String rolloverLogWriteToken, - LogFileCreationCallback fileCreationHook) { - this.storage = storage; - this.logFile = logFile; - this.sizeThreshold = sizeThreshold; - this.bufferSize = bufferSize != null ? bufferSize : storage.getDefaultBufferSize(); + LogFileCreationCallback fileCreationCallback, + HoodieTableVersion tableVersion, + Short replication + ) throws IOException { + super(bufferSize, storage, parentPath, logFileId, fileExtension, instantTime, logVersion, logWriteToken, + suffix, fileSize, sizeThreshold, fileCreationCallback, tableVersion); + // outputStream is not initialized here, it will be lazily initialized in getOutputStream() this.replication = replication != null ? replication : storage.getDefaultReplication(logFile.getPath().getParent()); - this.rolloverLogWriteToken = rolloverLogWriteToken; - this.fileCreationHook = fileCreationHook; addShutDownHook(); } @@ -81,8 +80,8 @@ public HoodieLogFormatWriter( * Overrides the output stream, only for test purpose. */ @VisibleForTesting - public void withOutputStream(FSDataOutputStream output) { - this.output = output; + public void withOutputStream(FSDataOutputStream outputStream) { + this.outputStream = outputStream; } /** @@ -91,7 +90,7 @@ public void withOutputStream(FSDataOutputStream output) { * @throws IOException */ private FSDataOutputStream getOutputStream() throws IOException { - if (this.output == null) { + if (outputStream == null) { boolean created = false; while (!created) { try { @@ -116,7 +115,7 @@ private FSDataOutputStream getOutputStream() throws IOException { } } } - return output; + return outputStream; } @Override @@ -207,23 +206,32 @@ private int getLogBlockLength(int contentLength, int headerLength, int footerLen private void rolloverIfNeeded() throws IOException { // Roll over if the size is past the threshold - if (getCurrentSize() > sizeThreshold) { - log.info("CurrentSize {} has reached threshold {}. Rolling over to the next version", getCurrentSize(), sizeThreshold); + if (getCurrentSize() > getSizeThreshold()) { + log.info("CurrentSize {} has reached threshold {}. Rolling over to the next version", getCurrentSize(), getSizeThreshold()); rollOver(); } } private void rollOver() throws IOException { closeStream(); - this.logFile = logFile.rollOver(rolloverLogWriteToken); + this.logFile = getLogFile().rollOver(getLogWriteToken()); this.closed = false; } private void createNewFile() throws IOException { - fileCreationHook.preFileCreation(this.logFile); - this.output = new FSDataOutputStream( - storage.create(this.logFile.getPath(), false, bufferSize, replication, WriterBuilder.DEFAULT_SIZE_THRESHOLD), - new FileSystem.Statistics(storage.getScheme()) + getFileCreationCallback().preFileCreation(this.getLogFile()); + this.outputStream = new FSDataOutputStream( + getStorage().create( + this.getLogFile().getPath(), + false, + getBufferSize(), + getReplication(), + // HDFS block size is intentionally a fixed constant, independent of the + // log rollover threshold (getSizeThreshold()). A small rollover threshold + // must not shrink the underlying file's block size below HDFS limits. + DEFAULT_SIZE_THRESHOLD + ), + new FileSystem.Statistics(getStorage().getScheme()) ); } @@ -237,25 +245,25 @@ public void close() throws IOException { } private void closeStream() throws IOException { - if (output != null) { + if (outputStream != null) { // Persist all buffered data to DataNodes before closing so downstream // readers can observe a fully-written log file at commit-level visibility. sync(); - output.close(); - output = null; + outputStream.close(); + outputStream = null; closed = true; } } @Override public void sync() throws IOException { - if (output == null) { + if (outputStream == null) { return; // Presume closed } - output.flush(); + outputStream.flush(); // NOTE: the following API call makes sure that the data is flushed to disk on DataNodes (akin to POSIX fsync()) // See more details here: https://issues.apache.org/jira/browse/HDFS-744 - output.hsync(); + outputStream.hsync(); } @Override @@ -264,27 +272,25 @@ public long getCurrentSize() throws IOException { throw new IllegalStateException("Cannot get current size as the underlying stream has been closed already"); } - if (output == null) { + if (outputStream == null) { return 0; } - return output.getPos(); + return outputStream.getPos(); } /** * Close the output stream when the JVM exits. */ private void addShutDownHook() { - shutdownThread = new Thread() { - public void run() { - try { - log.info("running HoodieLogFormatWriter shutdown hook to close output stream for log file: {}", logFile); - closeStream(); - } catch (Exception e) { - log.warn("unable to close output stream for log file: {}", logFile, e); - // fail silently for any sort of exception - } + shutdownThread = new Thread(() -> { + try { + log.info("Running HoodieLogFormatWriter shutdown hook to close output stream for log file: {}", logFile); + closeStream(); + } catch (Exception e) { + log.warn("Unable to close output stream for log file: {}", logFile, e); + // fail silently for any sort of exception } - }; + }); Runtime.getRuntime().addShutdownHook(shutdownThread); } } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index 7e3ab4ebbbb86..c24f928b53f1e 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -45,7 +45,6 @@ import org.apache.hudi.common.table.log.HoodieLogFileReader; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; -import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.log.TestLogReaderUtils; @@ -207,10 +206,14 @@ public void testHoodieLogBlockTypeIsDataOrDeleteBlock() { @Test public void testEmptyLog() throws IOException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); assertEquals(0, writer.getCurrentSize(), "Just created this log, size should be 0"); assertTrue(writer.getLogFile().getFileName().startsWith("."), "Check all log files should start with a ."); assertEquals(1, writer.getLogFile().getLogVersion(), "Version should be 1 for new log created"); @@ -220,15 +223,18 @@ public void testEmptyLog() throws IOException { @ParameterizedTest @EnumSource(names = {"AVRO_DATA_BLOCK", "HFILE_DATA_BLOCK", "PARQUET_DATA_BLOCK"}) public void testBasicAppend(HoodieLogBlockType dataBlockType) throws IOException, InterruptedException, URISyntaxException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString()); - long pos = writer.getCurrentSize(); HoodieDataBlock dataBlock = getDataBlock(dataBlockType, records, header); AppendResult result = writer.appendBlock(dataBlock); @@ -245,10 +251,14 @@ public void testBasicAppend(HoodieLogBlockType dataBlockType) throws IOException @Test public void testRollover() throws IOException, InterruptedException, URISyntaxException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -265,10 +275,14 @@ public void testRollover() throws IOException, InterruptedException, URISyntaxEx // Create a writer with the size threshold as the size we just wrote - so this has to roll writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage) - .withSizeThreshold(size - 1).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .withSizeThreshold(size - 1) + .build(); records = SchemaTestUtil.generateTestRecords(0, 100); dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header); AppendResult secondAppend = writer.appendBlock(dataBlock); @@ -306,14 +320,19 @@ public void testConcurrentAppendOnFirstLogFileVersion() throws Exception { } private void testConcurrentAppend(boolean logFileExists, boolean newLogFileFormat) throws Exception { - HoodieLogFormat.WriterBuilder builder1 = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1") - .withInstantTime("100").withStorage(storage); - HoodieLogFormat.WriterBuilder builder2 = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1") - .withInstantTime("100").withStorage(storage); + HoodieLogFormatWriter.HoodieLogFormatWriterBuilder builder1 = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage); + + HoodieLogFormatWriter.HoodieLogFormatWriterBuilder builder2 = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage); if (newLogFileFormat && logFileExists) { // Assume there is an existing log-file with write token @@ -330,14 +349,14 @@ private void testConcurrentAppend(boolean logFileExists, boolean newLogFileForma } else { builder1 = builder1.withLogVersion(1).withLogWriteToken(HoodieLogFormat.UNKNOWN_WRITE_TOKEN); } - Writer writer = builder1.build(); + HoodieLogFormat.Writer writer = builder1.build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString()); HoodieDataBlock dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header); writer.appendBlock(dataBlock); - Writer writer2 = builder2.build(); + HoodieLogFormat.Writer writer2 = builder2.build(); writer2.appendBlock(dataBlock); HoodieLogFile logFile1 = writer.getLogFile(); HoodieLogFile logFile2 = writer2.getLogFile(); @@ -350,11 +369,15 @@ private void testConcurrentAppend(boolean logFileExists, boolean newLogFileForma @ParameterizedTest @EnumSource(names = {"AVRO_DATA_BLOCK", "HFILE_DATA_BLOCK", "PARQUET_DATA_BLOCK"}) public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withLogVersion(1).withInstantTime("100") - .withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withLogVersion(1) + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -365,10 +388,14 @@ public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOExcept writer.close(); writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withLogVersion(1).withInstantTime("100") - .withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withLogVersion(1) + .withInstantTime("100") + .withStorage(storage) + .build(); ((HoodieLogFormatWriter) writer).withOutputStream((FSDataOutputStream) storage.append(writer.getLogFile().getPath())); records = SchemaTestUtil.generateTestRecords(0, 100); @@ -384,10 +411,14 @@ public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOExcept // Close and Open again and append 100 more records writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withLogVersion(1).withInstantTime("100") - .withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withLogVersion(1) + .withInstantTime("100") + .withStorage(storage) + .build(); ((HoodieLogFormatWriter) writer).withOutputStream( (FSDataOutputStream) storage.append(writer.getLogFile().getPath())); records = SchemaTestUtil.generateTestRecords(0, 100); @@ -402,7 +433,7 @@ public void testMultipleAppend(HoodieLogBlockType dataBlockType) throws IOExcept writer.close(); // Cannot get the current size after closing the log - final Writer closedWriter = writer; + final HoodieLogFormat.Writer closedWriter = writer; assertThrows(IllegalStateException.class, closedWriter::getCurrentSize, "getCurrentSize should fail after the logAppender is closed"); } @@ -424,8 +455,10 @@ public void testAppendNotSupported(@TempDir java.nio.file.Path tempDir) throws I HoodieDataBlock dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header); for (int i = 0; i < 2; i++) { - Writer writer = HoodieLogFormat.newWriterBuilder().onParentPath(testPath) - .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits") + HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(testPath) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withLogFileId("commits") .withInstantTime("") .withStorage(localStorage).build(); writer.appendBlock(dataBlock); @@ -440,11 +473,12 @@ public void testAppendNotSupported(@TempDir java.nio.file.Path tempDir) throws I @ParameterizedTest @ValueSource(ints = {6, 8}) public void testBasicWriteAndScan(int tableVersion) throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withTableVersion(HoodieTableVersion.fromVersionCode(tableVersion)) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); HoodieSchema schema = getSimpleSchema(); List records = SchemaTestUtil.generateTestRecords(0, 100); List copyOfRecords = records.stream() @@ -475,10 +509,12 @@ private List convertAvroToSerializableIndexedRecords(List records = SchemaTestUtil.generateTestRecords(0, numRecords); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -1079,10 +1118,13 @@ private HoodieLogFile addValidBlock(String fileId, String commitTime, int numRec private HoodieLogFile appendValidBlock(StoragePath path, String fileId, String commitTime, int numRecords) throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId(fileId).withInstantTime(commitTime).withStorage(storage).build(); + .withLogFileId(fileId) + .withInstantTime(commitTime) + .withStorage(storage) + .build(); ((HoodieLogFormatWriter) writer).withOutputStream( (FSDataOutputStream) storage.append(path)); List records = SchemaTestUtil.generateTestRecords(0, numRecords); @@ -1097,10 +1139,13 @@ private HoodieLogFile appendValidBlock(StoragePath path, String fileId, String c @Test public void testValidateCorruptBlockEndPosition() throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -1153,11 +1198,14 @@ public void testAvroLogRecordReaderBasic(ExternalSpillableMap.DiskMapType diskMa throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage) - .withSizeThreshold(500).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .withSizeThreshold(500L) + .build(); SchemaTestUtil testUtil = new SchemaTestUtil(); // Write 1 @@ -1197,10 +1245,13 @@ public void testAvroLogRecordReaderWithRollbackTombstone(ExternalSpillableMap.Di throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1260,10 +1311,13 @@ public void testAvroLogRecordReaderWithFailedPartialBlock(ExternalSpillableMap.D throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1298,9 +1352,12 @@ public void testAvroLogRecordReaderWithFailedPartialBlock(ExternalSpillableMap.D outputStream.close(); writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 3 header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "103"); List records3 = testUtil.generateHoodieTestRecords(0, 100); @@ -1330,10 +1387,13 @@ public void testAvroLogRecordReaderWithDeleteAndRollback(ExternalSpillableMap.Di throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1471,10 +1531,13 @@ public void testAvroLogRecordReaderWithCommitBeforeAndAfterRollback(ExternalSpil HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version String fileId = "test-fileid111"; - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId(fileId).withInstantTime("100").withStorage(storage).build(); + .withLogFileId(fileId) + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 -> 100 records are written SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1579,10 +1642,13 @@ public void testAvroLogRecordReaderWithDisorderDelete(ExternalSpillableMap.DiskM throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1712,10 +1778,13 @@ public void testAvroLogRecordReaderWithFailedRollbacks(ExternalSpillableMap.Disk // Write a Data block and Delete block with same InstantTime (written in same batch) HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1781,10 +1850,13 @@ public void testAvroLogRecordReaderWithInsertDeleteAndRollback(ExternalSpillable // Write a Data block and Delete block with same InstantTime (written in same batch) HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1833,10 +1905,13 @@ public void testAvroLogRecordReaderWithInvalidRollback(ExternalSpillableMap.Disk throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1870,10 +1945,13 @@ public void testAvroLogRecordReaderWithInsertsDeleteAndRollback(ExternalSpillabl // Write a 3 Data blocs with same InstantTime (written in same batch) HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -1924,10 +2002,13 @@ void testLogReaderWithDifferentVersionsOfDeleteBlocks(ExternalSpillableMap.DiskM throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); List deleteKeyListInV2Block = Arrays.asList( "d448e1b8-a0d4-45c0-bf2d-a9e16ff3c8ce", "df3f71cd-5b68-406c-bb70-861179444adb", @@ -2047,10 +2128,13 @@ public void testAvroLogRecordReaderWithRollbackOlderBlocks() throws IOException, URISyntaxException, InterruptedException { HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -2107,10 +2191,13 @@ public void testAvroLogRecordReaderWithMixedInsertsCorruptsAndRollback(ExternalS // Write a 3 Data blocs with same InstantTime (written in same batch) HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -2152,9 +2239,13 @@ public void testAvroLogRecordReaderWithMixedInsertsCorruptsAndRollback(ExternalS outputStream.close(); writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); writer.appendBlock(dataBlock); writer.close(); @@ -2172,9 +2263,12 @@ public void testAvroLogRecordReaderWithMixedInsertsCorruptsAndRollback(ExternalS outputStream.close(); writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1 rollback block for the last commit instant header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101"); header.put(HeaderMetadataType.TARGET_INSTANT_TIME, "100"); @@ -2202,10 +2296,13 @@ public void testAvroLogRecordReaderWithMixedInsertsCorruptsRollbackAndMergedLogB // Write a 3 Data blocks with same InstantTime (written in same batch) HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); // Set a small threshold so that every block is a new version - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); // Write 1st data blocks multiple times. SchemaTestUtil testUtil = new SchemaTestUtil(); @@ -2275,9 +2372,12 @@ public void testAvroLogRecordReaderWithMixedInsertsCorruptsRollbackAndMergedLogB outputStream.close(); writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); ((HoodieLogFormatWriter) writer).withOutputStream( (FSDataOutputStream) storage.append(writer.getLogFile().getPath())); @@ -2405,9 +2505,13 @@ private void testAvroLogRecordReaderMergingMultipleLogFiles(int numRecordsInLog1 List records2 = new ArrayList<>(records); // Write1 with numRecordsInLog1 records written to log.1 - Writer writer = HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1") - .withInstantTime("100").withStorage(storage).build(); + HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -2419,9 +2523,14 @@ private void testAvroLogRecordReaderMergingMultipleLogFiles(int numRecordsInLog1 writer.close(); // write2 with numRecordsInLog2 records written to log.2 - Writer writer2 = HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1") - .withInstantTime("100").withStorage(storage).withSizeThreshold(size - 1).build(); + HoodieLogFormat.Writer writer2 = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .withSizeThreshold(size - 1) + .build(); Map header2 = new HashMap<>(); header2.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); @@ -2498,10 +2607,14 @@ public void testAvroLogRecordReaderTasksSucceededInBothStageAttempts(ExternalSpi @Test public void testBasicAppendAndReadInReverse() throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); HoodieSchema schema = getSimpleSchema(); List records1 = SchemaTestUtil.generateTestRecords(0, 100); List copyOfRecords1 = records1.stream() @@ -2568,10 +2681,13 @@ public void testBasicAppendAndReadInReverse() @Test public void testAppendAndReadOnCorruptedLogInReverse() throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); HoodieSchema schema = getSimpleSchema(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); @@ -2602,9 +2718,12 @@ public void testAppendAndReadOnCorruptedLogInReverse() // Should be able to append a new block writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); ((HoodieLogFormatWriter) writer).withOutputStream( (FSDataOutputStream) storage.append(writer.getLogFile().getPath())); records = SchemaTestUtil.generateTestRecords(0, 100); @@ -2630,10 +2749,13 @@ public void testAppendAndReadOnCorruptedLogInReverse() @Test public void testBasicAppendAndTraverseInReverse() throws IOException, URISyntaxException, InterruptedException { - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); HoodieSchema schema = getSimpleSchema(); List records1 = SchemaTestUtil.generateTestRecords(0, 100); List copyOfRecords1 = records1.stream() @@ -2717,10 +2839,10 @@ public void testV0Format() throws IOException, URISyntaxException { public void testDataBlockFormatAppendAndReadWithProjectedSchema( HoodieLogBlockType dataBlockType ) throws IOException, URISyntaxException, InterruptedException { - Writer writer = HoodieLogFormat.newWriterBuilder() - .onParentPath(partitionPath) + HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1") + .withLogFileId("test-fileid1") .withInstantTime("100") .withStorage(storage) .build(); @@ -2862,10 +2984,14 @@ private static List sort(List records) { private HoodieLogFormat.Reader createCorruptedFile(String fileId) throws Exception { // block is corrupted, but check is skipped. - Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId(fileId).withInstantTime("100").withStorage(storage).build(); + .withLogFileId(fileId) + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java index 7c08785ccf3a8..17cb86692ecbf 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java @@ -21,8 +21,8 @@ import org.apache.hudi.common.model.HoodieArchivedLogFile; import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; @@ -120,9 +120,13 @@ public void testFailedToGetAppendStreamFromHDFSNameNode() HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records, header, HoodieRecord.RECORD_KEY_METADATA_FIELD); - Writer writer = HoodieLogFormat.newWriterBuilder().onParentPath(testPath) - .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits") - .withInstantTime("").withStorage(storage).build(); + Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(testPath) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withLogFileId("commits") + .withInstantTime("") + .withStorage(storage) + .build(); writer.appendBlock(dataBlock); // get the current log file version to compare later @@ -152,9 +156,13 @@ public void testFailedToGetAppendStreamFromHDFSNameNode() // Opening a new Writer right now will throw IOException. The code should handle this, rollover the logfile and // return a new writer with a bumped up logVersion - writer = HoodieLogFormat.newWriterBuilder().onParentPath(testPath) - .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits") - .withInstantTime("").withStorage(storage).build(); + writer = HoodieLogFormatWriter.builder() + .withParentPath(testPath) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withLogFileId("commits") + .withInstantTime("") + .withStorage(storage) + .build(); header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE, String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_BLOCK.ordinal())); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java index 193bb0173dfda..f2c6d2b49b204 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -195,8 +196,13 @@ private String initTestDir(String folderName) throws IOException { private StoragePath writeLogFile(StoragePath partitionPath, Schema schema) throws IOException, URISyntaxException, InterruptedException { HoodieStorage storage = HoodieTestUtils.getStorage(partitionPath); HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build(); + HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(0, 100); Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogWriterBuilder.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogFormatWriterBuilder.java similarity index 87% rename from hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogWriterBuilder.java rename to hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogFormatWriterBuilder.java index e362a033cfe4a..c7ed090fb85fe 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogWriterBuilder.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieLogFormatWriterBuilder.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; @@ -38,21 +39,21 @@ import static org.mockito.Mockito.mock; /** - * Test class for {@link HoodieLogFormat#newWriterBuilder()}. + * Test class for {@link HoodieLogFormatWriter#builder()}. */ -public class TestHoodieLogWriterBuilder { +public class TestHoodieLogFormatWriterBuilder { - HoodieLogFormat.WriterBuilder builder; + HoodieLogFormatWriter.HoodieLogFormatWriterBuilder builder; HoodieLogFormat.Writer writer; HoodieStorage storage; @BeforeEach public void setup() { storage = mock(HoodieStorage.class); - builder = HoodieLogFormat.newWriterBuilder() - .withFileId("test-fileid1") + builder = HoodieLogFormatWriter.builder() + .withLogFileId("test-fileid1") .withInstantTime("100") - .onParentPath(new StoragePath("/tmp")) + .withParentPath(new StoragePath("/tmp")) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) .withStorage(storage); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java index 3142338119a92..d576087bb3f13 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java @@ -40,6 +40,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanV2MigrationHandler; @@ -835,16 +836,23 @@ private HoodieArchivedMetaEntry createArchivedMetaWrapper(HoodieInstant hoodieIn } private Writer buildWriter(StoragePath archiveFilePath) throws IOException { - return HoodieLogFormat.newWriterBuilder().onParentPath(archiveFilePath.getParent()) - .withFileId(archiveFilePath.getName()).withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) - .withStorage(metaClient.getStorage()).withInstantTime("").build(); + return HoodieLogFormatWriter.builder() + .withParentPath(archiveFilePath.getParent()) + .withLogFileId(archiveFilePath.getName()) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + .withStorage(metaClient.getStorage()) + .withInstantTime("") + .build(); } private Writer buildWriter(StoragePath archiveFilePath, int logVersion) throws IOException { - return HoodieLogFormat.newWriterBuilder().onParentPath(archiveFilePath.getParent()) - .withFileId(archiveFilePath.getName()).withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) + return HoodieLogFormatWriter.builder().withParentPath(archiveFilePath.getParent()) + .withLogFileId(archiveFilePath.getName()) + .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) .withLogVersion(logVersion) - .withStorage(metaClient.getStorage()).withInstantTime("").build(); + .withStorage(metaClient.getStorage()) + .withInstantTime("") + .build(); } private void writeArchiveLog(Writer writer, List records) throws Exception { diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java index a3602b3ff9e81..ed6a6be4105f5 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java @@ -358,12 +358,14 @@ protected static List writeLogFiles(StoragePath partitionPath, String commitTime, String logBlockInstantTime) throws IOException, InterruptedException { - HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withSizeThreshold(1024).withFileId(fileId) - .withInstantTime(commitTime) - .withStorage(storage).build(); + HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withSizeThreshold(1024L) + .withLogFileId(fileId) + .withInstantTime(commitTime) + .withStorage(storage) + .build(); if (storage.exists(writer.getLogFile().getPath())) { // enable append for reader test. ((HoodieLogFormatWriter) writer).withOutputStream( diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java similarity index 98% rename from hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java rename to hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java index 71e3fc6706818..d0f5bde6f41ed 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java @@ -36,6 +36,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCDCDataBlock; import org.apache.hudi.common.table.log.block.HoodieDataBlock; @@ -288,10 +289,10 @@ public static HoodieLogFile createLogFile( Map keyToPositionMap ) throws InterruptedException, IOException { try (HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder() - .onParentPath(new StoragePath(logFilePath).getParent()) + HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(logFilePath).getParent()) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId(fileId) + .withLogFileId(fileId) .withInstantTime(logInstantTime) .withLogVersion(version) .withStorage(storage).build()) { diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java index 274b2e21ac2b3..06f8113b5ee6e 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieDataBlock; @@ -371,10 +372,14 @@ public static HoodieLogFormat.Writer writeRollback(File partitionDir, HoodieStor int logVersion) throws InterruptedException, IOException { HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(new StoragePath(partitionDir.getPath())) - .withFileId(fileId) - .withInstantTime(baseCommit).withStorage(storage).withLogVersion(logVersion) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).build(); + HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(partitionDir.getPath())) + .withLogFileId(fileId) + .withInstantTime(baseCommit) + .withStorage(storage) + .withLogVersion(logVersion) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .build(); // generate metadata Map header = new HashMap<>(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, newCommit); @@ -406,8 +411,10 @@ public static HoodieLogFormat.Writer writeDataBlockToLogFile(File partitionDir, HoodieLogBlock.HoodieLogBlockType logBlockType) throws InterruptedException, IOException { HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(new StoragePath(partitionDir.getPath())) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(fileId) + HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(partitionDir.getPath())) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(fileId) .withLogVersion(logVersion) .withInstantTime(newCommit) .withStorage(storage) @@ -444,8 +451,10 @@ public static HoodieLogFormat.Writer writeRollbackBlockToLogFile(File partitionD String oldCommit, int logVersion) throws InterruptedException, IOException { HoodieLogFormat.Writer writer = - HoodieLogFormat.newWriterBuilder().onParentPath(new StoragePath(partitionDir.getPath())) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(fileId) + HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(partitionDir.getPath())) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(fileId) .withInstantTime(baseCommit) .withLogVersion(logVersion).withStorage(storage).build(); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java index d1f21ddb3c4dc..fcfbb2eb9832b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.AppendResult; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.LogFileCreationCallback; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieDataBlock; @@ -396,11 +397,11 @@ public void testSimpleInsertsGeneratedIntoLogFiles() throws Exception { final WriteMarkers writeMarkers = WriteMarkersFactory.get(config.getMarkersType(), HoodieSparkTable.create(config, context()), newCommitTime); - HoodieLogFormat.Writer fakeLogWriter = HoodieLogFormat.newWriterBuilder() - .onParentPath( + HoodieLogFormat.Writer fakeLogWriter = HoodieLogFormatWriter.builder() + .withParentPath( FSUtils.constructAbsolutePath(config.getBasePath(), correctWriteStat.getPartitionPath())) - .withFileId(correctWriteStat.getFileId()) + .withLogFileId(correctWriteStat.getFileId()) .withInstantTime(newCommitTime) .withLogVersion(correctLogFile.getLogVersion()) .withFileSize(0L) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowTimelineTableProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowTimelineTableProcedure.scala index c1e84cca83f18..c21b0ec8713b6 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowTimelineTableProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowTimelineTableProcedure.scala @@ -25,7 +25,7 @@ import org.apache.hudi.common.engine.{HoodieEngineContext, HoodieLocalEngineCont import org.apache.hudi.common.engine.LocalTaskContextSupplier import org.apache.hudi.common.model.{ActionType, HoodieArchivedLogFile, HoodieAvroIndexedRecord, HoodieCommitMetadata, HoodieLogFile, HoodieRecord, WriteOperationType} import org.apache.hudi.common.table.{HoodieTableMetaClient, HoodieTableVersion} -import org.apache.hudi.common.table.log.HoodieLogFormat +import org.apache.hudi.common.table.log.{HoodieLogFormat, HoodieLogFormatWriter} import org.apache.hudi.common.table.log.block.{HoodieAvroDataBlock, HoodieLogBlock} import org.apache.hudi.common.table.timeline.{ActiveAction, HoodieInstant, HoodieTimeline} import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion @@ -398,9 +398,9 @@ class TestShowTimelineTableProcedure extends HoodieSparkSqlTestBase { storage.createDirectory(archivePath) } - val writer = HoodieLogFormat.newWriterBuilder() - .onParentPath(archiveFilePath.getParent()) - .withFileId(archiveFilePath.getName()) + val writer = HoodieLogFormatWriter.builder() + .withParentPath(archiveFilePath.getParent()) + .withLogFileId(archiveFilePath.getName()) .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION) .withStorage(storage) .withInstantTime("") diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 24e76b2d9f695..0e603857f26dd 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -39,8 +39,8 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; @@ -725,8 +725,10 @@ private static HoodieLogFile generateLogData(StoragePath parquetFilePath, HoodieSchema schema = getTestDataSchema(isLogSchemaSimple); HoodieBaseFile dataFile = new HoodieBaseFile(storage.getPathInfo(parquetFilePath)); // Write a log file for this parquet file - Writer logWriter = HoodieLogFormat.newWriterBuilder().onParentPath(parquetFilePath.getParent()) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(dataFile.getFileId()) + Writer logWriter = HoodieLogFormatWriter.builder() + .withParentPath(parquetFilePath.getParent()) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(dataFile.getFileId()) .withInstantTime(dataFile.getCommitTime()).withStorage(storage).build(); List records = (isLogSchemaSimple ? SchemaTestUtil.generateTestRecords(0, 100) : SchemaTestUtil.generateEvolvedTestRecords(100, 100)).stream() @@ -745,9 +747,13 @@ private static HoodieLogFile generateLogData(StoragePath parquetFilePath, String HoodieSchema schema = SchemaTestUtil.getSchema(logSchemaPath); HoodieBaseFile dataFile = new HoodieBaseFile(storage.getPathInfo(parquetFilePath)); // Write a log file for this parquet file - Writer logWriter = HoodieLogFormat.newWriterBuilder().onParentPath(parquetFilePath.getParent()) - .withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(dataFile.getFileId()) - .withInstantTime(dataFile.getCommitTime()).withStorage(storage).build(); + Writer logWriter = HoodieLogFormatWriter.builder() + .withParentPath(parquetFilePath.getParent()) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId(dataFile.getFileId()) + .withInstantTime(dataFile.getCommitTime()) + .withStorage(storage) + .build(); List records = SchemaTestUtil.generateTestRecords(logSchemaPath, dataPath).stream().map(HoodieAvroIndexedRecord::new).collect(Collectors.toList()); Map header = new HashMap<>(2); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, dataFile.getCommitTime()); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java index b9fbe74b6d029..24ee11c8ba1ec 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java @@ -38,6 +38,7 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; @@ -927,10 +928,10 @@ private HoodieLogFile prepareLogFile(String fileId, String baseInstantTime, String instantTime, boolean writeDataBlock) throws IOException, InterruptedException { - try (HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder() - .onParentPath(new StoragePath(tempDir.toString())) + try (HoodieLogFormat.Writer writer = HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(tempDir.toString())) .withFileExtension(HoodieLogFile.DELTA_EXTENSION) - .withFileId(fileId) + .withLogFileId(fileId) .withInstantTime(instantTime) .withStorage(storage) .withSizeThreshold(Long.MAX_VALUE).build()) { From 8d0358e760dfa4e57a380b7c8171c5f795b4b33a Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 4 Jun 2026 11:32:01 +0800 Subject: [PATCH 047/255] chore(docker): silence Dockerfile lint warnings across image set (#18664) Three classes of buildkit warnings are addressed in the Hudi docker tree: - LegacyKeyValueFormat: rewrite all `ENV KEY VALUE` to `ENV KEY=VALUE` across 13 Dockerfiles. Quoted values and ${VAR} expansion preserved. - UndefinedVar: re-declare HADOOP_DN_PORT, HADOOP_WEBHDFS_PORT, and HADOOP_HISTORY_PORT as ARG after the FROM line in datanode, namenode, and historyserver. Pre-FROM ARGs do not propagate into the build stage, so the corresponding ENV expansions were silently empty. - JSONArgsRecommended: change `CMD startup.sh` to `CMD ["startup.sh"]` in hive_base so SIGTERM is delivered to the script directly instead of through a /bin/sh -c wrapper. No runtime behavior change beyond the metastore/hiveserver now receiving shutdown signals correctly. (cherry picked from commit c40f765ce58d306ed299499781690f785c6bfdd9) --- docker/hoodie/hadoop/base/Dockerfile | 10 +++---- docker/hoodie/hadoop/base_java11/Dockerfile | 10 +++---- docker/hoodie/hadoop/base_java17/Dockerfile | 2 +- docker/hoodie/hadoop/datanode/Dockerfile | 3 ++- docker/hoodie/hadoop/historyserver/Dockerfile | 3 ++- docker/hoodie/hadoop/hive_base/Dockerfile | 14 +++++----- docker/hoodie/hadoop/namenode/Dockerfile | 3 ++- docker/hoodie/hadoop/prestobase/Dockerfile | 20 +++++++------- docker/hoodie/hadoop/spark_base/Dockerfile | 26 +++++++++---------- docker/hoodie/hadoop/sparkadhoc/Dockerfile | 10 +++---- docker/hoodie/hadoop/sparkmaster/Dockerfile | 6 ++--- docker/hoodie/hadoop/sparkworker/Dockerfile | 6 ++--- docker/hoodie/hadoop/trinobase/Dockerfile | 8 +++--- 13 files changed, 62 insertions(+), 59 deletions(-) diff --git a/docker/hoodie/hadoop/base/Dockerfile b/docker/hoodie/hadoop/base/Dockerfile index 546da57459d28..1a36fc344e36e 100644 --- a/docker/hoodie/hadoop/base/Dockerfile +++ b/docker/hoodie/hadoop/base/Dockerfile @@ -20,12 +20,12 @@ MAINTAINER Hoodie USER root # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 +ENV LANG=C.UTF-8 -ARG HADOOP_VERSION=3.3.4 +ARG HADOOP_VERSION=3.3.4 ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -ENV HADOOP_VERSION ${HADOOP_VERSION} -ENV HADOOP_URL ${HADOOP_URL} +ENV HADOOP_VERSION=${HADOOP_VERSION} +ENV HADOOP_URL=${HADOOP_URL} RUN set -x \ && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat procps \ @@ -46,7 +46,7 @@ ENV MULTIHOMED_NETWORK=1 ENV HADOOP_HOME=${HADOOP_PREFIX} ENV HADOOP_INSTALL=${HADOOP_HOME} ENV USER=root -ENV PATH /usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH +ENV PATH=/usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH # Exposing a union of ports across hadoop versions # Well known ports including ssh diff --git a/docker/hoodie/hadoop/base_java11/Dockerfile b/docker/hoodie/hadoop/base_java11/Dockerfile index 42333067b5698..121f94cd09eaa 100644 --- a/docker/hoodie/hadoop/base_java11/Dockerfile +++ b/docker/hoodie/hadoop/base_java11/Dockerfile @@ -20,12 +20,12 @@ LABEL maintainer="Hoodie" USER root # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 +ENV LANG=C.UTF-8 -ARG HADOOP_VERSION=2.8.4 +ARG HADOOP_VERSION=2.8.4 ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -ENV HADOOP_VERSION ${HADOOP_VERSION} -ENV HADOOP_URL ${HADOOP_URL} +ENV HADOOP_VERSION=${HADOOP_VERSION} +ENV HADOOP_URL=${HADOOP_URL} RUN set -x \ && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat procps \ @@ -45,7 +45,7 @@ ENV MULTIHOMED_NETWORK=1 ENV HADOOP_HOME=${HADOOP_PREFIX} ENV HADOOP_INSTALL=${HADOOP_HOME} ENV USER=root -ENV PATH /usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH +ENV PATH=/usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH # Exposing a union of ports across hadoop versions # Well known ports including ssh diff --git a/docker/hoodie/hadoop/base_java17/Dockerfile b/docker/hoodie/hadoop/base_java17/Dockerfile index a5c265d3aaace..6d513fd3eb077 100644 --- a/docker/hoodie/hadoop/base_java17/Dockerfile +++ b/docker/hoodie/hadoop/base_java17/Dockerfile @@ -36,7 +36,7 @@ LABEL maintainer="Hoodie" USER root # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 +ENV LANG=C.UTF-8 ARG HADOOP_VERSION=3.4.0 ENV HADOOP_VERSION=${HADOOP_VERSION} diff --git a/docker/hoodie/hadoop/datanode/Dockerfile b/docker/hoodie/hadoop/datanode/Dockerfile index bc157214f1825..b37a60fd1005e 100644 --- a/docker/hoodie/hadoop/datanode/Dockerfile +++ b/docker/hoodie/hadoop/datanode/Dockerfile @@ -20,7 +20,8 @@ ARG HADOOP_DN_PORT=50075 ARG BASE_IMAGE_TAG=java11 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest -ENV HADOOP_DN_PORT ${HADOOP_DN_PORT} +ARG HADOOP_DN_PORT +ENV HADOOP_DN_PORT=${HADOOP_DN_PORT} ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data RUN mkdir -p /hadoop/dfs/data diff --git a/docker/hoodie/hadoop/historyserver/Dockerfile b/docker/hoodie/hadoop/historyserver/Dockerfile index 0c77188e3e51c..a7700399f89dc 100644 --- a/docker/hoodie/hadoop/historyserver/Dockerfile +++ b/docker/hoodie/hadoop/historyserver/Dockerfile @@ -35,7 +35,8 @@ RUN wget https://repo1.maven.org/maven2/org/openlabtesting/leveldbjni/leveldbjni ENV LD_LIBRARY_PATH="/usr/lib" ENV JAVA_LIBRARY_PATH="/usr/lib" -ENV HADOOP_HISTORY_PORT ${HADOOP_HISTORY_PORT} +ARG HADOOP_HISTORY_PORT +ENV HADOOP_HISTORY_PORT=${HADOOP_HISTORY_PORT} ENV YARN_CONF_yarn_timeline___service_leveldb___timeline___store_path=/hadoop/yarn/timeline RUN mkdir -p /hadoop/yarn/timeline diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile index f77c4c4e455ea..b302e9002767c 100644 --- a/docker/hoodie/hadoop/hive_base/Dockerfile +++ b/docker/hoodie/hadoop/hive_base/Dockerfile @@ -19,16 +19,16 @@ ARG HADOOP_VERSION=3.3.4 ARG BASE_IMAGE_TAG=java11 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest -ENV HIVE_HOME /opt/hive -ENV PATH $HIVE_HOME/bin:$PATH -ENV HADOOP_HOME /opt/hadoop-$HADOOP_VERSION +ENV HIVE_HOME=/opt/hive +ENV PATH=$HIVE_HOME/bin:$PATH +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION WORKDIR /opt ARG HIVE_VERSION=3.1.3 ARG HIVE_URL=https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz -ENV HIVE_VERSION ${HIVE_VERSION} -ENV HIVE_URL ${HIVE_URL} +ENV HIVE_VERSION=${HIVE_VERSION} +ENV HIVE_URL=${HIVE_URL} #Install Hive MySQL, PostgreSQL JDBC RUN echo "Hive URL is :${HIVE_URL}" && wget ${HIVE_URL} -O hive.tar.gz && \ @@ -62,9 +62,9 @@ RUN chmod +x /usr/local/bin/startup.sh COPY entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/entrypoint.sh -ENV PATH $HIVE_HOME/bin/:$PATH +ENV PATH=$HIVE_HOME/bin/:$PATH # NOTE: This is the only battle-proven method to inject jars into Hive CLI ENV AUX_CLASSPATH=file://${HUDI_HADOOP_BUNDLE} ENTRYPOINT ["entrypoint.sh"] -CMD startup.sh +CMD ["startup.sh"] diff --git a/docker/hoodie/hadoop/namenode/Dockerfile b/docker/hoodie/hadoop/namenode/Dockerfile index 33e2ab4b9955c..402d7306297e9 100644 --- a/docker/hoodie/hadoop/namenode/Dockerfile +++ b/docker/hoodie/hadoop/namenode/Dockerfile @@ -20,7 +20,8 @@ ARG HADOOP_WEBHDFS_PORT=50070 ARG BASE_IMAGE_TAG=java11 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest -ENV HADOOP_WEBHDFS_PORT ${HADOOP_WEBHDFS_PORT} +ARG HADOOP_WEBHDFS_PORT +ENV HADOOP_WEBHDFS_PORT=${HADOOP_WEBHDFS_PORT} ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name RUN mkdir -p /hadoop/dfs/name diff --git a/docker/hoodie/hadoop/prestobase/Dockerfile b/docker/hoodie/hadoop/prestobase/Dockerfile index d40aa9c8f273e..7cc82d5421c43 100644 --- a/docker/hoodie/hadoop/prestobase/Dockerfile +++ b/docker/hoodie/hadoop/prestobase/Dockerfile @@ -25,15 +25,15 @@ FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest as h ARG PRESTO_VERSION=0.271 -ENV PRESTO_VERSION ${PRESTO_VERSION} -ENV PRESTO_HOME /opt/presto-server-${PRESTO_VERSION} -ENV PRESTO_CONF_DIR ${PRESTO_HOME}/etc -ENV PRESTO_LOG_DIR /var/log/presto -ENV PRESTO_JVM_MAX_HEAP 2G -ENV PRESTO_QUERY_MAX_MEMORY 1GB -ENV PRESTO_QUERY_MAX_MEMORY_PER_NODE 512MB -ENV PRESTO_DISCOVERY_URI http://presto-coordinator-1:8090 -ENV PATH $PATH:${PRESTO_HOME}/bin +ENV PRESTO_VERSION=${PRESTO_VERSION} +ENV PRESTO_HOME=/opt/presto-server-${PRESTO_VERSION} +ENV PRESTO_CONF_DIR=${PRESTO_HOME}/etc +ENV PRESTO_LOG_DIR=/var/log/presto +ENV PRESTO_JVM_MAX_HEAP=2G +ENV PRESTO_QUERY_MAX_MEMORY=1GB +ENV PRESTO_QUERY_MAX_MEMORY_PER_NODE=512MB +ENV PRESTO_DISCOVERY_URI=http://presto-coordinator-1:8090 +ENV PATH=$PATH:${PRESTO_HOME}/bin RUN set -x \ && DEBIAN_FRONTEND=noninteractive apt-get -yq update \ @@ -78,7 +78,7 @@ COPY lib/* /usr/local/lib/ RUN chmod +x /usr/local/bin/entrypoint.sh ADD target/ /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/ -ENV HUDI_PRESTO_BUNDLE /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/hudi-presto-bundle.jar +ENV HUDI_PRESTO_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/hudi-presto-bundle.jar RUN cp ${HUDI_PRESTO_BUNDLE} ${PRESTO_HOME}/plugin/hive-hadoop2/ # TODO: the latest master of Presto relies on hudi-presto-bundle, while current Presto releases # rely on hudi-common and hudi-hadoop-mr 0.9.0, which are pulled in plugin/hive-hadoop2/ in the diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile index 4210faf21bfe6..4a064971536e6 100644 --- a/docker/hoodie/hadoop/spark_base/Dockerfile +++ b/docker/hoodie/hadoop/spark_base/Dockerfile @@ -19,15 +19,15 @@ ARG HADOOP_VERSION=3.3.4 ARG HIVE_VERSION=3.1.3 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION} -ENV ENABLE_INIT_DAEMON true -ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon -ENV INIT_DAEMON_STEP spark_master_init +ENV ENABLE_INIT_DAEMON=true +ENV INIT_DAEMON_BASE_URI=http://identifier/init-daemon +ENV INIT_DAEMON_STEP=spark_master_init ARG SPARK_VERSION=3.5.3 ARG SPARK_HADOOP_VERSION=3 -ENV SPARK_VERSION ${SPARK_VERSION} -ENV HADOOP_VERSION ${SPARK_HADOOP_VERSION} +ENV SPARK_VERSION=${SPARK_VERSION} +ENV HADOOP_VERSION=${SPARK_HADOOP_VERSION} COPY wait-for-step.sh / COPY execute-step.sh / @@ -52,16 +52,16 @@ RUN chmod +x /wait-for-step.sh && chmod +x /execute-step.sh && chmod +x /finish- # Fix the value of PYTHONHASHSEED # Note: this is needed when you use Python 3.3 or greater -ENV PYTHONHASHSEED 1 +ENV PYTHONHASHSEED=1 -ENV SPARK_HOME /opt/spark -ENV SPARK_INSTALL ${SPARK_HOME} -ENV SPARK_CONF_DIR ${SPARK_HOME}/conf -ENV PATH $SPARK_INSTALL/bin:$PATH +ENV SPARK_HOME=/opt/spark +ENV SPARK_INSTALL=${SPARK_HOME} +ENV SPARK_CONF_DIR=${SPARK_HOME}/conf +ENV PATH=$SPARK_INSTALL/bin:$PATH -ENV SPARK_DRIVER_PORT 5001 -ENV SPARK_UI_PORT 5002 -ENV SPARK_BLOCKMGR_PORT 5003 +ENV SPARK_DRIVER_PORT=5001 +ENV SPARK_UI_PORT=5002 +ENV SPARK_BLOCKMGR_PORT=5003 EXPOSE $SPARK_DRIVER_PORT $SPARK_UI_PORT $SPARK_BLOCKMGR_PORT diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile b/docker/hoodie/hadoop/sparkadhoc/Dockerfile index 70105bf3cf362..7c8e900fca539 100644 --- a/docker/hoodie/hadoop/sparkadhoc/Dockerfile +++ b/docker/hoodie/hadoop/sparkadhoc/Dockerfile @@ -24,11 +24,11 @@ ARG PRESTO_VERSION=0.268 ARG TRINO_VERSION=368 COPY adhoc.sh /opt/spark -ENV SPARK_WORKER_WEBUI_PORT 8081 -ENV SPARK_WORKER_LOG /spark/logs -ENV SPARK_MASTER "spark://spark-master:7077" -ENV PRESTO_VERSION ${PRESTO_VERSION} -ENV TRINO_VERSION ${TRINO_VERSION} +ENV SPARK_WORKER_WEBUI_PORT=8081 +ENV SPARK_WORKER_LOG=/spark/logs +ENV SPARK_MASTER="spark://spark-master:7077" +ENV PRESTO_VERSION=${PRESTO_VERSION} +ENV TRINO_VERSION=${TRINO_VERSION} ENV BASE_URL=https://repo1.maven.org/maven2 ENV SPARK_BUNDLE_JAR=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile b/docker/hoodie/hadoop/sparkmaster/Dockerfile index 94898dcdcfb0a..9acaf027606dd 100644 --- a/docker/hoodie/hadoop/sparkmaster/Dockerfile +++ b/docker/hoodie/hadoop/sparkmaster/Dockerfile @@ -22,9 +22,9 @@ FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${S COPY master.sh /opt/spark -ENV SPARK_MASTER_PORT 7077 -ENV SPARK_MASTER_WEBUI_PORT 8080 -ENV SPARK_MASTER_LOG /opt/spark/logs +ENV SPARK_MASTER_PORT=7077 +ENV SPARK_MASTER_WEBUI_PORT=8080 +ENV SPARK_MASTER_LOG=/opt/spark/logs EXPOSE 8080 7077 6066 diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile b/docker/hoodie/hadoop/sparkworker/Dockerfile index c8515c735b4ca..5b03834067ec2 100644 --- a/docker/hoodie/hadoop/sparkworker/Dockerfile +++ b/docker/hoodie/hadoop/sparkworker/Dockerfile @@ -22,9 +22,9 @@ FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${S COPY worker.sh /opt/spark -ENV SPARK_WORKER_WEBUI_PORT 8081 -ENV SPARK_WORKER_LOG /spark/logs -ENV SPARK_MASTER "spark://spark-master:7077" +ENV SPARK_WORKER_WEBUI_PORT=8081 +ENV SPARK_WORKER_LOG=/spark/logs +ENV SPARK_MASTER="spark://spark-master:7077" EXPOSE 8081 diff --git a/docker/hoodie/hadoop/trinobase/Dockerfile b/docker/hoodie/hadoop/trinobase/Dockerfile index 0700fa2f6bfb5..a6f7701a4ef4a 100644 --- a/docker/hoodie/hadoop/trinobase/Dockerfile +++ b/docker/hoodie/hadoop/trinobase/Dockerfile @@ -41,8 +41,8 @@ RUN apt-get install -y \ uuid-runtime \ less -ENV JAVA_HOME /usr/java/default -ENV PATH $PATH:$JAVA_HOME/bin +ENV JAVA_HOME=/usr/java/default +ENV PATH=$PATH:$JAVA_HOME/bin WORKDIR /usr/local/bin RUN wget -q ${BASE_URL}/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar @@ -54,10 +54,10 @@ RUN wget -q ${BASE_URL}/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TR RUN tar xvzf trino-server-${TRINO_VERSION}.tar.gz -C /usr/local/ RUN ln -s /usr/local/trino-server-${TRINO_VERSION} $TRINO_HOME -ENV TRINO_BASE_WS /var/hoodie/ws/docker/hoodie/hadoop/trinobase +ENV TRINO_BASE_WS=/var/hoodie/ws/docker/hoodie/hadoop/trinobase RUN mkdir -p ${TRINO_BASE_WS}/target/ ADD target/ ${TRINO_BASE_WS}/target/ -ENV HUDI_TRINO_BUNDLE ${TRINO_BASE_WS}/target/hudi-trino-bundle.jar +ENV HUDI_TRINO_BUNDLE=${TRINO_BASE_WS}/target/hudi-trino-bundle.jar RUN cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/ ADD scripts ${TRINO_HOME}/scripts From d36f87f74ae3ea765274a4d77112abbd2f07622d Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Thu, 4 Jun 2026 17:03:18 +0800 Subject: [PATCH 048/255] refactor(core): Unify record key/index key splitting and extraction (#18842) (cherry picked from commit cb14ca7275833e9e76f1c3116ba96cf31af6f8e9) --- .../apache/hudi/config/HoodieIndexConfig.java | 11 ++++---- .../hudi/index/bucket/BucketIdentifier.java | 2 +- .../hudi/index/bucket/HoodieBucketIndex.java | 4 +-- .../hudi/keygen/CustomAvroKeyGenerator.java | 6 +---- .../org/apache/hudi/keygen/KeyGenUtils.java | 26 ++++++++++++------- .../BucketIndexBulkInsertPartitioner.java | 4 +-- .../client/TestBaseHoodieWriteClient.java | 5 ++-- .../hudi/config/TestHoodieWriteConfig.java | 20 ++++++++++++++ .../apache/hudi/keygen/TestKeyGenUtils.java | 21 +++++++++++++++ ...nsistentBucketDuplicateUpdateStrategy.java | 4 +-- .../hudi/keygen/CustomKeyGenerator.java | 8 +----- .../hudi/configuration/OptionsResolver.java | 13 +++------- .../ConsistentBucketAssignFunction.java | 2 +- .../ConsistentBucketStreamWriteFunction.java | 4 +-- .../apache/hudi/table/HoodieTableSource.java | 2 +- .../configuration/TestOptionsResolver.java | 4 +-- .../org/apache/hudi/BucketIndexSupport.scala | 4 +-- .../command/DeleteHoodieTableCommand.scala | 5 +++- .../command/DeleteHoodieTableCommand.scala | 5 +++- 19 files changed, 94 insertions(+), 56 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index 9cedac1be74d0..1d2c6bb2b426b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -29,6 +29,7 @@ import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bucket.partition.PartitionBucketIndexRule; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import lombok.Getter; @@ -39,9 +40,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; +import java.util.List; import java.util.Properties; -import java.util.stream.Collectors; import static org.apache.hudi.common.config.HoodieStorageConfig.BLOOM_FILTER_DYNAMIC_MAX_ENTRIES; import static org.apache.hudi.common.config.HoodieStorageConfig.BLOOM_FILTER_FPP_VALUE; @@ -777,10 +777,9 @@ private void validateBucketIndexConfig() { hoodieIndexConfig.setValue(BUCKET_INDEX_HASH_FIELD, hoodieIndexConfig.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME)); } else { - boolean valid = Arrays - .stream(hoodieIndexConfig.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME).split(",")) - .collect(Collectors.toSet()) - .containsAll(Arrays.asList(hoodieIndexConfig.getString(BUCKET_INDEX_HASH_FIELD).split(","))); + List recordKeyFields = KeyGenUtils.getRecordKeyFields(hoodieIndexConfig.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME)); + List indexKeyFields = KeyGenUtils.getIndexKeyFields(hoodieIndexConfig.getString(BUCKET_INDEX_HASH_FIELD)); + boolean valid = recordKeyFields.containsAll(indexKeyFields); if (!valid) { throw new HoodieIndexException("Bucket index key (if configured) must be subset of record key."); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/BucketIdentifier.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/BucketIdentifier.java index eed3ab39599c1..2bde3aec815b4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/BucketIdentifier.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/BucketIdentifier.java @@ -42,7 +42,7 @@ public static int getBucketId(List hashKeyFields, int numBuckets) { } protected static List getHashKeys(String recordKey, String indexKeyFields) { - return getHashKeysUsingIndexFields(recordKey, Arrays.asList(indexKeyFields.split(","))); + return getHashKeysUsingIndexFields(recordKey, KeyGenUtils.getIndexKeyFields(indexKeyFields)); } protected static List getHashKeys(String recordKey, List indexKeyFields) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java index 38c7cb5319a3f..7b9c28b462169 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java @@ -29,13 +29,13 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.table.HoodieTable; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.io.Serializable; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,7 +57,7 @@ public HoodieBucketIndex(HoodieWriteConfig config) { super(config); this.numBuckets = config.getBucketIndexNumBuckets(); - this.indexKeyFields = Arrays.asList(config.getBucketIndexHashField().split(",")); + this.indexKeyFields = KeyGenUtils.getIndexKeyFields(config.getBucketIndexHashField()); log.info("Use bucket index, numBuckets = " + numBuckets + ", indexFields: " + indexKeyFields); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java index beeb4947acd42..6ab6378367ae7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java @@ -62,11 +62,7 @@ public enum PartitionKeyType { public CustomAvroKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Option.ofNullable(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), null)) - .map(recordKeyConfigValue -> - Arrays.stream(recordKeyConfigValue.split(FIELD_SEPARATOR)) - .map(String::trim).collect(Collectors.toList()) - ).orElse(Collections.emptyList()); + this.recordKeyFields = KeyGenUtils.getRecordKeyFields(props); this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(FIELD_SEPARATOR)).map(String::trim).collect(Collectors.toList()); this.recordKeyGenerator = getRecordKeyFieldNames().size() == 1 ? new SimpleAvroKeyGenerator(config) : new ComplexAvroKeyGenerator(config); this.partitionKeyGenerators = getPartitionKeyGenerators(this.partitionPathFields, config); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java index 292f261c2223c..81faa0cdb2ba0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java @@ -39,11 +39,9 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.BiFunction; -import java.util.stream.Collectors; import static org.apache.hudi.config.HoodieWriteConfig.COMPLEX_KEYGEN_NEW_ENCODING; import static org.apache.hudi.config.HoodieWriteConfig.WRITE_TABLE_VERSION; @@ -72,7 +70,7 @@ public class KeyGenUtils { */ public static KeyGeneratorType inferKeyGeneratorType( Option recordsKeyFields, String partitionFields) { - int numRecordKeyFields = recordsKeyFields.map(fields -> fields.split(",").length).orElse(0); + int numRecordKeyFields = recordsKeyFields.map(keyStr -> getRecordKeyFields(keyStr).size()).orElse(0); KeyGeneratorType partitionKeyGeneratorType = inferKeyGeneratorTypeFromPartitionFields(partitionFields); if (numRecordKeyFields <= 1) { return partitionKeyGeneratorType; @@ -331,13 +329,21 @@ public static KeyGenerator createKeyGeneratorByClassName(TypedProperties props) } public static List getRecordKeyFields(TypedProperties props) { - return Option.ofNullable(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), null)) - .map(recordKeyConfigValue -> - Arrays.stream(recordKeyConfigValue.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList()) - ).orElse(Collections.emptyList()); + return getRecordKeyFields(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), null)); + } + + public static List getRecordKeyFields(String recordKeys) { + return getKeyFields(recordKeys); + } + + public static List getIndexKeyFields(String indexKeys) { + return getKeyFields(indexKeys); + } + + private static List getKeyFields(String keys) { + return Option.ofNullable(keys) + .map(value -> StringUtils.split(value, ",")) + .orElse(Collections.emptyList()); } /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java index 7d8d7c6900edd..5fd3909f05620 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java @@ -25,12 +25,12 @@ import org.apache.hudi.io.AppendHandleFactory; import org.apache.hudi.io.SingleFileHandleCreateFactory; import org.apache.hudi.io.WriteHandleFactory; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -50,7 +50,7 @@ public abstract class BucketIndexBulkInsertPartitioner extends BucketSortBulk public BucketIndexBulkInsertPartitioner(HoodieTable table, String sortString, boolean preserveHoodieMetadata) { super(table, sortString); - this.indexKeyFields = Arrays.asList(table.getConfig().getBucketIndexHashField().split(",")); + this.indexKeyFields = KeyGenUtils.getIndexKeyFields(table.getConfig().getBucketIndexHashField()); this.consistentLogicalTimestampEnabled = table.getConfig().isConsistentLogicalTimestampEnabled(); this.preserveHoodieMetadata = preserveHoodieMetadata; // Multiple bulk inserts into COW using `BucketIndexBulkInsertPartitioner` is restricted, otherwise AppendHandleFactory will produce MOR log files diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java index 377a54eedcd8a..9190dde5ab1e1 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java @@ -46,6 +46,7 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.simple.HoodieSimpleIndex; import org.apache.hudi.keygen.ComplexAvroKeyGenerator; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; @@ -242,7 +243,7 @@ void testWithComplexKeyGeneratorValidation(String keyGeneratorClass, if (tableVersion <= 8 && enableComplexKeyGeneratorValidation && (ComplexAvroKeyGenerator.class.getCanonicalName().equals(keyGeneratorClass) || "org.apache.hudi.keygen.ComplexKeyGenerator".equals(keyGeneratorClass)) - && recordKeyFields.split(",").length == 1) { + && KeyGenUtils.getRecordKeyFields(recordKeyFields).size() == 1) { assertComplexKeyGeneratorValidationThrows(() -> writeClient.initTable(WriteOperationType.INSERT, Option.empty()), "ingestion"); } else { writeClient.initTable(WriteOperationType.INSERT, Option.empty()); @@ -399,4 +400,4 @@ protected void updateColumnsToIndexWithColStats(HoodieTableMetaClient metaClient } } -} \ No newline at end of file +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 6c1d1418b1907..044e41ed89d2b 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.config.HoodieWriteConfig.Builder; +import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; @@ -619,6 +620,25 @@ public void testSimpleBucketIndexPartitionerConfig() { assertEquals("org.apache.hudi.table.action.commit.UpsertPartitioner", overwritePartitioner.getString(HoodieLayoutConfig.LAYOUT_PARTITIONER_CLASS_NAME)); } + @Test + public void testBucketIndexKeyFieldValidationTrimsFields() { + Properties props = new Properties(); + props.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "uuid, name"); + + HoodieIndexConfig indexConfig = HoodieIndexConfig.newBuilder() + .fromProperties(props) + .withIndexType(HoodieIndex.IndexType.BUCKET) + .withIndexKeyField(" name") + .build(); + assertEquals(" name", indexConfig.getString(HoodieIndexConfig.BUCKET_INDEX_HASH_FIELD)); + + assertThrows(HoodieIndexException.class, () -> HoodieIndexConfig.newBuilder() + .fromProperties(props) + .withIndexType(HoodieIndex.IndexType.BUCKET) + .withIndexKeyField("missing") + .build()); + } + @Test void testBloomIndexFileIdKeySortingConfig() { Properties props = new Properties(); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java index 777c4792901fb..cd837c0608e89 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java @@ -18,9 +18,11 @@ package org.apache.hudi.keygen; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieKeyException; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.avro.Schema; @@ -31,6 +33,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.apache.hudi.common.table.HoodieTableConfig.KEY_GENERATOR_TYPE; @@ -121,6 +124,24 @@ public void testInferKeyGeneratorTypeFromPartitionFields() { KeyGenUtils.inferKeyGeneratorTypeFromPartitionFields(null)); } + @Test + public void testGetRecordKeyFields() { + assertEquals(Collections.emptyList(), KeyGenUtils.getRecordKeyFields((String) null)); + assertEquals(Collections.emptyList(), KeyGenUtils.getRecordKeyFields("")); + assertEquals(Arrays.asList("id", "ts", "name"), KeyGenUtils.getRecordKeyFields(" id,ts, name,, ")); + + TypedProperties props = new TypedProperties(); + props.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), " id,ts "); + assertEquals(Arrays.asList("id", "ts"), KeyGenUtils.getRecordKeyFields(props)); + } + + @Test + public void testGetIndexKeyFields() { + assertEquals(Collections.emptyList(), KeyGenUtils.getIndexKeyFields(null)); + assertEquals(Collections.emptyList(), KeyGenUtils.getIndexKeyFields("")); + assertEquals(Arrays.asList("id", "ts", "name"), KeyGenUtils.getIndexKeyFields(" id,ts, name,, ")); + } + @Test public void testExtractRecordKeys() { // if for recordKey one column only is used, then there is no added column name before value diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java index 2d8247de92e77..eb4b308a668af 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java @@ -28,11 +28,11 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.index.bucket.ConsistentBucketIdentifier; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.cluster.strategy.UpdateStrategy; import org.apache.hudi.table.action.cluster.util.ConsistentHashingUpdateStrategyUtils; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -74,7 +74,7 @@ public Pair>, Set> handleUpdate(Ho ConsistentHashingUpdateStrategyUtils.constructPartitionToIdentifier(partitions, table); // Produce records tagged with new record location - List indexKeyFields = Arrays.asList(table.getConfig().getBucketIndexHashField().split(",")); + List indexKeyFields = KeyGenUtils.getIndexKeyFields(table.getConfig().getBucketIndexHashField()); HoodieData> redirectedRecordsRDD = filteredRecordsRDD.map(r -> { Pair identifierPair = partitionToIdentifier.get(r.getPartitionPath()); ConsistentHashingNode node = identifierPair.getValue().getBucket(r.getKey(), indexKeyFields); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 954af2000c1b1..45f77ff9ce289 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -63,12 +63,7 @@ public CustomKeyGenerator(TypedProperties props) { // NOTE: We have to strip partition-path configuration, since it could only be interpreted by // this key-gen super(stripPartitionPathConfig(props)); - this.recordKeyFields = Option.ofNullable(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), null)) - .map(recordKeyConfigValue -> - Arrays.stream(recordKeyConfigValue.split(",")) - .map(String::trim) - .collect(Collectors.toList()) - ).orElse(Collections.emptyList()); + this.recordKeyFields = KeyGenUtils.getRecordKeyFields(props); String partitionPathFields = props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()); this.partitionPathFields = partitionPathFields == null ? Collections.emptyList() @@ -167,4 +162,3 @@ private static TypedProperties stripPartitionPathConfig(TypedProperties props) { return filtered; } } - diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java index 74c5257cf44fb..cc32351e62cfc 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java @@ -40,6 +40,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bucket.partition.PartitionBucketIndexUtils; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.sink.buffer.BufferMemoryType; import org.apache.hudi.sink.overwrite.PartitionOverwriteMode; @@ -168,21 +169,15 @@ public static String getRecordKeyStr(Configuration conf) { */ public static String[] getRecordKeys(Configuration conf) { final String recordKeyStr = conf.get(FlinkOptions.RECORD_KEY_FIELD); - if (StringUtils.isNullOrEmpty(recordKeyStr)) { - return new String[]{}; - } - return recordKeyStr.split(","); + return KeyGenUtils.getRecordKeyFields(recordKeyStr).toArray(new String[0]); } /** * Return the bucket index keys as an array. */ public static String[] getBucketIndexKeys(Configuration conf) { - final String indexKeyStr = conf.get(FlinkOptions.INDEX_KEY_FIELD); - if (StringUtils.isNullOrEmpty(indexKeyStr)) { - return new String[]{}; - } - return indexKeyStr.split(","); + final String indexKeyStr = getIndexKeyField(conf); + return KeyGenUtils.getIndexKeyFields(indexKeyStr).toArray(new String[0]); } /** diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/ConsistentBucketAssignFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/ConsistentBucketAssignFunction.java index ec2fa104c58e8..089224ac985d1 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/ConsistentBucketAssignFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/ConsistentBucketAssignFunction.java @@ -68,7 +68,7 @@ public class ConsistentBucketAssignFunction extends ProcessFunctionAdapter indexKeyFields = Arrays.asList(config.get(FlinkOptions.INDEX_KEY_FIELD).split(",")); + List indexKeyFields = Arrays.asList(OptionsResolver.getBucketIndexKeys(config)); this.updateStrategy = new ConsistentBucketUpdateStrategy(this.writeClient, indexKeyFields); log.info("Create update strategy with index key fields: {}", indexKeyFields); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java index d1b13ff7e3754..b5fe77b8e0826 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java @@ -494,7 +494,7 @@ private Option> getDataBucketFunc(List indexKeyFields = Arrays.stream(OptionsResolver.getIndexKeyField(conf).split(",")).collect(Collectors.toSet()); + Set indexKeyFields = Arrays.stream(OptionsResolver.getBucketIndexKeys(conf)).collect(Collectors.toSet()); List indexKeyFilters = dataFilters.stream().filter(expr -> ExpressionUtils.isEqualsLitExpr(expr, indexKeyFields)).collect(Collectors.toList()); if (!ExpressionUtils.isFilteringByAllFields(indexKeyFilters, indexKeyFields)) { return Option.empty(); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java index aef93b7d10fba..0ed8e61fd3500 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java @@ -66,7 +66,7 @@ void testGetRecordKeys() { assertArrayEquals(new String[]{}, OptionsResolver.getRecordKeys(conf)); conf.set(FlinkOptions.RECORD_KEY_FIELD, "uuid, name"); - assertArrayEquals(new String[]{"uuid", " name"}, OptionsResolver.getRecordKeys(conf)); + assertArrayEquals(new String[]{"uuid", "name"}, OptionsResolver.getRecordKeys(conf)); } @Test @@ -78,7 +78,7 @@ void testGetBucketIndexKeys() { assertArrayEquals(new String[]{}, OptionsResolver.getBucketIndexKeys(conf)); conf.set(FlinkOptions.INDEX_KEY_FIELD, "uuid, name"); - assertArrayEquals(new String[]{"uuid", " name"}, OptionsResolver.getBucketIndexKeys(conf)); + assertArrayEquals(new String[]{"uuid", "name"}, OptionsResolver.getBucketIndexKeys(conf)); } @Test diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala index 71c0ac4d379a1..e9aee3ab78b48 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala @@ -27,6 +27,7 @@ import org.apache.hudi.index.HoodieIndex import org.apache.hudi.index.HoodieIndex.IndexType import org.apache.hudi.index.bucket.BucketIdentifier import org.apache.hudi.keygen.KeyGenerator +import org.apache.hudi.keygen.KeyGenUtils import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory import org.apache.avro.generic.GenericData @@ -212,7 +213,7 @@ class BucketIndexSupport(spark: SparkSession, if (bucketHashFields == null || bucketHashFields.isEmpty) { Option.apply(null) } else { - Option.apply(JavaConverters.seqAsJavaListConverter(bucketHashFields.split(",")).asJava) + Option.apply(KeyGenUtils.getIndexKeyFields(bucketHashFields)) } } @@ -224,4 +225,3 @@ class BucketIndexSupport(spark: SparkSession, object BucketIndexSupport { val INDEX_NAME = "BUCKET" } - diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala index b03b6fec3b8fc..bd4ca0c751e76 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.{HoodieSparkSqlWriter, SparkAdapterSupport} import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_OPTIMIZED_WRITES, SPARK_SQL_WRITES_PREPPED_KEY} import org.apache.hudi.common.table.HoodieTableConfig +import org.apache.hudi.keygen.KeyGenUtils import org.apache.spark.sql._ import org.apache.spark.sql.SparkSession @@ -34,6 +35,8 @@ import org.apache.spark.sql.hudi.ProvidesHoodieConfig import org.apache.spark.sql.hudi.command.HoodieCommandMetrics.updateCommitMetrics import org.apache.spark.sql.hudi.command.HoodieLeafRunnableCommand.stripMetaFieldAttributes +import scala.collection.JavaConverters._ + case class DeleteHoodieTableCommand(catalogTable: HoodieCatalogTable, query: LogicalPlan, config: Map[String, String]) extends DataWritingCommand with SparkAdapterSupport with ProvidesHoodieConfig { @@ -76,7 +79,7 @@ object DeleteHoodieTableCommand extends SparkAdapterSupport with ProvidesHoodieC } val recordKeysStr = config.getOrElse(HoodieTableConfig.RECORDKEY_FIELDS.key(), "") - val recordKeys = recordKeysStr.split(",").filter(_.nonEmpty) + val recordKeys = KeyGenUtils.getRecordKeyFields(recordKeysStr).asScala.toSeq // get all columns which are used in condition val conditionColumns = if (condition == null) { diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala index e40cd2e4840be..b05d826fbb1b6 100644 --- a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.{HoodieSparkSqlWriter, SparkAdapterSupport} import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_OPTIMIZED_WRITES, SPARK_SQL_WRITES_PREPPED_KEY} import org.apache.hudi.common.table.HoodieTableConfig +import org.apache.hudi.keygen.KeyGenUtils import org.apache.spark.sql import org.apache.spark.sql._ @@ -35,6 +36,8 @@ import org.apache.spark.sql.hudi.ProvidesHoodieConfig import org.apache.spark.sql.hudi.command.HoodieCommandMetrics.updateCommitMetrics import org.apache.spark.sql.hudi.command.HoodieLeafRunnableCommand.stripMetaFieldAttributes +import scala.collection.JavaConverters._ + case class DeleteHoodieTableCommand(catalogTable: HoodieCatalogTable, query: LogicalPlan, config: Map[String, String]) extends DataWritingCommand with SparkAdapterSupport with ProvidesHoodieConfig { @@ -81,7 +84,7 @@ object DeleteHoodieTableCommand extends SparkAdapterSupport with ProvidesHoodieC } val recordKeysStr = config.getOrElse(HoodieTableConfig.RECORDKEY_FIELDS.key(), "") - val recordKeys = recordKeysStr.split(",").filter(_.nonEmpty) + val recordKeys = KeyGenUtils.getRecordKeyFields(recordKeysStr).asScala.toSeq // get all columns which are used in condition val conditionColumns = if (condition == null) { From da2e4efa17d241b997628a54c254f656c41f045b Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 4 Jun 2026 14:42:57 -0700 Subject: [PATCH 049/255] fix(streamer): Use checkpoint V1 for non-incremental streamer sources (#18896) (cherry picked from commit 9cddb237be14242be4ba096413ab057942af79aa) --- .../table/checkpoint/CheckpointUtils.java | 32 +- .../table/checkpoint/TestCheckpointUtils.java | 66 +--- .../examples/common/RandomJsonSource.java | 4 +- .../examples/common/TestRandomJsonSource.java | 76 ++++ .../helpers/DFSTestSuitePathSelector.java | 7 +- .../hudi/streaming/HoodieStreamSourceV2.scala | 2 +- .../utilities/sources/GcsEventsSource.java | 12 +- .../utilities/sources/HiveIncrPullSource.java | 11 +- .../utilities/sources/HoodieIncrSource.java | 4 +- .../hudi/utilities/sources/InputBatch.java | 9 - .../hudi/utilities/sources/JdbcSource.java | 8 +- .../hudi/utilities/sources/KafkaSource.java | 7 +- .../hudi/utilities/sources/KinesisSource.java | 5 +- .../hudi/utilities/sources/PulsarSource.java | 4 +- .../apache/hudi/utilities/sources/Source.java | 53 +-- .../utilities/sources/SqlFileBasedSource.java | 4 +- .../sources/debezium/DebeziumSource.java | 5 +- .../sources/helpers/DFSPathSelector.java | 6 +- .../helpers/DatePartitionPathSelector.java | 6 +- .../sources/helpers/S3EventsMetaSelector.java | 9 +- .../hudi/utilities/streamer/StreamSync.java | 7 +- .../streamer/StreamerCheckpointUtils.java | 26 +- .../HoodieDeltaStreamerTestBase.java | 19 +- .../TestHoodieDeltaStreamer.java | 36 +- .../TestSourceFormatAdapter.java | 6 +- .../utilities/sources/TestDataSource.java | 4 +- .../sources/TestGcsEventsSource.java | 3 +- .../sources/TestHiveIncrPullSource.java | 99 +++++ .../utilities/sources/TestInputBatch.java | 6 +- .../utilities/sources/TestJdbcSource.java | 5 +- .../TestStreamerSourceCheckpointVersion.java | 351 ++++++++++++++++++ .../TestDFSPathSelectorCommonMethods.java | 15 + .../streamer/TestHoodieIncrSourceE2E.java | 9 +- .../TestParquetDfsCheckpointFormatOnV6.java | 128 +++++++ .../utilities/streamer/TestStreamSync.java | 31 +- .../streamer/TestStreamerCheckpointUtils.java | 52 +++ .../sources/DistributedTestDataSource.java | 6 +- .../checkpoint-v6/parquet-dfs-v1-fixture.zip | Bin 0 -> 103450 bytes 38 files changed, 917 insertions(+), 216 deletions(-) create mode 100644 hudi-examples/hudi-examples-spark/src/test/java/org/apache/hudi/examples/common/TestRandomJsonSource.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHiveIncrPullSource.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestParquetDfsCheckpointFormatOnV6.java create mode 100644 hudi-utilities/src/test/resources/checkpoint-v6/parquet-dfs-v1-fixture.zip diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/CheckpointUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/CheckpointUtils.java index 15084a74ae46d..443383c77297a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/CheckpointUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/CheckpointUtils.java @@ -21,7 +21,6 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineUtils; @@ -71,23 +70,24 @@ public static Checkpoint getCheckpoint(HoodieCommitMetadata commitMetadata) { throw new HoodieException("Checkpoint is not found in the commit metadata: " + commitMetadata.getExtraMetadata()); } - public static Checkpoint buildCheckpointFromGeneralSource( - String sourceClassName, int writeTableVersion, String checkpointToResume) { - return CheckpointUtils.shouldTargetCheckpointV2(writeTableVersion, sourceClassName) - ? new StreamerCheckpointV2(checkpointToResume) : new StreamerCheckpointV1(checkpointToResume); + /** + * For sources that do not have a semantic change in the checkpoint, always use checkpoint V1. + * + * @param checkpointToResume value of the checkpoint to resume + * @return {@link Checkpoint} instance + */ + public static Checkpoint createCheckpoint(String checkpointToResume) { + return new StreamerCheckpointV1(checkpointToResume); } - // Whenever we create checkpoint from streamer config checkpoint override, we should use this function - // to build checkpoints. - public static Checkpoint buildCheckpointFromConfigOverride( - String sourceClassName, int writeTableVersion, String checkpointToResume) { - return CheckpointUtils.shouldTargetCheckpointV2(writeTableVersion, sourceClassName) - ? new UnresolvedStreamerCheckpointBasedOnCfg(checkpointToResume) : new StreamerCheckpointV1(checkpointToResume); - } - - public static boolean shouldTargetCheckpointV2(int writeTableVersion, String sourceClassName) { - return writeTableVersion >= HoodieTableVersion.EIGHT.versionCode() - && !DATASOURCES_NOT_SUPPORTED_WITH_CKPT_V2.contains(sourceClassName); + /** + * For sources that do not have a semantic change in the checkpoint, always use checkpoint V1. + * + * @param checkpointToResume the checkpoint to resume + * @return {@link Checkpoint} instance + */ + public static Checkpoint createCheckpoint(Checkpoint checkpointToResume) { + return new StreamerCheckpointV1(checkpointToResume); } // TODO(yihua): for checkpoint translation, handle cases where the checkpoint is not exactly the diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/checkpoint/TestCheckpointUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/table/checkpoint/TestCheckpointUtils.java index a4b1be058799a..876c8a53162a6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/checkpoint/TestCheckpointUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/checkpoint/TestCheckpointUtils.java @@ -21,7 +21,6 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -31,8 +30,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import java.util.stream.Stream; @@ -56,7 +53,6 @@ public class TestCheckpointUtils { private HoodieActiveTimeline activeTimeline; private static final String CHECKPOINT_TO_RESUME = "20240101000000"; - private static final String GENERAL_SOURCE = "org.apache.hudi.utilities.sources.GeneralSource"; @BeforeEach public void setUp() { @@ -197,64 +193,10 @@ public void testConvertCheckpointWithUseTransitionTime() { assertEquals(completionTime, translatedCheckpoint.getCheckpointKey()); } - @ParameterizedTest - @CsvSource({ - // version, sourceClassName, expectedResult - // Version >= 8 with allowed sources should return true - "8, org.apache.hudi.utilities.sources.TestSource, true", - "9, org.apache.hudi.utilities.sources.AnotherSource, true", - // Version < 8 should return false regardless of source - "7, org.apache.hudi.utilities.sources.TestSource, false", - "6, org.apache.hudi.utilities.sources.AnotherSource, false", - // Disallowed sources should return false even with version >= 8 - "8, org.apache.hudi.utilities.sources.S3EventsHoodieIncrSource, false", - "8, org.apache.hudi.utilities.sources.GcsEventsHoodieIncrSource, false", - "8, org.apache.hudi.utilities.sources.MockS3EventsHoodieIncrSource, false", - "8, org.apache.hudi.utilities.sources.MockGcsEventsHoodieIncrSource, false" - }) - public void testTargetCheckpointV2(int version, String sourceClassName, boolean isV2Checkpoint) { - assertEquals(isV2Checkpoint, CheckpointUtils.buildCheckpointFromGeneralSource(sourceClassName, version, "ignored") instanceof StreamerCheckpointV2); - } - - @Test - public void testBuildCheckpointFromGeneralSource() { - // Test V2 checkpoint creation (newer table version + general source) - Checkpoint checkpoint1 = CheckpointUtils.buildCheckpointFromGeneralSource( - GENERAL_SOURCE, - HoodieTableVersion.EIGHT.versionCode(), - CHECKPOINT_TO_RESUME - ); - assertInstanceOf(StreamerCheckpointV2.class, checkpoint1); - assertEquals(CHECKPOINT_TO_RESUME, checkpoint1.getCheckpointKey()); - - // Test V1 checkpoint creation (older table version) - Checkpoint checkpoint2 = CheckpointUtils.buildCheckpointFromGeneralSource( - GENERAL_SOURCE, - HoodieTableVersion.SEVEN.versionCode(), - CHECKPOINT_TO_RESUME - ); - assertInstanceOf(StreamerCheckpointV1.class, checkpoint2); - assertEquals(CHECKPOINT_TO_RESUME, checkpoint2.getCheckpointKey()); - } - @Test - public void testBuildCheckpointFromConfigOverride() { - // Test checkpoint from config creation (newer table version + general source) - Checkpoint checkpoint1 = CheckpointUtils.buildCheckpointFromConfigOverride( - GENERAL_SOURCE, - HoodieTableVersion.EIGHT.versionCode(), - CHECKPOINT_TO_RESUME - ); - assertInstanceOf(UnresolvedStreamerCheckpointBasedOnCfg.class, checkpoint1); - assertEquals(CHECKPOINT_TO_RESUME, checkpoint1.getCheckpointKey()); - - // Test V1 checkpoint creation (older table version) - Checkpoint checkpoint2 = CheckpointUtils.buildCheckpointFromConfigOverride( - GENERAL_SOURCE, - HoodieTableVersion.SEVEN.versionCode(), - CHECKPOINT_TO_RESUME - ); - assertInstanceOf(StreamerCheckpointV1.class, checkpoint2); - assertEquals(CHECKPOINT_TO_RESUME, checkpoint2.getCheckpointKey()); + void testCreateCheckpoint() { + Checkpoint checkpoint = CheckpointUtils.createCheckpoint(CHECKPOINT_TO_RESUME); + assertInstanceOf(StreamerCheckpointV1.class, checkpoint); + assertEquals(CHECKPOINT_TO_RESUME, checkpoint.getCheckpointKey()); } } diff --git a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/RandomJsonSource.java b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/RandomJsonSource.java index c8fcd6b04e2a3..97c0a5ec4fa7d 100644 --- a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/RandomJsonSource.java +++ b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/RandomJsonSource.java @@ -37,6 +37,8 @@ import java.util.List; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; + public class RandomJsonSource extends JsonSource { private final HoodieExampleDataGenerator dataGen; private final TimeGenerator timeGenerator; @@ -54,6 +56,6 @@ protected InputBatch> readFromCheckpoint(Option last String commitTime = TimelineUtils.generateInstantTime(true, timeGenerator); List inserts = dataGen.convertToStringList(dataGen.generateInserts(commitTime, 20)); - return new InputBatch<>(Option.of(sparkContext.parallelize(inserts, 1)), commitTime); + return new InputBatch<>(Option.of(sparkContext.parallelize(inserts, 1)), createCheckpoint(commitTime)); } } diff --git a/hudi-examples/hudi-examples-spark/src/test/java/org/apache/hudi/examples/common/TestRandomJsonSource.java b/hudi-examples/hudi-examples-spark/src/test/java/org/apache/hudi/examples/common/TestRandomJsonSource.java new file mode 100644 index 0000000000000..0691834abcd30 --- /dev/null +++ b/hudi-examples/hudi-examples-spark/src/test/java/org/apache/hudi/examples/common/TestRandomJsonSource.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.examples.common; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.utilities.sources.InputBatch; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.apache.hudi.config.HoodieWriteConfig.WRITE_TABLE_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for RandomJsonSource. + */ +class TestRandomJsonSource { + + private static JavaSparkContext jsc; + private static SparkSession spark; + + @BeforeAll + static void initSpark() { + spark = SparkSession.builder().master("local[1]").appName("TestRandomJsonSource").getOrCreate(); + jsc = new JavaSparkContext(spark.sparkContext()); + } + + @AfterAll + static void stopSpark() { + if (jsc != null) { + jsc.stop(); + } + if (spark != null) { + spark.stop(); + } + } + + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testFetchNextReturnsTwentyRecordsWithCorrectCheckpointVersion(int writeTableVersion) { + TypedProperties props = new TypedProperties(); + props.setProperty(WRITE_TABLE_VERSION.key(), String.valueOf(writeTableVersion)); + + RandomJsonSource source = new RandomJsonSource(props, jsc, spark, null); + InputBatch> batch = source.fetchNext(Option.empty(), Long.MAX_VALUE); + + assertNotNull(batch.getBatch()); + assertEquals(20, batch.getBatch().get().count()); + assertNotNull(batch.getCheckpointForNextBatch()); + assertEquals(StreamerCheckpointV1.class, batch.getCheckpointForNextBatch().getClass()); + } +} diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java index 2b8d95cc7df37..c4f85c6d98dcf 100644 --- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java +++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java @@ -20,7 +20,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; @@ -41,6 +40,7 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; /** @@ -99,14 +99,13 @@ public Pair, Checkpoint> getNextFilePathsAndMaxModificationTime( // no data to readAvro if (eligibleFiles.size() == 0) { return new ImmutablePair<>(Option.empty(), - lastCheckpoint.orElseGet(() -> new StreamerCheckpointV2(String.valueOf(Long.MIN_VALUE)))); + lastCheckpoint.orElseGet(() -> createCheckpoint(String.valueOf(Long.MIN_VALUE)))); } // readAvro the files out. String pathStr = eligibleFiles.stream().map(f -> f.getPath().toString()) .collect(Collectors.joining(",")); - return new ImmutablePair<>(Option.ofNullable(pathStr), - new StreamerCheckpointV2(String.valueOf(nextBatchId))); + return new ImmutablePair<>(Option.ofNullable(pathStr), createCheckpoint(String.valueOf(nextBatchId))); } catch (IOException ioe) { throw new HoodieIOException( "Unable to readAvro from source from checkpoint: " + lastCheckpoint, ioe); diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala index d104108f6c0cf..11a44f58a0b63 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala @@ -184,7 +184,7 @@ class HoodieStreamSourceV2(sqlContext: SQLContext, } private def translateCheckpoint(commitTime: String): String = { - if (CheckpointUtils.shouldTargetCheckpointV2(writeTableVersion.versionCode(), getClass.getName)) { + if (writeTableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT)) { commitTime } else { CheckpointUtils.convertToCheckpointV1ForCommitTime( diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java index 2625c7d1da492..a7e699db4e905 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java @@ -20,7 +20,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; @@ -45,6 +44,7 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getIntWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; @@ -112,7 +112,7 @@ public class GcsEventsSource extends RowSource { private final List messagesToAck = new ArrayList<>(); - private static final Checkpoint CHECKPOINT_VALUE_ZERO = new StreamerCheckpointV2("0"); + private static final String CHECKPOINT_VALUE_ZERO = "0"; public GcsEventsSource(TypedProperties props, JavaSparkContext jsc, SparkSession spark, SchemaProvider schemaProvider) { @@ -153,7 +153,7 @@ protected Pair>, Checkpoint> fetchNextBatch(Option>, Checkpoint> fetchNextBatch(Option findCommitToPull(Option latestTargetCommi if (!latestTargetCommit.isPresent()) { // start from the beginning - return Option.of(new StreamerCheckpointV2(commitTimes.get(0))); + return Option.of(createCheckpoint(commitTimes.get(0))); } for (String instantTime : commitTimes) { // TODO(vc): Add an option to delete consumed commits if (instantTime.compareTo(latestTargetCommit.get().getCheckpointKey()) > 0) { - return Option.of(new StreamerCheckpointV2(instantTime)); + return Option.of(createCheckpoint(instantTime)); } } return Option.empty(); @@ -123,7 +123,8 @@ protected InputBatch> readFromCheckpoint(Option commitToPull = findCommitToPull(lastCheckpoint); if (!commitToPull.isPresent()) { - return new InputBatch<>(Option.empty(), lastCheckpoint.isPresent() ? lastCheckpoint.get() : new StreamerCheckpointV2("")); + return new InputBatch<>(Option.empty(), + lastCheckpoint.isPresent() ? createCheckpoint(lastCheckpoint.get()) : createCheckpoint("")); } // read the files out. @@ -133,7 +134,7 @@ protected InputBatch> readFromCheckpoint(Option(Option.of(avroRDD.keys().map(r -> ((GenericRecord) r.datum()))), - String.valueOf(commitToPull.get())); + createCheckpoint(String.valueOf(commitToPull.get()))); } catch (Exception e) { throw new HoodieReadFromSourceException("Unable to read from source from checkpoint: " + lastCheckpoint, e); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java index c09196d37c480..937219d1721c0 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java @@ -25,7 +25,6 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.CheckpointUtils; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.table.checkpoint.UnresolvedStreamerCheckpointBasedOnCfg; @@ -49,6 +48,7 @@ import org.apache.hudi.utilities.streamer.SourceProfile; import org.apache.hudi.utilities.streamer.SourceProfileSupplier; import org.apache.hudi.utilities.streamer.StreamContext; +import org.apache.hudi.utilities.streamer.StreamerCheckpointUtils; import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; @@ -203,7 +203,7 @@ public Pair>, Checkpoint> fetchNextBatch(Option String srcPath = getStringWithAltKeys(props, HoodieIncrSourceConfig.HOODIE_SRC_BASE_PATH); HoodieTableVersion sourceTableVersion = HoodieTableConfig.loadFromHoodieProps( HoodieStorageUtils.getStorage(srcPath, HadoopFSUtils.getStorageConf(sparkContext.hadoopConfiguration())), srcPath).getTableVersion(); - if (sourceTableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) && CheckpointUtils.shouldTargetCheckpointV2(writeTableVersion, getClass().getName())) { + if (sourceTableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) && StreamerCheckpointUtils.shouldTargetCheckpointV2(writeTableVersion, getClass().getName())) { return fetchNextBatchBasedOnCompletionTime(lastCheckpoint, sourceLimit); } else { return fetchNextBatchBasedOnRequestedTime(lastCheckpoint, sourceLimit); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java index 9717ba726866b..94b5c22afe7e6 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java @@ -21,7 +21,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.utilities.schema.SchemaProvider; @@ -40,14 +39,6 @@ public class InputBatch { @Getter(AccessLevel.NONE) private final SchemaProvider schemaProvider; - public InputBatch(Option batch, String checkpointForNextBatch, SchemaProvider schemaProvider) { - this(batch, new StreamerCheckpointV2(checkpointForNextBatch), schemaProvider); - } - - public InputBatch(Option batch, String checkpointForNextBatch) { - this(batch, checkpointForNextBatch, null); - } - public InputBatch(Option batch, Checkpoint checkpointForNextBatch) { this(batch, checkpointForNextBatch, null); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JdbcSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JdbcSource.java index b236b9587a4bc..0f5cd6fa68e9b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JdbcSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JdbcSource.java @@ -21,7 +21,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; @@ -52,6 +51,7 @@ import java.util.List; import java.util.Set; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.checkRequiredConfigProperties; import static org.apache.hudi.common.util.ConfigUtils.containsConfigProperty; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; @@ -263,12 +263,12 @@ private Checkpoint checkpoint(Dataset rowDataset, boolean isIncremental, Op final String max = rowDataset.agg(functions.max(incrementalColumn).cast(DataTypes.StringType)).first().getString(0); log.info("Checkpointing column {} with value: {}", incrementalColumn, max); if (max != null) { - return new StreamerCheckpointV2(max); + return createCheckpoint(max); } return lastCheckpoint.isPresent() && !StringUtils.isNullOrEmpty(lastCheckpoint.get().getCheckpointKey()) - ? lastCheckpoint.get() : new StreamerCheckpointV2(StringUtils.EMPTY_STRING); + ? createCheckpoint(lastCheckpoint.get()) : createCheckpoint(StringUtils.EMPTY_STRING); } else { - return new StreamerCheckpointV2(StringUtils.EMPTY_STRING); + return createCheckpoint(StringUtils.EMPTY_STRING); } } catch (Exception e) { log.error("Failed to checkpoint"); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java index 943be5f30a9dc..215a9c5b88333 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.Map; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getLongWithAltKeys; @@ -130,11 +131,13 @@ private InputBatch toInputBatch(OffsetRange[] offsetRanges) { totalNewMsgs, offsetGen.getTopicName(), Arrays.toString(offsetRanges)); if (totalNewMsgs <= 0) { metrics.updateStreamerSourceNewMessageCount(METRIC_NAME_KAFKA_MESSAGE_IN_COUNT, 0); - return new InputBatch<>(Option.empty(), KafkaOffsetGen.CheckpointUtils.offsetsToStr(offsetRanges)); + return new InputBatch<>( + Option.empty(), createCheckpoint(KafkaOffsetGen.CheckpointUtils.offsetsToStr(offsetRanges))); } metrics.updateStreamerSourceNewMessageCount(METRIC_NAME_KAFKA_MESSAGE_IN_COUNT, totalNewMsgs); T newBatch = toBatch(offsetRanges); - return new InputBatch<>(Option.of(newBatch), KafkaOffsetGen.CheckpointUtils.offsetsToStr(offsetRanges)); + return new InputBatch<>( + Option.of(newBatch), createCheckpoint(KafkaOffsetGen.CheckpointUtils.offsetsToStr(offsetRanges))); } /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java index a5ec35994d8e0..91b52567fb077 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java @@ -51,6 +51,7 @@ import java.util.NoSuchElementException; import java.util.concurrent.ThreadLocalRandom; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; @Slf4j @@ -97,7 +98,7 @@ protected InputBatch readFromCheckpoint(Option lastCheckpoint, lo if (shardRangesWithUnreadRecords.length == 0) { metrics.updateStreamerSourceNewMessageCount(METRIC_NAME_KINESIS_MESSAGE_IN_COUNT, 0); String checkpointStr = lastCheckpoint.isPresent() ? lastCheckpoint.get().getCheckpointKey() : ""; - return new InputBatch<>(Option.empty(), checkpointStr); + return new InputBatch<>(Option.empty(), createCheckpoint(checkpointStr)); } // STEP 3: Otherwise, do the read. T batch = toBatch(shardRangesWithUnreadRecords, sourceLimit); @@ -111,7 +112,7 @@ protected InputBatch readFromCheckpoint(Option lastCheckpoint, lo log.info("Read {} records from Kinesis stream {} with {} shards, checkpoint: {}", totalMsgs, offsetGen.getStreamName(), shardRangesWithUnreadRecords.length, checkpointStr); - return new InputBatch<>(Option.of(batch), checkpointStr); + return new InputBatch<>(Option.of(batch), createCheckpoint(checkpointStr)); } /** Upper bound on consecutive empty GetRecords responses before giving up on a shard. */ diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/PulsarSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/PulsarSource.java index d7696efdbf6c9..cf6cd4093438d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/PulsarSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/PulsarSource.java @@ -21,7 +21,6 @@ import org.apache.hudi.HoodieConversionUtils; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; @@ -54,6 +53,7 @@ import java.util.Collections; import java.util.concurrent.TimeUnit; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.checkRequiredConfigProperties; import static org.apache.hudi.common.util.ConfigUtils.getLongWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; @@ -130,7 +130,7 @@ protected Pair>, Checkpoint> fetchNextBatch(Option readFromCheckpoint(Option lastCheckpoint, lo * After the checkpoint value is decided based on the existing configurations at * org.apache.hudi.utilities.streamer.StreamerCheckpointUtils#resolveWhatCheckpointToResume, * - * For most of the data sources the there is no difference between checkpoint V1 and V2, it's - * merely changing the wrapper class. + * Non-incremental sources always operate on V1 checkpoints regardless of the write table version, + * so any V2 input read from older commit metadata is normalized to V1 here. * - * Check child class method overrides to see special case handling. + * Hudi incremental sources have their own V1/V2 semantics (requested time vs completion time) + * and override this method. * */ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING) protected Option translateCheckpoint(Option lastCheckpoint) { if (lastCheckpoint.isEmpty()) { return Option.empty(); } - if (CheckpointUtils.shouldTargetCheckpointV2(writeTableVersion, getClass().getName())) { - // V2 -> V2 - if (lastCheckpoint.get() instanceof StreamerCheckpointV2) { - return lastCheckpoint; - } - // V1 -> V2 - if (lastCheckpoint.get() instanceof StreamerCheckpointV1) { - StreamerCheckpointV2 newCheckpoint = new StreamerCheckpointV2(lastCheckpoint.get()); - newCheckpoint.addV1Props(); - return Option.of(newCheckpoint); - } - } else { - // V2 -> V1 - if (lastCheckpoint.get() instanceof StreamerCheckpointV2) { - return Option.of(new StreamerCheckpointV1(lastCheckpoint.get())); - } - // V1 -> V1 - if (lastCheckpoint.get() instanceof StreamerCheckpointV1) { - return lastCheckpoint; - } + if (lastCheckpoint.get() instanceof StreamerCheckpointV1) { + return lastCheckpoint; } - throw new UnsupportedOperationException("Unsupported checkpoint type: " + lastCheckpoint.get()); - } - - public void assertCheckpointVersion(Option lastCheckpoint, Option lastCheckpointTranslated, Checkpoint checkpoint) { - if (checkpoint != null) { - boolean shouldBeV2Checkpoint = shouldTargetCheckpointV2(writeTableVersion, getClass().getName()); - String errorMessage = String.format( - "Data source should return checkpoint version V%s. The checkpoint resumed in the iteration is %s, whose translated version is %s. " - + "The checkpoint returned after the iteration %s.", - shouldBeV2Checkpoint ? "2" : "1", - lastCheckpoint.isEmpty() ? "null" : lastCheckpointTranslated.get(), - lastCheckpointTranslated.isEmpty() ? "null" : lastCheckpointTranslated.get(), - checkpoint); - if (shouldBeV2Checkpoint && !(checkpoint instanceof StreamerCheckpointV2)) { - throw new IllegalStateException(errorMessage); - } - if (!shouldBeV2Checkpoint && !(checkpoint instanceof StreamerCheckpointV1)) { - throw new IllegalStateException(errorMessage); - } + if (lastCheckpoint.get() instanceof StreamerCheckpointV2) { + return Option.of(new StreamerCheckpointV1(lastCheckpoint.get())); } + throw new UnsupportedOperationException("Unsupported checkpoint type: " + lastCheckpoint.get()); } /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java index 60cac125a0293..9138064203233 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java @@ -20,7 +20,6 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; @@ -39,6 +38,7 @@ import java.util.Collections; import java.util.Scanner; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.checkRequiredConfigProperties; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; @@ -92,7 +92,7 @@ protected Pair>, Checkpoint> fetchNextBatch( rows = sparkSession.sql(sqlStr); } } - return Pair.of(Option.of(rows), shouldEmitCheckPoint ? new StreamerCheckpointV2(String.valueOf(System.currentTimeMillis())) : null); + return Pair.of(Option.of(rows), shouldEmitCheckPoint ? createCheckpoint(String.valueOf(System.currentTimeMillis())) : null); } catch (IOException ioe) { throw new HoodieIOException("Error reading source SQL file.", ioe); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/debezium/DebeziumSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/debezium/DebeziumSource.java index 030f5e0e671b7..d45cac3922ff9 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/debezium/DebeziumSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/debezium/DebeziumSource.java @@ -21,7 +21,6 @@ import org.apache.hudi.AvroConversionUtils; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.utilities.config.HoodieSchemaProviderConfig; @@ -59,6 +58,7 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; import static org.apache.hudi.utilities.config.KafkaSourceConfig.KAFKA_AVRO_VALUE_DESERIALIZER_CLASS; @@ -128,7 +128,8 @@ protected Pair>, Checkpoint> fetchNextBatch(Option, Checkpoint> getNextFilePathsAndMaxModificationTime(O // no data to read if (filteredFiles.isEmpty()) { - return new ImmutablePair<>(Option.empty(), new StreamerCheckpointV2(String.valueOf(newCheckpointTime))); + return new ImmutablePair<>(Option.empty(), createCheckpoint(String.valueOf(newCheckpointTime))); } // read the files out. String pathStr = filteredFiles.stream().map(f -> f.getPath().toString()).collect(Collectors.joining(",")); - return new ImmutablePair<>(Option.ofNullable(pathStr), new StreamerCheckpointV2(String.valueOf(newCheckpointTime))); + return new ImmutablePair<>(Option.ofNullable(pathStr), createCheckpoint(String.valueOf(newCheckpointTime))); } catch (IOException ioe) { throw new HoodieIOException("Unable to read from source from checkpoint: " + lastCheckpointStr, ioe); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java index 989be9163da15..0892ca4d35648 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java @@ -21,7 +21,6 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ImmutablePair; @@ -44,6 +43,7 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getIntWithAltKeys; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; import static org.apache.hudi.utilities.config.DFSPathSelectorConfig.ROOT_INPUT_PATH; @@ -159,13 +159,13 @@ public Pair, Checkpoint> getNextFilePathsAndMaxModificationTime(J // no data to read if (filteredFiles.isEmpty()) { - return new ImmutablePair<>(Option.empty(), new StreamerCheckpointV2(String.valueOf(newCheckpointTime))); + return new ImmutablePair<>(Option.empty(), createCheckpoint(String.valueOf(newCheckpointTime))); } // read the files out. String pathStr = filteredFiles.stream().map(f -> f.getPath().toString()).collect(Collectors.joining(",")); - return new ImmutablePair<>(Option.ofNullable(pathStr), new StreamerCheckpointV2(String.valueOf(newCheckpointTime))); + return new ImmutablePair<>(Option.ofNullable(pathStr), createCheckpoint(String.valueOf(newCheckpointTime))); } /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java index 3cd073e721095..b5ccaa58f374a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java @@ -20,7 +20,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.table.checkpoint.CheckpointUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.collection.ImmutablePair; @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys; /** @@ -155,8 +156,10 @@ public Pair, Checkpoint> getNextEventsFromQueue(SqsClient sqs, for (Map eventRecord : eventRecords) { filteredEventRecords.add(SdkHttpUtils.urlDecode(MAPPER.writeValueAsString(eventRecord))); } - // Return the old checkpoint if no messages to consume from queue. - Checkpoint newCheckpoint = newCheckpointTime == 0 ? lastCheckpoint.orElse(null) : new StreamerCheckpointV2(String.valueOf(newCheckpointTime)); + // Re-wrap a prior V2 checkpoint as V1 to avoid leaking it back to commit metadata. + Checkpoint newCheckpoint = newCheckpointTime == 0 + ? lastCheckpoint.map(CheckpointUtils::createCheckpoint).orElse(null) + : createCheckpoint(String.valueOf(newCheckpointTime)); return new ImmutablePair<>(filteredEventRecords, newCheckpoint); } catch (JSONException | IOException e) { throw new HoodieException("Unable to read from SQS: ", e); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 1af9c503b6130..7f38d80f83080 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -55,6 +55,7 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -153,7 +154,7 @@ import static org.apache.hudi.common.table.HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE; import static org.apache.hudi.common.table.HoodieTableConfig.TIMELINE_HISTORY_PATH; import static org.apache.hudi.common.table.HoodieTableConfig.URL_ENCODE_PARTITIONING; -import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.buildCheckpointFromGeneralSource; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys; import static org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE; import static org.apache.hudi.config.HoodieClusteringConfig.INLINE_CLUSTERING; @@ -173,6 +174,7 @@ import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; import static org.apache.hudi.utilities.streamer.StreamerCheckpointUtils.getLatestInstantWithValidCheckpointInfo; +import static org.apache.hudi.utilities.streamer.StreamerCheckpointUtils.shouldTargetCheckpointV2; /** * Sync's one batch of data to hoodie table. @@ -1031,7 +1033,8 @@ Map extractCheckpointMetadata(InputBatch inputBatch, TypedProper } // Otherwise create new checkpoint based on version - Checkpoint checkpoint = buildCheckpointFromGeneralSource(cfg.sourceClassName, versionCode, null); + Checkpoint checkpoint = shouldTargetCheckpointV2(versionCode, cfg.sourceClassName) + ? new StreamerCheckpointV2((String) null) : createCheckpoint((String) null); return checkpoint.getCheckpointCommitMetadata(cfg.checkpoint, cfg.ignoreCheckpoint); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java index 323a2b95149de..2b28e09e3a2cb 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java @@ -26,6 +26,8 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.checkpoint.Checkpoint; import org.apache.hudi.common.table.checkpoint.CheckpointUtils; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.table.checkpoint.UnresolvedStreamerCheckpointBasedOnCfg; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineUtils; @@ -44,8 +46,8 @@ import java.io.IOException; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.DATASOURCES_NOT_SUPPORTED_WITH_CKPT_V2; import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.HOODIE_INCREMENTAL_SOURCES; -import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.buildCheckpointFromConfigOverride; import static org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2; import static org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2.STREAMER_CHECKPOINT_RESET_KEY_V2; import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN; @@ -56,6 +58,28 @@ @Slf4j public class StreamerCheckpointUtils { + /** + * Wraps a user-supplied checkpoint override string. For HoodieIncrSource family on table version + * 8+, returns {@link UnresolvedStreamerCheckpointBasedOnCfg} so the source can resolve V1/V2 + * cursor semantics from the override's prefix; for everything else returns V1. + */ + public static Checkpoint buildCheckpointFromConfigOverride( + String sourceClassName, int writeTableVersion, String checkpointToResume) { + return shouldTargetCheckpointV2(writeTableVersion, sourceClassName) + ? new UnresolvedStreamerCheckpointBasedOnCfg(checkpointToResume) + : new StreamerCheckpointV1(checkpointToResume); + } + + /** + * True only for the HoodieIncrSource family on table version 8+, where V2 (completion-time) + * cursor semantics differ from V1 (requested-time). Every other source operates on V1 only. + */ + public static boolean shouldTargetCheckpointV2(int writeTableVersion, String sourceClassName) { + return writeTableVersion >= HoodieTableVersion.EIGHT.versionCode() + && HOODIE_INCREMENTAL_SOURCES.contains(sourceClassName) + && !DATASOURCES_NOT_SUPPORTED_WITH_CKPT_V2.contains(sourceClassName); + } + /** * The first phase of checkpoint resolution - read the checkpoint configs from 2 sources and resolve * conflicts: diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java index f38f8db4f48b8..8bece4c3df2c2 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java @@ -31,7 +31,6 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; -import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; @@ -83,11 +82,13 @@ import java.util.function.BooleanSupplier; import java.util.function.Function; +import static org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2; import static org.apache.hudi.common.table.timeline.InstantComparison.GREATER_THAN; import static org.apache.hudi.common.table.timeline.InstantComparison.compareTimestamps; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j @@ -740,10 +741,24 @@ static HoodieInstant assertCommitMetadata(String expected, String tablePath, int HoodieInstant lastInstant = timeline.lastInstant().get(); HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(lastInstant); assertEquals(totalCommits, timeline.countInstants()); + assertEquals(expected, commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_KEY)); + assertNull(commitMetadata.getMetadata(STREAMER_CHECKPOINT_KEY_V2)); + return lastInstant; + } + + static HoodieInstant assertCommitMetadataForIncrSource(String expected, String tablePath, int totalCommits) + throws IOException { + HoodieTableMetaClient meta = createMetaClient(storage.getConf(), tablePath); + HoodieTimeline timeline = meta.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(); + HoodieInstant lastInstant = timeline.lastInstant().get(); + HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(lastInstant); + assertEquals(totalCommits, timeline.countInstants()); if (meta.getTableConfig().getTableVersion().greaterThanOrEquals(HoodieTableVersion.EIGHT)) { - assertEquals(expected, commitMetadata.getMetadata(StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2)); + assertEquals(expected, commitMetadata.getMetadata(STREAMER_CHECKPOINT_KEY_V2)); + assertNull(commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_KEY)); } else { assertEquals(expected, commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_KEY)); + assertNull(commitMetadata.getMetadata(STREAMER_CHECKPOINT_KEY_V2)); } return lastInstant; } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 96bb18044a45b..0584a1ba26755 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -462,17 +462,17 @@ public void testInferKeyGenerator(String propsFilename, expectedKeyGeneratorClassName, metaClient.getTableConfig().getKeyGeneratorClassName()); Dataset res = sqlContext.read().format("hudi").load(tableBasePath); assertEquals(1000, res.count()); - assertUseV2Checkpoint(metaClient); + assertCheckpointVersion(metaClient); } - private static void assertUseV2Checkpoint(HoodieTableMetaClient metaClient) { + private static void assertCheckpointVersion(HoodieTableMetaClient metaClient) { metaClient.reloadActiveTimeline(); Option metadata = HoodieClientTestUtils.getCommitMetadataForInstant( metaClient, metaClient.getActiveTimeline().lastInstant().get()); assertFalse(metadata.isEmpty()); Map extraMetadata = metadata.get().getExtraMetadata(); - assertTrue(extraMetadata.containsKey(STREAMER_CHECKPOINT_KEY_V2)); - assertFalse(extraMetadata.containsKey(STREAMER_CHECKPOINT_KEY_V1)); + assertTrue(extraMetadata.containsKey(STREAMER_CHECKPOINT_KEY_V1)); + assertFalse(extraMetadata.containsKey(STREAMER_CHECKPOINT_KEY_V2)); } @Test @@ -633,7 +633,7 @@ public void testSchemaEvolution(String tableType, boolean useUserProvidedSchema, cfg.configs.add(DataSourceWriteOptions.RECONCILE_SCHEMA().key() + "=true"); syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1000, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00000", tableBasePath, 1); @@ -646,7 +646,7 @@ public void testSchemaEvolution(String tableType, boolean useUserProvidedSchema, cfg.configs.add("hoodie.streamer.schemaprovider.target.schema.file=" + basePath + "/source_evolved.avsc"); cfg.configs.add(DataSourceWriteOptions.RECONCILE_SCHEMA().key() + "=true"); syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); // out of 1000 new records, 500 are inserts, 450 are updates and 50 are deletes. assertRecordCount(1450, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00001", tableBasePath, 2); @@ -711,7 +711,7 @@ public void testTimestampMillis() throws Exception { syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1000, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00000", tableBasePath, 1); TableSchemaResolver tableSchemaResolver = new TableSchemaResolver( @@ -735,7 +735,7 @@ public void testTimestampMillis() throws Exception { cfg.configs.add("hoodie.datasource.write.row.writer.enable=false"); syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1450, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00001", tableBasePath, 2); tableSchemaResolver = new TableSchemaResolver( @@ -778,7 +778,7 @@ public void testLogicalTypes() throws Exception { cfg.configs.add("hoodie.datasource.write.row.writer.enable=false"); syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1000, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00000", tableBasePath, 1); TableSchemaResolver tableSchemaResolver = new TableSchemaResolver( @@ -798,7 +798,7 @@ public void testLogicalTypes() throws Exception { cfg.configs.add("hoodie.datasource.write.row.writer.enable=false"); syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1450, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata("00001", tableBasePath, 2); tableSchemaResolver = new TableSchemaResolver( @@ -873,7 +873,7 @@ void testLogicalTypesWithJsonSource(boolean hasTransformer, String orderingField tableBasePath, WriteOperationType.INSERT, hasTransformer, orderingField, recordType, tableType); syncOnce(cfg); // Validate. - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1000, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata(topicName + ",0:500,1:500", tableBasePath, 1); TableSchemaResolver tableSchemaResolver = new TableSchemaResolver( @@ -889,7 +889,7 @@ void testLogicalTypesWithJsonSource(boolean hasTransformer, String orderingField tableBasePath, WriteOperationType.UPSERT, hasTransformer, orderingField, recordType, tableType); syncOnce(cfg); // Validate. - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(1500, tableBasePath, sqlContext); TestHelpers.assertCommitMetadata(topicName + ",0:1250,1:1250", tableBasePath, 2); tableSchemaResolver = new TableSchemaResolver( @@ -1472,7 +1472,7 @@ public void testUpsertsCOW_ContinuousModeDisabled() throws Exception { cfg.configs.add(String.format("%s=%s", HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key(), MetricsReporterType.INMEMORY.name())); cfg.continuousMode = false; syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(SQL_SOURCE_NUM_RECORDS, tableBasePath, sqlContext); assertFalse(Metrics.isInitialized(tableBasePath), "Metrics should be shutdown"); UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath); @@ -1501,7 +1501,7 @@ public void testUpsertsMOR_ContinuousModeDisabled() throws Exception { cfg.configs.add(String.format("%s=%s", HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key(), MetricsReporterType.INMEMORY.name())); cfg.continuousMode = false; syncOnce(cfg); - assertUseV2Checkpoint(HoodieTestUtils.createMetaClient(storage, tableBasePath)); + assertCheckpointVersion(HoodieTestUtils.createMetaClient(storage, tableBasePath)); assertRecordCount(SQL_SOURCE_NUM_RECORDS, tableBasePath, sqlContext); assertFalse(Metrics.isInitialized(tableBasePath), "Metrics should be shutdown"); UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath); @@ -2414,7 +2414,7 @@ private void testBulkInsertRowWriterMultiBatches(Boolean useSchemaProvider, List entry, metaClient, WriteOperationType.BULK_INSERT)); } } - assertUseV2Checkpoint(createMetaClient(jsc, tableBasePath)); + assertCheckpointVersion(createMetaClient(jsc, tableBasePath)); } finally { deltaStreamer.shutdownGracefully(); } @@ -2534,7 +2534,7 @@ public void testBulkInsertsAndUpsertsWithSQLBasedTransformerFor2StepPipeline() t assertRecordCount(1000, downstreamTableBasePath, sqlContext); assertDistanceCount(1000, downstreamTableBasePath, sqlContext); assertDistanceCountWithExactValue(1000, downstreamTableBasePath, sqlContext); - TestHelpers.assertCommitMetadata(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 1); + TestHelpers.assertCommitMetadataForIncrSource(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 1); // No new data => no commits for upstream table cfg.sourceLimit = 0; @@ -2552,7 +2552,7 @@ public void testBulkInsertsAndUpsertsWithSQLBasedTransformerFor2StepPipeline() t assertRecordCount(1000, downstreamTableBasePath, sqlContext); assertDistanceCount(1000, downstreamTableBasePath, sqlContext); assertDistanceCountWithExactValue(1000, downstreamTableBasePath, sqlContext); - TestHelpers.assertCommitMetadata(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 1); + TestHelpers.assertCommitMetadataForIncrSource(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 1); // upsert() #1 on upstream hudi table cfg.sourceLimit = 2000; @@ -2576,7 +2576,7 @@ public void testBulkInsertsAndUpsertsWithSQLBasedTransformerFor2StepPipeline() t assertDistanceCount(2000, downstreamTableBasePath, sqlContext); assertDistanceCountWithExactValue(2000, downstreamTableBasePath, sqlContext); HoodieInstant finalInstant = - TestHelpers.assertCommitMetadata(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 2); + TestHelpers.assertCommitMetadataForIncrSource(lastInstantForUpstreamTable.getCompletionTime(), downstreamTableBasePath, 2); counts = countsPerCommit(downstreamTableBasePath, sqlContext); assertEquals(2000, counts.stream().mapToLong(entry -> entry.getLong(1)).sum()); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java index 5fdae40ed5b8d..7a97e62fd77e9 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java @@ -95,13 +95,13 @@ public void teardown() { } private void setupRowSource(Dataset ds, TypedProperties properties, SchemaProvider schemaProvider) { - InputBatch> batch = new InputBatch<>(Option.of(ds), DUMMY_CHECKPOINT, schemaProvider); + InputBatch> batch = new InputBatch<>(Option.of(ds), new StreamerCheckpointV2(DUMMY_CHECKPOINT), schemaProvider); testRowDataSource = new TestRowDataSource(properties, jsc, spark, schemaProvider, batch); } private void setupJsonSource(JavaRDD ds, HoodieSchema schema) { SchemaProvider basicSchemaProvider = new BasicSchemaProvider(schema); - InputBatch> batch = new InputBatch<>(Option.of(ds), DUMMY_CHECKPOINT, basicSchemaProvider); + InputBatch> batch = new InputBatch<>(Option.of(ds), new StreamerCheckpointV2(DUMMY_CHECKPOINT), basicSchemaProvider); testJsonDataSource = new TestJsonDataSource(new TypedProperties(), jsc, spark, basicSchemaProvider, batch); } @@ -183,7 +183,7 @@ public void testTargetSchemaUsedForNonFileBasedProvider() { SchemaProvider schemaProvider = new TestSchemaProviderWithTransformation(sourceSchema, targetSchema); // Setup the row source - InputBatch> batch = new InputBatch<>(Option.of(testDataset), DUMMY_CHECKPOINT, schemaProvider); + InputBatch> batch = new InputBatch<>(Option.of(testDataset), new StreamerCheckpointV2(DUMMY_CHECKPOINT), schemaProvider); TestRowDataSource testSource = new TestRowDataSource(new TypedProperties(), jsc, spark, schemaProvider, batch); // Create SourceFormatAdapter and fetch data in Avro format diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java index 1ec7842fe4e12..ed993a837c99b 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java @@ -33,6 +33,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; + /** * An implementation of {@link Source}, that emits test upserts. */ @@ -69,6 +71,6 @@ protected InputBatch> readFromCheckpoint(Option records = fetchNextBatch(props, (int) sourceLimit, recordInstantTime.orElse(instantTime), DEFAULT_PARTITION_NUM).collect(Collectors.toList()); JavaRDD avroRDD = sparkContext.parallelize(records, 4); - return new InputBatch<>(Option.of(avroRDD), instantTime); + return new InputBatch<>(Option.of(avroRDD), createCheckpoint(instantTime)); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java index c45a9eaf7d107..e7b9b2b46c3cd 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; @@ -60,7 +61,7 @@ public class TestGcsEventsSource extends UtilitiesTestBase { protected FilebasedSchemaProvider schemaProvider; private TypedProperties props; - private static final Checkpoint CHECKPOINT_VALUE_ZERO = new StreamerCheckpointV2("0"); + private static final Checkpoint CHECKPOINT_VALUE_ZERO = new StreamerCheckpointV1("0"); @BeforeAll public static void beforeAll() throws Exception { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHiveIncrPullSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHiveIncrPullSource.java new file mode 100644 index 0000000000000..8fbb98e0ab6df --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHiveIncrPullSource.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.sources; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.testutils.HoodieSparkClientTestHarness; +import org.apache.hudi.utilities.config.HiveIncrPullSourceConfig; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestHiveIncrPullSource extends HoodieSparkClientTestHarness { + + private TypedProperties props; + private String incrPullRoot; + + @BeforeEach + void setUp() throws Exception { + initSparkContexts(); + initPath(); + initHoodieStorage(); + incrPullRoot = basePath + "/incrPullRoot"; + FileSystem fs = (FileSystem) storage.getFileSystem(); + fs.mkdirs(new Path(incrPullRoot)); + props = new TypedProperties(); + props.setProperty(HiveIncrPullSourceConfig.ROOT_INPUT_PATH.key(), incrPullRoot); + } + + @AfterEach + void teardown() throws Exception { + cleanupResources(); + } + + private void createCommitDir(String commitTime) throws Exception { + FileSystem fs = (FileSystem) storage.getFileSystem(); + fs.mkdirs(new Path(incrPullRoot, commitTime)); + } + + @Test + void findCommitToPullReturnsV1CheckpointForFirstCommitWhenNoLatestTargetCommit() throws Exception { + createCommitDir("20240101"); + createCommitDir("20240102"); + HiveIncrPullSource source = new HiveIncrPullSource(props, jsc, sparkSession, null); + Method findCommitToPull = HiveIncrPullSource.class.getDeclaredMethod("findCommitToPull", Option.class); + findCommitToPull.setAccessible(true); + + @SuppressWarnings("unchecked") + Option result = (Option) findCommitToPull.invoke(source, Option.empty()); + + assertTrue(result.isPresent()); + assertInstanceOf(StreamerCheckpointV1.class, result.get()); + assertEquals("20240101", result.get().getCheckpointKey()); + } + + @Test + void readFromCheckpointRewrapsV2LastCheckpointAsV1WhenNoCommitToPull() throws Exception { + createCommitDir("20240101"); + createCommitDir("20240102"); + HiveIncrPullSource source = new HiveIncrPullSource(props, jsc, sparkSession, null); + + // lastCheckpoint past the last commit so findCommitToPull returns empty and the early return hits. + InputBatch batch = source.readFromCheckpoint( + Option.of(new StreamerCheckpointV2("20240103")), Long.MAX_VALUE); + + assertFalse(batch.getBatch().isPresent()); + assertInstanceOf(StreamerCheckpointV1.class, batch.getCheckpointForNextBatch()); + assertEquals("20240103", batch.getCheckpointForNextBatch().getCheckpointKey()); + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java index f7bb36076a7a3..bf969ae2ced43 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java @@ -34,7 +34,7 @@ public class TestInputBatch { @Test public void getSchemaProviderShouldThrowException() { - final InputBatch inputBatch = new InputBatch<>(Option.of("foo"), (String) null, null); + final InputBatch inputBatch = new InputBatch<>(Option.of("foo"), null, null); Throwable t = assertThrows(HoodieException.class, inputBatch::getSchemaProvider); assertEquals("Schema provider is required for this operation and for the source of interest. " + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. " @@ -45,7 +45,7 @@ public void getSchemaProviderShouldThrowException() { @Test public void getSchemaProviderShouldReturnNullSchemaProvider() { - final InputBatch inputBatch = new InputBatch<>(Option.empty(), (String) null, null); + final InputBatch inputBatch = new InputBatch<>(Option.empty(), null, null); SchemaProvider schemaProvider = inputBatch.getSchemaProvider(); assertTrue(schemaProvider instanceof InputBatch.NullSchemaProvider); } @@ -53,7 +53,7 @@ public void getSchemaProviderShouldReturnNullSchemaProvider() { @Test public void getSchemaProviderShouldReturnGivenSchemaProvider() { SchemaProvider schemaProvider = new RowBasedSchemaProvider(null); - final InputBatch inputBatch = new InputBatch<>(Option.of("foo"), (String) null, schemaProvider); + final InputBatch inputBatch = new InputBatch<>(Option.of("foo"), null, schemaProvider); assertSame(schemaProvider, inputBatch.getSchemaProvider()); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java index 6d00c9c91bf6f..9fc76f94f5139 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; @@ -273,7 +274,7 @@ public void testIncrementalFetchWhenLastCheckpointMoreThanTableRecords() { InputBatch> batch = runSource(Option.empty(), 100); Dataset rowDataset = batch.getBatch().get(); assertEquals(100, rowDataset.count()); - assertEquals(new StreamerCheckpointV2("100"), batch.getCheckpointForNextBatch()); + assertEquals(new StreamerCheckpointV1("100"), batch.getCheckpointForNextBatch()); // Add 100 records with commit time "001" insert("001", 100, connection, DATA_GENERATOR, PROPS); @@ -361,7 +362,7 @@ public void testFullFetchWithCheckpoint() { InputBatch> batch = runSource(Option.empty(), 10); Dataset rowDataset = batch.getBatch().get(); assertEquals(10, rowDataset.count()); - assertEquals(new StreamerCheckpointV2(""), batch.getCheckpointForNextBatch()); + assertEquals(new StreamerCheckpointV1(""), batch.getCheckpointForNextBatch()); // Get max of incremental column Column incrementalColumn = rowDataset diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java new file mode 100644 index 0000000000000..32480348a5654 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.sources; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.CheckpointUtils; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.utilities.config.DFSPathSelectorConfig; +import org.apache.hudi.utilities.config.JdbcSourceConfig; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.DFSPathSelector; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen; +import org.apache.hudi.utilities.sources.helpers.KinesisOffsetGen; +import org.apache.hudi.utilities.sources.helpers.KinesisOffsetGen.KinesisShardRange; +import org.apache.hudi.utilities.sources.helpers.gcs.PubsubMessagesFetcher; +import org.apache.hudi.utilities.streamer.DefaultStreamContext; +import org.apache.hudi.utilities.streamer.StreamerCheckpointUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.streaming.kafka010.OffsetRange; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.hudi.config.HoodieWriteConfig.WRITE_TABLE_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Validates that every non-incremental streamer source emits a V1 checkpoint regardless of the + * configured write table version or the wrapper class of the input checkpoint (so V2 keys + * persisted by older releases are read on input but never written back as V2). Hudi incremental + * sources own their own V1/V2 semantics and are exercised by + * {@link #testS3AndGcsIncrSourcesStayV1OnBothTableVersions()} plus + * {@code TestHoodieIncrSource} family. + */ +class TestStreamerSourceCheckpointVersion { + + private static JavaSparkContext jsc; + private static SparkSession spark; + + @TempDir + static java.nio.file.Path tempDir; + + @BeforeAll + static void initSpark() { + spark = SparkSession.builder().master("local[1]").appName("TestSourceCheckpointVersion").getOrCreate(); + jsc = new JavaSparkContext(spark.sparkContext()); + } + + @AfterAll + static void stopSpark() { + if (jsc != null) { + jsc.stop(); + } + if (spark != null) { + spark.stop(); + } + } + + // Covers AvroDFSSource, JsonDFSSource, CsvDFSSource, ParquetDFSSource, ORCDFSSource. + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testDfsPathSelector(int writeTableVersion) throws IOException { + TypedProperties props = propsWith(writeTableVersion); + props.setProperty(DFSPathSelectorConfig.ROOT_INPUT_PATH.key(), tempDir.toString()); + Configuration hadoopConf = jsc.hadoopConfiguration(); + FileSystem fs = FileSystem.get(hadoopConf); + fs.mkdirs(new Path(tempDir.toString())); + DFSPathSelector selector = new DFSPathSelector(props, hadoopConf); + Pair, Checkpoint> result = + selector.getNextFilePathsAndMaxModificationTime(jsc, Option.empty(), Long.MAX_VALUE); + assertV1(result.getRight()); + } + + // Covers AvroKafkaSource, JsonKafkaSource, ProtoKafkaSource. + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testKafkaSource(int writeTableVersion, InputCheckpointKind inputKind) { + TestableKafkaSource source = new TestableKafkaSource(propsWith(writeTableVersion), jsc, spark); + KafkaOffsetGen offsetGen = mock(KafkaOffsetGen.class); + when(offsetGen.getNextOffsetRanges(any(), anyLong(), any())).thenReturn( + new OffsetRange[] {OffsetRange.create("t", 0, 0L, 0L)}); + when(offsetGen.getTopicName()).thenReturn("t"); + source.offsetGen = offsetGen; + InputBatch batch = source.fetchNext(makeInputCheckpoint(inputKind, "k"), 1L); + assertV1(batch.getCheckpointForNextBatch()); + } + + // Covers JsonKinesisSource. + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testKinesisSource(int writeTableVersion, InputCheckpointKind inputKind) { + TestableKinesisSource source = new TestableKinesisSource(propsWith(writeTableVersion), jsc, spark); + KinesisOffsetGen offsetGen = mock(KinesisOffsetGen.class); + when(offsetGen.getStreamName()).thenReturn("s"); + when(offsetGen.getNextShardRanges(any(), anyLong())).thenReturn( + new KinesisShardRange[0]); + source.setOffsetGen(offsetGen); + InputBatch> batch = source.fetchNext(makeInputCheckpoint(inputKind, "s"), 1L); + assertV1(batch.getCheckpointForNextBatch()); + } + + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testJdbcSource(int writeTableVersion, InputCheckpointKind inputKind) throws Exception { + JdbcSource source = new JdbcSource(propsWith(writeTableVersion), jsc, spark, null); + Method m = JdbcSource.class.getDeclaredMethod( + "checkpoint", Dataset.class, boolean.class, Option.class); + m.setAccessible(true); + Checkpoint c = (Checkpoint) m.invoke(source, null, false, makeInputCheckpoint(inputKind, "k")); + assertV1(c); + } + + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testSqlFileBasedSource(int writeTableVersion, InputCheckpointKind inputKind) throws IOException { + java.nio.file.Path sqlFile = tempDir.resolve("q.sql"); + Files.write(sqlFile, "SELECT 1".getBytes()); + TypedProperties props = propsWith(writeTableVersion); + props.setProperty("hoodie.streamer.source.sql.file", sqlFile.toString()); + props.setProperty("hoodie.streamer.source.sql.checkpoint.emit", "true"); + SqlFileBasedSource source = new SqlFileBasedSource(props, jsc, spark, null); + Pair>, Checkpoint> result = + invokeRowSourceFetch(source, makeInputCheckpoint(inputKind, "k")); + assertV1(result.getRight()); + } + + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testHiveIncrPullSource(int writeTableVersion, InputCheckpointKind inputKind) throws Exception { + Files.createDirectories(tempDir.resolve("20200101000000")); + TypedProperties props = propsWith(writeTableVersion); + props.setProperty("hoodie.streamer.source.incrpull.root", tempDir.toString()); + HiveIncrPullSource source = new HiveIncrPullSource(props, jsc, spark, null); + Method m = HiveIncrPullSource.class.getDeclaredMethod("findCommitToPull", Option.class); + m.setAccessible(true); + @SuppressWarnings("unchecked") + Option result = (Option) m.invoke( + source, makeInputCheckpoint(inputKind, "00000000000000")); + assertV1(result.get()); + } + + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testGcsEventsSource(int writeTableVersion, InputCheckpointKind inputKind) { + PubsubMessagesFetcher fetcher = mock(PubsubMessagesFetcher.class); + when(fetcher.fetchMessages()).thenReturn(Collections.emptyList()); + TypedProperties props = propsWith(writeTableVersion); + props.setProperty("hoodie.streamer.source.gcs.project.id", "p"); + props.setProperty("hoodie.streamer.source.gcs.subscription.id", "s"); + GcsEventsSource source = new GcsEventsSource(props, jsc, spark, null, fetcher); + Pair>, Checkpoint> result = + invokeRowSourceFetch(source, makeInputCheckpoint(inputKind, "k")); + assertV1(result.getRight()); + } + + @ParameterizedTest + @EnumSource(InputCheckpointKind.class) + void testJdbcSourceIncrementalWithMaxValueEmitsV1(InputCheckpointKind inputKind) throws Exception { + TypedProperties props = propsWith(8); + props.setProperty(JdbcSourceConfig.INCREMENTAL_COLUMN.key(), "idx"); + JdbcSource source = new JdbcSource(props, jsc, spark, null); + StructType schema = new StructType().add("idx", DataTypes.StringType, true); + List rows = Arrays.asList(RowFactory.create("100"), RowFactory.create("200")); + Dataset dataset = spark.createDataFrame(rows, schema); + Method m = JdbcSource.class.getDeclaredMethod( + "checkpoint", Dataset.class, boolean.class, Option.class); + m.setAccessible(true); + Checkpoint c = (Checkpoint) m.invoke(source, dataset, true, makeInputCheckpoint(inputKind, "k")); + assertV1(c); + assertEquals("200", c.getCheckpointKey()); + } + + // Drives the isIncremental + max==null pass-through to confirm it re-wraps a V2 input as V1. + @ParameterizedTest + @EnumSource(InputCheckpointKind.class) + void testJdbcSourcePassThroughEmitsV1(InputCheckpointKind inputKind) throws Exception { + TypedProperties props = propsWith(8); + props.setProperty(JdbcSourceConfig.INCREMENTAL_COLUMN.key(), "idx"); + JdbcSource source = new JdbcSource(props, jsc, spark, null); + StructType schema = new StructType().add("idx", DataTypes.StringType, true); + List rows = Collections.singletonList(RowFactory.create((Object) null)); + Dataset nullColDataset = spark.createDataFrame(rows, schema); + Method m = JdbcSource.class.getDeclaredMethod( + "checkpoint", Dataset.class, boolean.class, Option.class); + m.setAccessible(true); + Checkpoint c = (Checkpoint) m.invoke(source, nullColDataset, true, makeInputCheckpoint(inputKind, "k")); + assertV1(c); + } + + // Input key sorts after the only commit on disk so findCommitToPull returns empty and + // readFromCheckpoint enters its pass-through branch. + @ParameterizedTest + @EnumSource(InputCheckpointKind.class) + void testHiveIncrPullSourcePassThroughEmitsV1(InputCheckpointKind inputKind) throws IOException { + Files.createDirectories(tempDir.resolve("20200101000000")); + TypedProperties props = propsWith(8); + props.setProperty("hoodie.streamer.source.incrpull.root", tempDir.toString()); + HiveIncrPullSource source = new HiveIncrPullSource(props, jsc, spark, null); + InputBatch batch = source.readFromCheckpoint( + makeInputCheckpoint(inputKind, "30000000000000"), 1L); + assertV1(batch.getCheckpointForNextBatch()); + } + + @ParameterizedTest + @CsvSource({"6, V1", "6, V2", "8, V1", "8, V2"}) + void testTranslateCheckpointNormalizesToV1(int writeTableVersion, InputCheckpointKind inputKind) { + TestableKafkaSource source = new TestableKafkaSource(propsWith(writeTableVersion), jsc, spark); + Option translated = source.translateCheckpoint(makeInputCheckpoint(inputKind, "k")); + assertV1(translated.get()); + assertEquals("k", translated.get().getCheckpointKey()); + } + + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testTranslateCheckpointPreservesEmpty(int writeTableVersion) { + TestableKafkaSource source = new TestableKafkaSource(propsWith(writeTableVersion), jsc, spark); + assertTrue(source.translateCheckpoint(Option.empty()).isEmpty()); + } + + @Test + void testS3AndGcsIncrSourcesStayV1OnBothTableVersions() { + String s3 = S3EventsHoodieIncrSource.class.getName(); + String gcs = GcsEventsHoodieIncrSource.class.getName(); + assertTrue(CheckpointUtils.DATASOURCES_NOT_SUPPORTED_WITH_CKPT_V2.contains(s3)); + assertTrue(CheckpointUtils.DATASOURCES_NOT_SUPPORTED_WITH_CKPT_V2.contains(gcs)); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(8, s3)); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(8, gcs)); + } + + private static TypedProperties propsWith(int writeTableVersion) { + TypedProperties props = new TypedProperties(); + props.setProperty(WRITE_TABLE_VERSION.key(), String.valueOf(writeTableVersion)); + return props; + } + + private static void assertV1(Checkpoint c) { + assertEquals(StreamerCheckpointV1.class, c.getClass()); + } + + @SuppressWarnings("unchecked") + private static Pair>, Checkpoint> invokeRowSourceFetch( + RowSource source, Option lastCheckpoint) { + try { + Method m = source.getClass().getDeclaredMethod("fetchNextBatch", Option.class, long.class); + m.setAccessible(true); + return (Pair>, Checkpoint>) m.invoke(source, lastCheckpoint, 1L); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + private enum InputCheckpointKind { V1, V2 } + + private static Option makeInputCheckpoint(InputCheckpointKind kind, String key) { + switch (kind) { + case V1: return Option.of(new StreamerCheckpointV1(key)); + case V2: return Option.of(new StreamerCheckpointV2(key)); + default: throw new IllegalArgumentException("Unsupported kind: " + kind); + } + } + + private static class TestableKafkaSource extends KafkaSource { + TestableKafkaSource(TypedProperties props, JavaSparkContext jsc, SparkSession spark) { + super(props, jsc, spark, SourceType.JSON, mock(HoodieIngestionMetrics.class), + new DefaultStreamContext(null, Option.empty())); + } + + @Override + protected String toBatch(OffsetRange[] offsetRanges) { + return "batch"; + } + } + + private static class TestableKinesisSource extends KinesisSource> { + TestableKinesisSource(TypedProperties props, JavaSparkContext jsc, SparkSession spark) { + super(props, jsc, spark, SourceType.JSON, mock(HoodieIngestionMetrics.class), + new DefaultStreamContext((SchemaProvider) null, Option.empty())); + } + + void setOffsetGen(KinesisOffsetGen gen) { + this.offsetGen = gen; + } + + @Override + protected JavaRDD toBatch(KinesisShardRange[] shardRanges, long sourceLimit) { + return jsc.emptyRDD(); + } + + @Override + protected String createCheckpointFromBatch(JavaRDD batch, + KinesisShardRange[] shardRangesWithUnreadRecords, + KinesisShardRange[] allOpenClosedShardRanges) { + return "checkpoint"; + } + + @Override + protected long getRecordCount(JavaRDD batch) { + return 1L; + } + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java index 5b3d970c03680..30e138d5e3e96 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; @@ -44,6 +45,7 @@ import static org.apache.hudi.utilities.config.DFSPathSelectorConfig.ROOT_INPUT_PATH; import static org.apache.hudi.utilities.config.DatePartitionPathSelectorConfig.PARTITIONS_LIST_PARALLELISM; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarness { @@ -164,4 +166,17 @@ public void getNextFilePathsAndMaxModificationTimeShouldIgnoreSourceLimitIfSameM String checkpointStr2ndRead = nextFilePathsAndCheckpoint.getRight().getCheckpointKey(); assertEquals(2000L, Long.parseLong(checkpointStr2ndRead), "should read up to foo5 (inclusive)"); } + + @ParameterizedTest + @ValueSource(classes = {DFSPathSelector.class, DatePartitionPathSelector.class}) + void getNextFilePathsAndMaxModificationTimeReturnsV1CheckpointWhenNoEligibleFiles(Class clazz) throws Exception { + DFSPathSelector selector = (DFSPathSelector) ReflectionUtils.loadClass(clazz.getName(), props, storageConf.unwrap()); + createBaseFile(basePath, "p1", "000", "foo1", 10, 1000); + createBaseFile(basePath, "p1", "000", "foo2", 10, 2000); + Pair, Checkpoint> result = selector + .getNextFilePathsAndMaxModificationTime(jsc, Option.of(new StreamerCheckpointV2("999999999")), 30); + assertTrue(result.getLeft().isEmpty()); + assertInstanceOf(StreamerCheckpointV1.class, result.getRight()); + assertEquals("999999999", result.getRight().getCheckpointKey()); + } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java index 409b2d9fe9c8b..1b010efab7cbc 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java @@ -23,7 +23,6 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableVersion; -import org.apache.hudi.common.table.checkpoint.CheckpointUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.util.Option; import org.apache.hudi.testutils.HoodieClientTestUtils; @@ -512,9 +511,9 @@ public void testSyncE2EForceSkip(String tableVersion) throws Exception { @Test public void testTargetCheckpointV2ForS3Gcs() { // To ensure we properly track sources that must use checkpoint V1. - assertFalse(CheckpointUtils.shouldTargetCheckpointV2(8, S3EventsHoodieIncrSource.class.getName())); - assertFalse(CheckpointUtils.shouldTargetCheckpointV2(6, S3EventsHoodieIncrSource.class.getName())); - assertFalse(CheckpointUtils.shouldTargetCheckpointV2(8, GcsEventsHoodieIncrSource.class.getName())); - assertFalse(CheckpointUtils.shouldTargetCheckpointV2(6, GcsEventsHoodieIncrSource.class.getName())); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(8, S3EventsHoodieIncrSource.class.getName())); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(6, S3EventsHoodieIncrSource.class.getName())); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(8, GcsEventsHoodieIncrSource.class.getName())); + assertFalse(StreamerCheckpointUtils.shouldTargetCheckpointV2(6, GcsEventsHoodieIncrSource.class.getName())); } } \ No newline at end of file diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestParquetDfsCheckpointFormatOnV6.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestParquetDfsCheckpointFormatOnV6.java new file mode 100644 index 0000000000000..596be797e36bf --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestParquetDfsCheckpointFormatOnV6.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.streamer; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamerTestBase; +import org.apache.hudi.utilities.sources.ParquetDFSSource; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Continues ingestion against a fixture table that was originally written with V1 checkpoint + * format only and is now being resumed on a Hudi 1.x release with + * {@code hoodie.write.table.version=6}. + * + *

    Expected: the next commit's {@code extraMetadata} carries the V1 key + * {@code deltastreamer.checkpoint.key} and NOT the V2 key {@code streamer.checkpoint.key.v2}, + * because non-incremental sources emit V1 regardless of write table version. + * + *

    If a V2 key shows up on the resumed commit, that reproduces the bug reported against + * {@code hoodie.write.table.version=6}. + */ +public class TestParquetDfsCheckpointFormatOnV6 extends HoodieDeltaStreamerTestBase { + + private static final String FIXTURE_RESOURCE = "checkpoint-v6/parquet-dfs-v1-fixture.zip"; + + @Test + public void resumedV6CommitKeepsV1CheckpointFormat() throws Exception { + String dirName = "checkpoint-v6-resume-" + System.currentTimeMillis(); + String dataPath = basePath + "/" + dirName; + Path zipOutput = Paths.get(new URI(dataPath)); + Files.createDirectories(zipOutput); + HoodieTestUtils.extractZipToDirectory(FIXTURE_RESOURCE, zipOutput, getClass()); + + // The fixture table is rooted directly at zipOutput (zip contains .hoodie/, partitions, etc.). + String tableBasePath = zipOutput.toString(); + assertTrue(Files.exists(zipOutput.resolve(".hoodie/hoodie.properties")), + "Fixture did not unpack a hudi table at " + tableBasePath); + + // Sanity-check that the fixture's last commit carries a V1 checkpoint and no V2 key. + HoodieCommitMetadata baselineCommit = readLatestCommitMetadata(tableBasePath); + assertNotNull(baselineCommit.getMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1), + "Fixture baseline expected to carry V1 checkpoint key."); + assertNull(baselineCommit.getMetadata(StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2), + "Fixture baseline must not have a V2 checkpoint key (was built on master)."); + + // Produce a fresh parquet source file under a new root so the streamer has work to do. + String parquetSourceRoot = basePath + "/" + dirName + "-source"; + prepareParquetDFSFiles(50, parquetSourceRoot, "resume.parquet", false, null, null).close(); + + String propsFile = dirName + "-source.properties"; + TypedProperties extraProps = new TypedProperties(); + extraProps.setProperty("hoodie.datasource.write.table.type", HoodieTableType.COPY_ON_WRITE.name()); + prepareParquetDFSSource(false, false, "source.avsc", "target.avsc", + propsFile, parquetSourceRoot, false, "partition_path", "", extraProps, false, false); + + HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, + ParquetDFSSource.class.getName(), Collections.emptyList(), propsFile, false, false, + 100_000, false, null, HoodieTableType.COPY_ON_WRITE.name(), "timestamp", null); + cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=6"); + new HoodieDeltaStreamer(cfg, jsc).sync(); + + // Confirm the on-disk table version stayed at 6 so we are testing the v6-write path, not an + // accidental v6 -> v9 auto-upgrade (which would make V2 the correct result). + HoodieTableMetaClient resumedMetaClient = HoodieTestUtils.createMetaClient(storage, tableBasePath); + assertEquals(6, resumedMetaClient.getTableConfig().getTableVersion().versionCode(), + "Table version on disk should still be 6 after resume"); + + HoodieCommitMetadata resumedCommit = readLatestCommitMetadata(tableBasePath); + // Under hoodie.write.table.version=6, ParquetDFSSource must keep emitting V1 keys; in the + // always-V1 design, this holds across every write table version for non-incremental sources. + assertNotNull(resumedCommit.getMetadata(StreamerCheckpointV1.STREAMER_CHECKPOINT_KEY_V1), + "Resumed v6 commit must persist V1 checkpoint key. extraMetadata=" + + resumedCommit.getExtraMetadata()); + assertNull(resumedCommit.getMetadata(StreamerCheckpointV2.STREAMER_CHECKPOINT_KEY_V2), + "Resumed v6 commit must NOT persist a V2 checkpoint key. extraMetadata=" + + resumedCommit.getExtraMetadata()); + } + + private static HoodieCommitMetadata readLatestCommitMetadata(String tableBasePath) throws IOException { + HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(storage, tableBasePath); + HoodieInstant lastInstant = metaClient.getActiveTimeline() + .getCommitsTimeline() + .filterCompletedInstants() + .lastInstant() + .orElseThrow(() -> new IllegalStateException("No completed commit found in " + tableBasePath)); + return metaClient.getActiveTimeline().readCommitMetadata(lastInstant); + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java index 26ba7599b451f..4a851e7c0a066 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.checkpoint.Checkpoint; +import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.collection.Triple; @@ -103,10 +104,10 @@ void testFetchNextBatchFromSource(Boolean useRowWriter, Boolean hasTransformer, SourceFormatAdapter sourceFormatAdapter = mock(SourceFormatAdapter.class); SchemaProvider inputBatchSchemaProvider = getSchemaProvider("InputBatch", false); Option> fakeDataFrame = Option.of(mock(Dataset.class)); - InputBatch> fakeRowInputBatch = new InputBatch<>(fakeDataFrame, "chkpt", inputBatchSchemaProvider); + InputBatch> fakeRowInputBatch = new InputBatch<>(fakeDataFrame, new StreamerCheckpointV2("chkpt"), inputBatchSchemaProvider); when(sourceFormatAdapter.fetchNewDataInRowFormat(any(), anyLong())).thenReturn(fakeRowInputBatch); //batch is empty because we don't want getBatch().map() to do anything because it calls static method we can't mock - InputBatch> fakeAvroInputBatch = new InputBatch<>(Option.empty(), "chkpt", inputBatchSchemaProvider); + InputBatch> fakeAvroInputBatch = new InputBatch<>(Option.empty(), new StreamerCheckpointV2("chkpt"), inputBatchSchemaProvider); when(sourceFormatAdapter.fetchNewDataInAvroFormat(any(),anyLong())).thenReturn(fakeAvroInputBatch); //transformer @@ -356,11 +357,33 @@ public void testExtractCheckpointMetadata_WhenCheckpointExists() { } @Test - public void testExtractCheckpointMetadata_WhenCheckpointIsNullV2() { + void testExtractCheckpointMetadata_WhenCheckpointIsNullNonIncrementalSourceOnV8UsesV1() { StreamSync streamSync = setupStreamSync(); HoodieStreamer.Config cfg = new HoodieStreamer.Config(); cfg.checkpoint = "test-checkpoint"; cfg.ignoreCheckpoint = "test-ignore"; + cfg.sourceClassName = "org.apache.hudi.utilities.sources.KafkaSource"; + TypedProperties props = new TypedProperties(); + + InputBatch inputBatch = mock(InputBatch.class); + when(inputBatch.getCheckpointForNextBatch()).thenReturn(null); + + Map result = streamSync.extractCheckpointMetadata( + inputBatch, props, HoodieTableVersion.EIGHT.versionCode(), cfg); + + Map expected = new HashMap<>(); + expected.put(CHECKPOINT_IGNORE_KEY, "test-ignore"); + expected.put(CHECKPOINT_RESET_KEY, "test-checkpoint"); + assertEquals(expected, result, "Should fall back to V1 keys for non-incremental sources on v8"); + } + + @Test + void testExtractCheckpointMetadata_WhenCheckpointIsNullIncrementalSourceOnV8UsesV2() { + StreamSync streamSync = setupStreamSync(); + HoodieStreamer.Config cfg = new HoodieStreamer.Config(); + cfg.checkpoint = "test-checkpoint"; + cfg.ignoreCheckpoint = "test-ignore"; + cfg.sourceClassName = "org.apache.hudi.utilities.sources.HoodieIncrSource"; TypedProperties props = new TypedProperties(); InputBatch inputBatch = mock(InputBatch.class); @@ -372,7 +395,7 @@ public void testExtractCheckpointMetadata_WhenCheckpointIsNullV2() { Map expected = new HashMap<>(); expected.put(CHECKPOINT_IGNORE_KEY, "test-ignore"); expected.put(STREAMER_CHECKPOINT_RESET_KEY_V2, "test-checkpoint"); - assertEquals(expected, result, "Should return default metadata when checkpoint is null"); + assertEquals(expected, result, "Should fall back to V2 keys for incremental sources on v8"); } @Test diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamerCheckpointUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamerCheckpointUtils.java index f4e30667b295c..0d9c3f7d15f45 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamerCheckpointUtils.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamerCheckpointUtils.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.table.checkpoint.Checkpoint; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV1; import org.apache.hudi.common.table.checkpoint.StreamerCheckpointV2; +import org.apache.hudi.common.table.checkpoint.UnresolvedStreamerCheckpointBasedOnCfg; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -41,6 +42,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; @@ -51,11 +54,13 @@ import static org.apache.hudi.utilities.streamer.HoodieStreamer.CHECKPOINT_KEY; import static org.apache.hudi.utilities.streamer.StreamSync.CHECKPOINT_IGNORE_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(MockitoExtension.class) public class TestStreamerCheckpointUtils extends SparkClientFunctionalTestHarness { + private static final String CHECKPOINT_TO_RESUME = "20240101000000"; private TypedProperties props; private HoodieStreamer.Config streamerConfig; protected HoodieTableMetaClient metaClient; @@ -68,6 +73,53 @@ public void setUp() throws IOException { streamerConfig.tableType = HoodieTableType.COPY_ON_WRITE.name(); } + @ParameterizedTest + @CsvSource({ + // version, sourceClassName, expectedV2 + // Only HoodieIncrSource family on v8+ targets V2. + "8, org.apache.hudi.utilities.sources.HoodieIncrSource, true", + "9, org.apache.hudi.utilities.sources.HoodieIncrSource, true", + "8, org.apache.hudi.utilities.sources.MockGeneralHoodieIncrSource, true", + // v6/v7 always V1. + "7, org.apache.hudi.utilities.sources.HoodieIncrSource, false", + "6, org.apache.hudi.utilities.sources.HoodieIncrSource, false", + // Non-incremental sources always V1 regardless of version. + "8, org.apache.hudi.utilities.sources.KafkaSource, false", + "9, org.apache.hudi.utilities.sources.JdbcSource, false", + // V2-not-supported allowlist always V1. + "8, org.apache.hudi.utilities.sources.S3EventsHoodieIncrSource, false", + "8, org.apache.hudi.utilities.sources.GcsEventsHoodieIncrSource, false", + "8, org.apache.hudi.utilities.sources.MockS3EventsHoodieIncrSource, false", + "8, org.apache.hudi.utilities.sources.MockGcsEventsHoodieIncrSource, false" + }) + void testShouldTargetCheckpointV2(int version, String sourceClassName, boolean expectedV2) { + assertEquals(expectedV2, StreamerCheckpointUtils.shouldTargetCheckpointV2(version, sourceClassName)); + } + + @Test + void testBuildCheckpointFromConfigOverride() { + Checkpoint hoodieIncrV8 = StreamerCheckpointUtils.buildCheckpointFromConfigOverride( + "org.apache.hudi.utilities.sources.HoodieIncrSource", + HoodieTableVersion.EIGHT.versionCode(), + CHECKPOINT_TO_RESUME); + assertInstanceOf(UnresolvedStreamerCheckpointBasedOnCfg.class, hoodieIncrV8); + assertEquals(CHECKPOINT_TO_RESUME, hoodieIncrV8.getCheckpointKey()); + + Checkpoint nonIncrV8 = StreamerCheckpointUtils.buildCheckpointFromConfigOverride( + "org.apache.hudi.utilities.sources.KafkaSource", + HoodieTableVersion.EIGHT.versionCode(), + CHECKPOINT_TO_RESUME); + assertInstanceOf(StreamerCheckpointV1.class, nonIncrV8); + assertEquals(CHECKPOINT_TO_RESUME, nonIncrV8.getCheckpointKey()); + + Checkpoint hoodieIncrV6 = StreamerCheckpointUtils.buildCheckpointFromConfigOverride( + "org.apache.hudi.utilities.sources.HoodieIncrSource", + HoodieTableVersion.SIX.versionCode(), + CHECKPOINT_TO_RESUME); + assertInstanceOf(StreamerCheckpointV1.class, hoodieIncrV6); + assertEquals(CHECKPOINT_TO_RESUME, hoodieIncrV6.getCheckpointKey()); + } + @Test public void testEmptyTimelineCase() throws IOException { Option checkpoint = StreamerCheckpointUtils.resolveCheckpointBetweenConfigAndPrevCommit( diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java index 017507f2f9ece..7b7c06d622d40 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/DistributedTestDataSource.java @@ -35,6 +35,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.apache.hudi.common.table.checkpoint.CheckpointUtils.createCheckpoint; + /** * A Test DataSource which scales test-data generation by using spark parallelism. */ @@ -57,7 +59,7 @@ protected InputBatch> readFromCheckpoint(Option(Option.empty(), instantTime); + return new InputBatch<>(Option.empty(), createCheckpoint(instantTime)); } TypedProperties newProps = new TypedProperties(); @@ -77,6 +79,6 @@ protected InputBatch> readFromCheckpoint(Option(Option.of(avroRDD), instantTime); + return new InputBatch<>(Option.of(avroRDD), createCheckpoint(instantTime)); } } diff --git a/hudi-utilities/src/test/resources/checkpoint-v6/parquet-dfs-v1-fixture.zip b/hudi-utilities/src/test/resources/checkpoint-v6/parquet-dfs-v1-fixture.zip new file mode 100644 index 0000000000000000000000000000000000000000..29c1b98044359d1d47eaa6c776beb819ca777a3e GIT binary patch literal 103450 zcmdS>Wl&sA)PRc)POw054aH##>)POoxPTwT?yIY9gI`ta6{acvonXR7`wBot*8&2s^4!203}%@12ahsa3v74DPu^O;3vGX!gVuf=r> z3Xy&KYy82aG5xvE66e^kYQ;ljC>mONCd0$iJ^MyRt{3lMKuoQQpuNH-mj`pKpPcea z?`(g4Htd^BjYzi&a75lT*wV;)ndZ5+oBCNjA5(I?T!GB=;_8pAJ-Euy^`ooiA zrV%yK$0@K$ZLJH~S3_+pQ}|+g|GZO#e?~twI^nlsx)7Q0eiQhmN#z?s*16&9c>%{z zngXiz<9cpEy<&P|V+F6TQp6q7g9MsYor%@g?Oja+O`XZ5B{>hiZ};~(Yaq)V&el-A z^eec$z$ZDB=YvDcPh%FZX0djcpwnX4ZVzbh%R*67>i zHP!SWEa%<0w;~lNE=0yZcKuj|HnN>hsyTXJecf8p-f)s$|6<<4x|ycT5>82``>=5r z(RAt!On>deGAzsj3Hi?U(lK;-w``L?Jf|BfpBF798$2!^6qRWBX*u0h-J8~* zo@H_XuWqH{>LZtwd@NovO`F|58}K>YzG(}kw>$@nquM>}b_C4Dqc2#Smt8-Vs?X2s zUHI-6rWX3xKKgVPMVN=k5l`}6t!~}vv%Q=Ltf7qwv~Sl=89SQ?3*siszZ{y|*}m;n zB=;Q)_$iJ>5qCMKlON7h4DHBMa=$cw@%%a{u_3y7W<42<3i2a+njBwwlk>a}Ghch} zb389p?RlNgcV*_|ds!%4t14Sh)l+d9kD=+Gjr)j9o-D-8lRa4EK|AC`-|L$NS;{2X ztB`5_Wam`kl@$!r6>s&xxm@#5RR2|M= ziGt|jY1{uilyu!@XY&-{U3PT7SzEM%mjWwkdUx{j=-A&zV{mtnh}gU)Mk~ULRj;?sN>K_nDGZUcI4dq}P%>t7(Vo;$S@9xBJR1 z)SMv#coWVzvo2r7k6gu zovh;a})rA6NTdE|Vp@XfubO?BCiOG?+3`!>=#B#gy+Q5&}UIeR1- zKb-c}i=BzWDffuvT4DaDOFyK)G#r|@psV{rqs3}_scReGugfW1!saVg-MLoV#|U4# z_cmv-`N_X~>gKWE_Ti$UoUX3BPr{-^{6dGMn?P=muR?Y|lU0*Ky z^fuFu+7_n@_Y57aH;3EP%RDrPhxh9??DVYN)7-)ylE&u`=wPD5Sx;CT3A5Gu~Uc4a%1We_26w>z|5>(xbA~)cCnD?!|jfhdyuk^hM z_ccmCFdr3&t<;&e{qiLz`Rcn9X!QB0iP8;LZuQ&8{u*iunGX}TN$8qs^X;r+ zO-cgm_fmzMUOs1?c>2CJQ?}vD@_; zJ-1E912uPj9nS+x435ve-S#_yPLC^9n=TvSZahGKh=E=;({A94#DO=Zj9XjF*VaFQ zz`fhIr+mfc`=aC{8SG7@Xm4&m@Abb&+`g}Wb5kW2k-nha8y)D~t+F}qP#E_DLCaI6 z2XUSueTPy$H+SNi?kzXqWYE2>%xhJr`S2V_=y_tga?vhc@{|kXwHk4NS4n(^_MmRx9fA zs7gTdQ{Y`RVj=R#D4l~dJI4~FiW!~%LrNkMUOtLldRu-gSEfS_=+DJ{dB}^E49%X$ z<3tue9^}bfeWUn{S2FeDZKS2O@SNa%*5;l1Y%HN))&5pc zRv@8BMv8;uvK{gyYoIUKC=rnidOSU~zIs88KchRTr*XVb9z5E;2iKfj z4E}iG_vJB>+N&KmaALGwa*xr=Bli+;laFs`d#spaIW@kQE&*<_yDmpE1n=Jo;kN)^ zfcDcB9_#yh*yIGcYio9)=ch?J*D-I4MXr~ca5qEIG;hUvLTdK9xBACYntHkIdoJek zB1>eCkV0-;c@plgZ|Ya=)xE{R`vxflEpGFVtB(+Lb3|dI5s-%1&+~!Q?q=AM-53{A z_+Y;)FqPNshQ+jSu+vWWSnFeg+mi*F5a#^tv*zeQVy? zAig=7dw8Fs36d*b)0$S0*@_901tAx27@=7d9$r@HFQv&)iY+|`1=mDP^nBjmG|mIB zT}({A@tSYJw=2m%z=|b?5l#>`S@qKJGtSe^{c-}x&g)wA>Yvjm}0u5s?B z93P^%kBf$zkxCV(jzaVVf$=w)Q1;IUF3OHg-+mOJNAB$)^5xu42ltJ+lCypL6A=-+LfS8wK)ZrMmo0La1UT}$pPS#es zHnW4|)3jEUKE}5mFxfY^R{3nV;#b%W-~MWuL>`^zRjB31TfX9k`Ww*}+|+IImL7av z`RbzEtMN%DuukmUX9%Kx`F4q=JpxZ5l~;v!qKW!6v(s(*+BGM-)6J2lIa~!c?fGGO z6y8Z$cEVB7)L?X7qeM~Rc@plze}QBrwddjjHT2T3U|S3VO2Xo{FRAcC>ctSzjc(brnw0u580D>OyCU%xzi5R8|S|~W3RV18M6w8pYz}mofQkHz!w{iCK zvvZVmv}>ain=xRh_?or)nsFdez=mMfN4U6PM$2iMjv7Vip4Q8+Fgs5j!;OQthqsr8 znxC_&99>;gI}#V4ZazrMI?wY_ByCY`H8uyGSkOh7Il12P)=Y_{VyH4PQ3?iBpJ{_v7>d9}~Pq7gr2~%X>N&`%_@8!X_3?9%u1r zZJHwPqWmadr($%5FgFADOc7jEQBx$YMHt#&q@KS)byh-Dur;xg!D%UJTigf;2h>jNUn(cSIwTN58Dq zV|BC_>vOC+jpbJruj)XY#@m&IbFkKylSCeMi57 z{EGW0XSFG^1=da^nyA_%xa=X|gIJdO$;Cj;r{6Wr3cX*XPdn4W+hoTy`Nf}NYU)7G zxIAbfs;}Ryd9p4e?2;57%3c@?pNk_-3Ux}X6mQh=rP>M|M~d3?M}Mu^{Z_oO%(<;O zaTnDQy$5TjQ8?*`Y78P@@M|6PUCQv>;v1>OHuKMGTD690Yw${3RcfcA?cv0ST2FwZQiGPH~Gn=Gl=ZuiR8M7zTE8S*Vs`q`#BB-{HRft?e?5ElHHB`rD z!{`0%Wbi$_b<(I}pg&BF^eCl2!s(MW(qLGEzM4zgw`umA858m{Z!bZ91Pb1+ju*MX1u&F(8ws7HK8~1krfL{bDy3Rh%NE5Vj6)@B3FUM?N)FzifO@? z3P#pbkEK!B$qYqS38LoH;3-u{>mz-fROIM3NE7T?l{2(&{_PGq{N6|Uh5Bs#IOh`M zTVbsmUv8!KPM}-Etg2r5 z>%jK|`3%dhjQf{6YO=A(fipPi*X4vn9+ z-?Q@oJp(;6eobAtL(8Fd{!*hW>5T0+Pa8es#~DVQkY|Wpx2>qHFJ9fR>>I6dqHnY2 z1K}1`hh37+(i&Msc&O>9ht&?A*P3S%N6A(*7G;OpPN?D5+ClNZOHa*sm6HuvZwdCq1-s!DA#x^blx8%$^CQ(t(8O`)4uwVHQvJHAl zs@W(hobtwt^PSyKSU%`C$~SuV97Z_Ne7ow^T;Nf4vuhJw9iLEHNYnYeXB%UY$=S{s z!znHn^>ji2tKN?KfY?Y-{Z~Rlfq67d<+pL%p{CEN{2!lF?cJy;{v0m)qOn9%RAws! zl$nM-j0xEr!XNLfw;Af@6AaWM(284BSOyEzoQkc^ck*3W)34-$Va<;;f@)ezlm&hE_ySi+T`WjaO2KQplHy45q9=uw-jUSentoXW_`9^uY0l-0BG`I=~Ji}|J5{Mj*&77&jU zFp(in(D=>4$&3zQmaFJp1i}4&A0%7+Sn0&ICF!w*NPAv*=8EF z65yP-mDn2&dL}_Bx4HUC6czp(TO^b2z5opXLH#7%AjZ_(1_J8=VRz69x*xi#!7)-Xvqd~W-qq)`9HneP&Z}RpJ(EuJ;+CKm>5`uu*f?tmT{J&Yw z)zNRqbw%Badl^^~7J730T+t9RmBKe%U&b14-~h`Vs99#7D@;p6o}F|s9bHol%X=)X zf(0!X*QLKuG35d0He}9M^17UxrDiIaCdx#rPmkMNJ3-YOQ>}Bz9cbmqRmn4SSc_TUw`4(iYx2M+7kBFDk5G9l9P}uimQM7lhwuC{=bU(Of#-p#5q13bPLDX)JmewS)%f2--MBI@_ia z9B(P#S(LrR1rXr{QMLfis2S4d5E21CXyAmd2@m9$jfKpTgET|{VBOJT$({0j7?7{m zq3fm)L7o!2ckeH?<;Vp+1J0DMtjV)hL^1yT0C)&xWx@V;6?FdlE(L}*`n{iE!245n zqFDYh9)tTuTd&wnA6_Xy&nFYOY+WC3ojeuo=Y7;0vFr(~RM=BV&9lW%g#Z?jp!WbG z@s(oRgqd@Zs7a25B#92wgxhtfA=SFdUc07S@kYBbHI+n|yIeGJj-NpfCR9Y82g@$h*CD3)9LZT zz|N&|y9(of_fa3lA5l_=H$kdMu^gaDu?-zRb3qDb#%i*E2*lYLS1@S(U_bJ|5sK)I zFGBsUSY_DQvf;aUZ?wQgK>CFd(sD$!vOm9)8??*=KqaKJP5J|1r~GtFL4{Waeb9CK zNUXVdd;(0pgKNM(&^La8c+MH6`&NjH12_z}lkr!~gGC3}Ns6KZ9FUrRKu7$)9|sx( zKdqsuyA|h0zpHH`NbH_&DSBu9EuMu>ix_IQ5Yf}ww{t550019SX}=pTZjI;@qF76* zqyg*N)!in!r7FQ8K&;G`5P@ge5M>>qpmd;2<=>XbAUv`@b*fC?YJF9!Obz zFe2i)PCp-W_&o)x#Liy4EGl>#j%o{ylnz0gPjBp^-P68x?j>=M5N>W=j zP%H3~6Dc^7Wns>HlQEOrv(5pxM7DI5e4Js4Fe_F7%wHc)s%d36Wba2)dqtm+)`@D; zU;xE^dawLWBLz_saaO*w^-tM}XJr0SL=T=^0RVcS0|`I?2FQ&G@E?Q}O=s5Mou1#G z+$sG^oykqG{J}izkoD5B!rKP`oQnIOItskV1_2aB;?b9XCJ}Phq>b{y!p4X|ji%H> ze-?bs+hy!P3oy-LLj|m-Tj)dWeoiN_jksLQwn&ipy(F(}w#32wpUaXcfnW{;Sd@?e z)2KX{000r}+KKWz7^syL#yyb%%fX-OD3rIXdF||NET(1qmeN#OY`qghSjzv49-J*# zzFaU&rqG7`BRo5s_XKKTvkCN-cK;`~Oyl|T0y+xEyU+pp5&HN5z&%bICdy24rG#b} z(3SwO99AI-m$BaUPk;WX68KCWtW6+5+s_g`OOK1fO00DlSIN$xV9{|nkdz~In(;Rk zcAFxv5tFZo!vK_Ux)?kFS-C&sMLY%e`I&uojxVzrL*{bP@gbxJjnyKR$V7mSlJg&U zC|h)$oVY)xlX)bjR%^;mMKLHZ3-jK3i~!hSA_r&ARQZp$)=cQXD(2( zBj6;2#(dKTPdEpJF9|boLs_0vNO1mQizUa`XYh!(xE|@4vWix*z;E4+$!0hbU!D`FJ)Qz)mC(NjGd)rlZ+m zBukdgUBZhV7QjqkkO~hC)C?`7FdcM4nMND z#qx6Q=#|LT*d{D!>o%?$pdOVK#262U zi`k^WCu)Fj*7MFcr(aY}VhPpX8I%42qsa$a3eL7d?HFmPdHi~?Z+A0nc;U&KgM*sVTjLKXK@Bzt;PV?pHe^3O;y)w;&< z6vskKpKpcV#p~#}`S`d86R^oiag*bJ=2+`OlU2HNe^Ji68EZWf1&BK zh-CXr^_4Krr3uoI^7k)oAO%15AZBt#CTTJ@Ha2-W1m%EdQHi(UuNq0B0Ri`{+V=!& zX5@%dGSVf%FQv{J(so&F!zsAmge!z{_Z-AU z_A-hmk5`k{(kJr27R6~I6-`bdO1%mHBLwg)1N?Hx0f5C`#Qz@<0$dzWYT*At2(WXq zv-16a5&|Od5-QHz-x$THSST&h=7w z_qP9;i4p=I^_$kmx$@pXKJXW&CHCsim>bE-DHYYHLPE5+3}3Ob#~kxCd(|uLb`MmY znXa*rg0q{XIaR$j{f$xA5o)U!muBb+TU;{sK&bz`6R3l6EYrq%Ge-$JdD(5d;4sj6 zF)=)AEn&ZjWq9Eok|JV#j;VH!Mr0%pnIyZ@c-*y_=L{t6rd{e0_frDdP4x`fT|u6_myl zX5jiX5#$bZzd!fBc)WHuYu&g21J6?g*#=PKoXGhtp1n2>th3&w2tdRSr`HPWGM$dk zwO(>}SlYeqZjO4g-(lDH^|TyNWHPf* zeEWvimU6#LQUTwY_e|x*IpPu~30zUvS>{?)O`PU_R0dul%<6_>yH%uA4%ZhCRkerE z+n;V;UFuq<;!k&_TWu7c^ch@0eydcM!9CL;K0zMW5|*nUvStzFr6BE3kAK(p##-vl zc9vgAg+ULdliOyBwfmKKUx5@hiu2hG;yQsmyPTj_o9T1vE~)g>w|3XfU+;!i59Yx3 zW$qP?n9RopY)6syL7B3OnGh9IKYXL>qMhDGss<_>edm*3`d!EI)fk>?5Yox0qN2Q4 z0zC)rJY|y@iSB}5kWmmy>fEFXO-r-gg$s!O2;OnjERlZq!jL~b4Lbtw5&3BEjiyaZ zTMC>8>8@*Q?QV;lu!^A~f2UN$yF8F&|CRJdGH4JY-?=#}Z$_ySb$qrkAcJC2Pee`aj+?5>}93ihEowIJT- zKkD$i7#DoZMk=hpQ(bcPGGCmG+Wf0W+$Z_aPQ%?qE{tVjZDePP5 z4r&iYaz7KkFnTj`T>M#Nc#^g7vF-Px19b}w4Gbq*_8$!dK)*>x3nXfn$bbt7R}dRv*GTy2-BI`^2+*O<2%%QqJ9 zzcckO_6AOO#zq%U!6SOP7@E_Y5rg{{6@`U=zsbRdLty$1@mS1#u^Xw`mX!Uup7Fd8p_CS+2wn9^P6Ka3s@jM~KHNq`E++7igYRP|HGH0N{(l%Qy zUIKEXu8&N05MO!)0oNYCHMmLTUDW%Y-(1O>ZzBnMo{)NLMrJ;CMDj!vskC_-8ep#_ z?~$SMy6+uJN36c*pVpmvo!FavkJ0jb>1nxgj3)Bk;bng)TyVf!7}x1R-F6NQUaQaC z{KOK6<_!p&?{odVCD1spZJ>vo<^FiM*sQv(dBXjXxC}_Ddu5~3U@C949C8;?25MFi zxT_fZU>@4~A#fT*ed{|UmGR*D+hZQ$iAJe;2?N_GD6drob35rIO6UR z$z3P?L3h3;&|MyR|ML5;;YL{lNVDdZ^}ee>D53(iTSnB7BQX`{{G6pd55~KF{x80TD zFuyXc2PX836Z4i@wry9JJ{Ao=xtu)j4Hi3}g+HYoUf3A43l%Hk30${lK69`IXmoi0zGNb&u?I3iLFWcHq9>B&TRGadHyS z6ysGU8`FO|6-6TKY-0aLu(H0Er@<3_3g&Nc)2e!%`?&IWG&3=^h%rg(F|aqjXn#7* zr65GUzvLWhH{ADHm`qNdo$g++qJHsqRH*ad{hXSu9_SjJn4TxpUQ~N};Y`Y@&(w6# zwF~?${F)`&^v3nVZyRa#+itHdzeJ$=Y!ad6`mQc@_xS?fC@2#@`ovO=4alyd>YEm& zT%y6aguOWLR5Idu6O$QL(f~7=uT$n+=!-9F^r$KD%jo^a)tULrLMf;B;7+ku`Jt%u zEksUOaIO7N@ucW4M$^V3|6Olss}4S=br%unA^ERy9`VT1-jw*9Wb0m<;|S7z^xIl{ z{o0=wvX%Sy_2odm3k6nPJef>A56`wOiYwUHoNr-nhAflk!iH)DMw}Z?-0-+|=}9kEGZX zFXX-pE&co7Y+Kf`Aaq)ykUwHFUuFK9Udb~^UzKUv;DZ%;BzyX#f(SW`K6T}C7;SZ3 z$701APV}O+3Q>PzYPHsFsrfD$+=*Ef6M;U}XIX1F5v$i6d~kCRYxuZ-e|^0l+25Z6 zf&NLe{ZJGY+9*xzs`}NYQMeRuq6ggfmvAz7tJ{b@7tL@Y_?zu_9~({M?{nYFC@5zw zLPLjHgDq7*bs6R6d(z=X=bxtUo7M7zX}+3r9pF4nL!{R7v+8BrL#`W4*9A4m?{`g! zB0UYY%Vw>d74ZuEADX=BTH#*S;e_u_u9Z|mPjqztqQ%B z_XT=uz4~!p7NZu*7~q(-YFH{!$|v>m&sq<}24Bl!2$Y(6-%aoBqd!)?JK{=98!gtn zx-_3QAKzAAkyY?;JDUk);opeL7hZenzg`kHEuKLl^Np7iE;YZAV}JRx>Sc-hQ9dbY zRD;|bgD}Ac%0(zxhSAC=;r-OqmMV*TV@buz_mp8Wu~aJ_ zb#+=9B}0a1b%Fiu#$o7r0L%0MUkc@YqMRQVRVl6Bqe~=fBjZd%Rc7j3nT^xsNRsWD zEaTEDwQ|a?7JF-{`NAYeJVIcv51p>nQprvKvaWamn}BdVE6WOj-77=Sb24Vxhv!9a zc->4Dc78z~e*Wr@;ExM^E zzbt#g+cu4sp(EfaUN+=n$K{)KZW)n~J1C{TEVk^$O68-|di;3_+}llgv$Gjy`!2-w zMg`oZFpr>6T|CHeF%GYhtgGxuNwyctD!Bst468dKRNLFyKqH+SC-N?-+8hBhP% z#$0Awt?-A|0ukdtcI8gfP zXZh*Y)_sDMR@4qZ;{>wF4yp&rV1FRM@hFntS5>*{V`1%G>SGw+m0vPFtIv z%*_!Jtt`gC4g5e{8A1OiB7@;)2nohOyILA~2r|(K`G8m&{k4sBF+E6RsXm-;p(lz9 ze|oa&ol0(9V4954$Ac`qZI>OjH?mLg7Prfvwu{6Ry|h(o9p6$XGZxgG$91$-6somx zy>D{ij|9`)lWU}O>~L4rO2p>4YfXSucP*Cn)WuxQ)02}q2AuDzq6te2xr#9oeT=qD zT0E0o*cH~Om+3Pj&60c}-ytdScjGpfzwyxYj`4BEOUgxc8(jSJP@@oTm*QOZ6&f_T zt9+|7PL90aMF^_p*z4hb_DI}r1~+i;xCfZEYCj5G`vZSdvDeB;hgchzM{VB@4V)ZN zel8Z<>3<^{m#?Ir-+_(52sF(*bSWx z9#jlSEu@!Jx>KD+%N8n<@Q_zXKH@v=_BVgVtgoEtxs(JoiHE8Mjat|)KpKB?+B)P| z26AYJFt=%}W>aa`qiovR+>?5C6Q5yvLBtf{wYYUv#?tCqIwFw zuS@f{+@4FoHHy8jj_DtBqz86>6#bD;U)#uOT&H0A{lk*%`y$#USktlmrcubH-ld%9 zF215suM-!0V5CyUMAP9XIM&e86ABq*SZDV0@x>0f4~E%TTTAD2*8Gt%(piF+74Mn! zij9Zx&V30pKaGu}Wv8m-yT1H9NZPYI4lirmHwhc}Kc$-6pI^DDDvOMNlC$@;Z{ZoP{D5UErg(>PARxy&Xeb)t8i+Ol(G5R@rVj zp_FjvviKz$8u#3GAJ&Qxq<2D81Bt)PHIUA|JbpIPncK|`ApLkK(Ji8|zet3+gprH> z(}X+BmSq|7-mkE%0i*HtHm9>N3#Kq5o%g3~4|ck<94@jkhvd4IJ`SiM{YX3HTcQt+j;mselxBU zS}#f365l5Hn6DGsNQF5`^%cV#+&;whENiwG6ZZ*JHP$sGG}O;&MtCXSgidPER2Nm3 zP?Nz|ltaN6q@^_pdQd@&|#k}nIwvmgHnwm)-(_yun4X?N!6BiTgV2( zm?V!R#_S(g4OG>A%dFGDGt^uTSIbPVS6+xt_%)3X%Y){ zT^33PO#{=zU610;aLX-}$OrW@3sNfe@2#b}i7|SsPgto3kENZvo&{WFWxWNcHYU4? z>_+(~k+e%Wk;5U88apjFAnu{NOTCT_)AX@1pYPh-1znYdQykSdm(kI2gR{dveD=)} z7T_JQX+DBciGbY^Cgf7Qe|D%Rg^qc!-qBej@h3^VTHD4DLuVYgB$bETTFhBZC7r*7 zvjO5qr?wNTBf*kr?oU@5do~Yi?B>VC4tbW;qXiWY7KL{vM@>@j@C-r5Dr;@kMItIa zRS@b(n zXS>?1hB~w+WKx2k88}UX(>2&Duvy(_YIz2Fsi~-LLQi9-r!d!mUsY6W+~RDq56pvG zL^1?S<%oVK#Ix~!c?UQwkP<)hf4Q;SCxvDpWhjg5=K+1E(8OJ$b7OEWsg-;(zgdU$ z^5i%_G-p+4E`A2x@*frNjIz!8mM4nk1Pft9&>RH#r+tk3C8*}S|1ViL?2!u)iOndP zN=>3K_DUC-5HyP141Q9+W*S-g3oWPGSCv2*oxxY<54RhIW~OqG5FNB@;FN4-EXa9b zA7I_p?ca~!s;P46dgcsMVANV0)9(LNK}#kEO3u6oH~{cH(srEG>mr;3WMTcos(BhJ zbiMfHJ-3dv-tm7-1FSD+6ezebdiBFJz(NrNJC%V3w%*>R_(%Sw1QGyF1S|%I^FmLL zG1C0|ErgY{bbRW;qx^s^T@8(077gf+(3;_ z7_jRkIL91U0Z?kECE0-W;L%#pWro17o6&-CqDRGf(f7!7N5(m!iYXLdYlGvHejefr zM!#)m-2az^0b~%xl>9q4L>C)8Ih?{cL9d~?nTLyOod2$9f5o@-p@}#ypC7$jKL0)8 zPvY6y*ga>*5sstT#RW@?2Th>vY%VPkB}{Hgxm)1FqDS}RIB*7gY2x!CGysTv0OOce zvo?99!Pr*6V{@(=O7 zA7S7nY4E>$j`%VRn+dZAo)doE6W*^t}rBFEq^m7oD zh|ypEmoRXh9#3rC98ZV2+t7GHPDm7qI{#DA*`PjwwkN5Ewv#JGgkijvJGIbP1Ot(= zJ^LZ{hh8}Z&Vl5yYrPJlzXFdw%#deIHYf>Ah$5#%ueA}_hSFgQJ{FAXinI!Gq^Tx=>B5NxBy zM{xhA2%avO4h#YnxCRoVj~)6r3vy2Qk#9(VY;MRuz#xxx5}IhN|33i(b`?TRF;i&v ziUjs!1HhY0%;P;Qz$yU7M5NnMS4vHb!M}}}rgzDI@UJH{p!gFz_2)qzZO13Y5doTF zpb&;!zd%@46!`LhJx72N$RZ=#jX=Rb;oXskxo_cPL*;Z(5-`(YKryIm3I)rdhc~d!SR^A*^^kL5Qp0T>`@b|6&#P1z&RH2Pt1h|PbUpqigK?-qRk(89xYzyMA|=0tHq zeJPYLBS;ayZR5xz90kU&E_Kj zurr}j#HUHVvT^uymXFNAAVH@(cAT;y@Dz~e7zG0$2?4Ae@IED@z>NAc6Z66gazO0W z^_=116#|q90ofk7tLPh!zoypJ0E1%Ycz_PUiPnGgk%mVwGK z<&=(R!C3tdUx0!OHO)=H*)44d0GhQWCJ=xJV2ESsfGjR%$V1$@Irq!9tn}#0>)Hpq z@iqU`?BQBsTN)Gyf;QAKs=L18oTUw;goaP+{6Bgxj$k8!cD%oV2uSXMpaKBPCTfBN%pG781ZAm~;ExOc9UbGF=1jU%7?6XmUoy?opcSjuaF zPBN12DMOxLXXr5-IRF4H$_8!*KvB6CA6Jea z^f&?TjeplO2o5*Ac+COpy2#l9UR@C{|0*8`o|Pd0*26v6rYe!8NCI?il?0gK05paB zUU7T<&J?`f@UmDQ-WyW!J#%QCU_17)X;QxUvhre=Exmh(j zT^&W|aQQ;R_q|~p2*s%GI)DFLgm5@*VR%y_xIwg}q-9Bb$7vNJS1z5a#aD;%5lMmo zT^uSw0KS_2#)pcs?uF^cw3>UZG z=QtHCMKZ1JhJWYZ1N^%$jGV=D+vjXuG!c*~ot@EBdlQa_GZF-hz3j3mc0{)t{N0ov zU5wRyeAH6dD5*a&exc-MmLn&`BE+*D9zQbvG&$Ma+oMxf281jx5)vYc=L<1&%aIq( z1WQP{oe&kw6-osk2u7#QSBshY-WIyJNn)3orFc>T8e` zy!$tX;bC@o;>ldf&D>8l7r`kt<;oMi*D3e3WfbKl+$-5Ul7A%$@@gFs#nal1*TLMP z^of+OsxSBC_Udo>0a$PE{{aR=R-0<;P{5$_{}nKxf|}s}dQ$?=|9`wGfrrJAkJXTm zkJXHcgO{6=iPMOSi^+)3jFpLv&(PS6i-*U=kedhkrUZ7Te;fqQ|0@UKgKfInaq+nZ zoG{fB#)Ma-{z*-HDx&WD7rGFqu$|kEdiivm;@17nN`3$Ej-P)RtNcUvQ5rVGkdbL` zM6hDJdr*}dx0OsL5F{jg&EpEBExiol2hN_9Q{>FDpeRJD`70EJ_?57|5&JWSRfT}z z+3R&-PKQc{+=nxb8F)1}?^=@&Cd`Y3T6jK7h`cP#a1bNYlE7ZNKEHNyyS?RgmerC_ zlv0=eGs@V@UVjPG&mc^?cfy@(T(ObYH8lyG5#We)4x z3UH%G+2#t10XtXz2>Nbr(wIAxr3#=c83qQwAK<;*sa1V%|7w2JeD>(~MwwokiRb&U zBG2MR^4v+zunUCftn}?&5c>oLENPH%%iRid{~2m3yBluubCap~KS+D)s5ri8&$kmo zkf6amB*ER?LV(~7!GpUy4Kz;h1b250?(Xgo++7-Xnd0}m@4mTrX5Lz}Uj5U(x=x*{ z+Gn5B)&2SIy@lW&J;(XRp>eQ-RW(gJZ{S%T{h*@{7gFZ63C>z4XKz-%Jp(^Z$Lv3nq1F-1pmK&5t>sd-=rm_GeZWmiV_z!+VH4OWd|wm-RN@(aenSoSbK* z3KTb0Y-FlU*0wSU2;m+l#a#)np9;q&vuSC_9A636#>OU*+)-ZTt@nxSoQajVgC*zT zyh)oip1)t5pnAAP++Pmd2Ma)smk{8x76_Ma+Dc9r?nCRpw(eg)kZLeHiH0^6bbeV0 zO1tqGcU;@ckk|USd`k)0O{j`e9R;8cg6OwF|=7f#LDXNrsr4r(K3V*GLAR+E*2Iw zV%JW3c7%y9irr&n*2QEm5=Y0waX5s5x0b_ZK3wQL4SF;6yP%eC0elU&^6YkZbRxdT zVO8%w{{6W)J$keSI_-IE4GE5rcG-R^GcJ)t?wfau+YaR5A?9&Ao}E=KYc6cGJ=l2W zVfuJ>*xPul#pQNSwA1=Nyoud(N7Z_3vHUD-(9?>b<@)%#f`n(mZthp+D;9@|_EU5p zf6tCszH^9W0_E;NTDaJRWuE>}uWgU)x}Lfj{}C?H*oY^)Bigz@%R7PP;-Y;_sc{qj zV~y6 z&bt+*Yoxe2ovnrn5E4J(7+XG18(SZjG}N>@Ynczv=xmgs{Mo_!g!F!-_-Vr+1ksh~ z_c7Cw6A~WolYH^G%b%4s!IbSQ!wC94r5%ls2&MPJek4v0HX_|k*;{Z9E93fq_$v8( zeH;?kwPYU{>WHt)xwv(sWL%Co+J5(#*5Fy@a=9PAAd2LoI)zu`FfGeZhhlQM)5yc_PMNAXy{AJ3E7tXkWP8bF zx{ggx;?9`mdBG{9?y<TEN1A!+nJ#E z_|fBOYxk5btlDBL^#C11n zR;Lj`MX3)zrqIi)PlJ(px}jYpqdKRTHLt*33~mlj1sgoQT_<_5B*#@Z@g}ZMtxVQw z0w3>4<}`UNhULqjNeFUVTrd9ISX_tKs`2yFrOt8g9Zy!ugpe_4{<#=yrhal1w)-39 zOo$Qk%5PLsui~vQdNlPBuvY+(?SI{!)W!y!LT3s-03kR zqRV?JR_cy)E87I$^-;j+@d(S-?D$SdOiUIt4bN?Q-j&Qcgo4A9=i$mc{O6~}(5A)m zX?5Kw6_>3v4b``GEBgejWieKlNrN>Ho*TocNA5>7!z8(?*q^VTZx?Y6h*RTF`r^$! zu2ziBw3b{K7RgW-?GHD{I=Ey#8eNVuvRYBYiKZM+)H|KCh&lNV$_XFy$T+4NiizEI zmt`X!?<;r24qL~}8!m52te-gRI%^r|kQ!|#Qh$oIswHhbOAijFCNLWBZ|QbO*YY}d z`$IhD`M?Wfw~cxM!=C$5*RsP6o-4E6^)RYkh9yZ(`f*O@$4}SWqt2VS6FyGJotMy72UD4d-x0K}4v-Kood=h#YDu@}i}4XI�&cYi!i1mSRuoW7?6w zCg`&6XVMd1^}6C&IxhTFR0h@)*4;jJn`!NO^VND2!Q(CBM2NGORtwxpNH?hXhd(5r>sM^~(UiMcZ@6ZG=;Nh8?-U>UT5HpX7O+r^SoZn8}5|2wa z;5^!GLUe8@5MSX;=Omb#!g)-V!H=6yM16T+Er%g-c~SuLpCMcHe`3i~Rf{l6mE@wN z6-b}vCA_SyU5Fui#86*5UxGePEL(ETtMXmy4RKp4{ZN{)F5|OVkKi~CIykwnM=t9f zd1#fF^@Khv#r^3x$VcxYKxoPA$TfB^5wCpHIE= zT0*~bKN@JdBvo~EOv28}@-V5!HS%Uh$Q?7RWMB~6)lxC zN=O82d%Db5$zRu4jIY;nY}c4ZHTSWeEYU3Vau~@1USZXZE-aI=54f_bpNa0yBT(M6epI=@_kX#nHnB-`VN}Y@s zr;go<3y9fLAXIWZGB<{0)jd*M86_S=h3DUxD4^n$S?mny-44pqTWqk0JOuG4o^qk$ zX;!#gW}u=}7juWCC}3|GTe-};a+84|_&`v&3!148|I zr|Yx^nFX|D{&qr5%m(AK8k_tw&!=jHVUuk?2xpr{;TS+$VB_ z?fJACa<_R;E@viI!R0b8suV-+_Rud7vGQmdnIl5!jOOpGH3Ay6-w5l^R>#J3kr$jG zGaj8`xt7kn%=b!n@RQfF4wQQY_EDw_I}tMPF7ON*><;oeah}<`F2>^~XP}hmK=zrv z6=)bVAmO-mr}W+RF)Gt#;?#!Ah=6E3JDd*NaWk8*6dm*(p403dDfysBb7$_cIP22X z#(mg*>lKTCuE52x(BoFFwyd8vhI^-tVwdFe0(&04zhdpADE$NpKkt1rBqlCJbh7?@ zlx4GCC~nv1HA4`y=p}29sF(z#S5)G;SCf^2ntUMs!>-Ro4h=k(%C|5HHf`3Gnn>2x zyun~qF%bef6>}vEm!@ufJYozSQ)F4$Z+-9)p|w3C-2{^unY6zr^b!&hCVKsbPD11J z^H1V0F8-c4NNPH`xI4(|CwIS3`VkcBB+mX}02dFO!gSZrS7%3}iOrON5xVX7M&HQA z!F?2$s{cDxzn_%8i-8ZLoZ&xv3>DC~ zHW!S)*{!rstYqC?ka&g+;*=m((9ZbSWR34K6YV9|<$N&D)JNEXn-zH=8gF3TgKrze zk}&0@k>>umYG)qT6EuDjauo%bbMEC-Ny0hGMpwFe99G@28RGY@|NI z9s}2*O~TRaY_fSx{2;mJtcfL0AwSUoiTtyDYtU>Q@rnL3hvkI`+@k88%Vcd$>jFhR zc^*Ne#eJuv%8J1PNofJkzMbiWErHE#Lw7#<;~}J>bR)kaTU?AzZHZew>;bYD`s2}} z#3oN$KZ&;pYs6eKPqrG3{WeBs)SAv9R_54DgPp6=V{UWrYQ!H$Q-%Hh`wz7VvXlYx zxVLfA{D0Lrb7)fdC%d^kK1^{nHy7EvX>qJY>%*w}aqWyEIFGDAwA7wPMG;|hq6Rbl za>8?FdpVu$jgo&I{jI*K=$A$qsmi;imxO+`UTf07iruWpN!!!c`9eMMP8GS%-CaAC zY!F{QURCp#ZrC4)L^b5S9_yE+(uTJ1f}Bd0GdsJAV1d+hR6-~Kl<>;6=8fmOBXRXcmy&n_Q=4PfeEA3(2 zv#mAyHI%*^jP%zs9HpuM;@Df)oL5eDE6VgKewfOny4`XboK@MX8#%31WeQh&bym1H zW2;tQFt?p~mUeD;l%}C>Pd8$Hl(q?Vuq*#wksUTN{3ln!eDixqF|7a9a&&8J{;ri6yk*R%(cqt6n^(P@v9w_bYuh{ap%jWp(?P zr1b9skhBlA6xFL)sN>e-`?97vBFIned~8e9Z^^9maHQo>>y_T55Tf=&OfOKwF)!}K z?hyCzJLe`e*`9=s8L%{JC>}0$swXOf6mpOMRVE~2Z@O=3#)V_PzBIl=gvIYHz#h?D z(uCCTuE393pGCz+pBImZH5Nu6dZY_ON3l81VW?D! z;>2-7!U1_>+UDU13R)y;~xNTg-2uw5;(`-4w;BxWawRh>f3I1%_9ZAKA0+m7j4O~2T1T>Qq3sWK zp{v&!*nVIgpOdq#(cYw`24W`Gxwt}LYG`Ymn>j=5z~#;6nU&=m7O(o%UXJeOvBQ=O z98fDhz@`;R%el@C7W&1vA*z5gorXm=Ub}Dess* z(rzvROE+kQJ-$+x94ggaUpki(Q<-F2*~DVs;S2jbaVXKr5=Rt|k!oT+Ft^}%hQcWj zQ^eNujjF}IvI*wC%)TD;kSUd}yZ`P;eNVFU_U8{Vnz7T`=wlK4#c5>yY?}4nB>z_K~t+K~_axq?m=E%DLa-{>8hlv040bJl;yPRJM-9aDf#2k060PPNtqJx}f%( z|JgBP`v(z{(`lm(4SWM0=6Nh@P68ZSj+U+GtP{a}r3Jq<=p`lr3-!4V}eyCG>kUDoSI4Ruqz&cd!A8kNge zfzd*9RY@r>V#`5%o&JnrYhSHhDH`rfeb%)!;I6Q}4^WQA_C^n~yD?Ih5Jy_xx&{J6 zzm)NKhVp2(kU`gP2$!!HyXAb$e`~fFX~Ahycsue)30>?#KO5nj6r1-7*|0ZG>W&qO zYD>h48p$bi9<;wC?CX{cbk;xQLEL9D3h&`e=d4d zGefPSE{#h_F)n3x2JcvPX>@E{>~HK{wJp!a0KpjOmvFVx9t&IfjHHmtLc-1X>@z=x7KB)OtI zg>H=i%NFeV7F7`|1&6(XR0VaI`7tu6ko`|RhhW$p`!&G5fUri>@4h)-ff(PD_a2EX z_S~&}-#Y!(F26!pF|*ao3xyXXyW_Zio4$-W*@y)4 zavNiO4HEQl!39krGJsz?1-9ir$+rQ_n<$_kJIxVYbvUmZ)J=gnVWL-= zS}!0;Yh3>|+v)O7fESKw>@o@iUU;DTNhS|UUhBX(5Gd`X;3Dq_&=Fv3!hSXa5F9{g zQJ_y8=-L3a7mTUv1}Ee5iss+a&^OV%U^V|w20*v}ZphN4Wv6FgGtiU->2xtop{8KT zjq|H?&rl$Fz}8yeOL(;$kE8h#&7UHo6HSMT0_4^xU(wDe3RZ--jG)v&m-lR<`N2Jk zX)Pbw_*OngzfOd&Wu3xnX8{fz!$sr=ff7mh+=j*JEuxeKHOXsYKzi;I5_Td=DEo4KdAQ8&+iBmvNCpN4Upmf{%1BSvH z)PmGUK=W`ypV2n^BcHHd^mlPSEK1ysmlAA zxf4SIKlA5T^e@7(X&9{7!rZ0<3!8gDUK!ZHq2ngZFd#FbBU$%E!sbj6W-zfd2!!dQ zYYIwk(vK(c+7{|%0D(*hzv9WU1`M;267#>#huJq3q;pvX#(7^>8ixF5_$mT#wX_V! zXF!#(R?rT=iqiTCq$DFv1lYiZW?jprD8OCPhl3tet+;^p>i7xdwcYmXgY4HAbAl-0 zeI0&u?k*;p`%$>CM)M&=*sQH7X|176gUqsONU?^4dQGQ4_Q8PC<&_t-6&=m7K>I&) z3Gv@9l#rOseFKfd|~uihMrSv2i}J zv3TbX_(n;305byN6_$c>u`Hc` zkBBuLC|x(dxxcs&gdvCOqekGlM*>=<`dJt=st?X`bzRkCqeJhGPM!y2fdrL#*?)Lx zIMB8avJOzm3b3ux)y==msW!#&%=6MzkGp1h;?5XQIyr6=EGVA^U)O7AtpRow&umU9 z$If?vY8y+CH6_5cMd%1#Nwhu-ezy#cH^@1x*Ho&DcDMt^ii`M95YU|*;@eOTuYF>X zF~ql+zNLW7CjA4F*%zYmNH|biBhzPqt^>0QS(+K-*u6pIjHyOF-~muuzw>WGxC!2g zKZ;u$Tori2xd&lvXW^PK+1Ov&r!%=ISd))c?E(@2*~~sBU635ygC;2mWHy!t=JrY~ zywu0v#{uQPwbn?-D>%B?IOFfi9@%cj~3jBaT3{;(cbA>xr zXR^V;ZHAX#a4KodmVQ{na!n4*_Gk|{-`ewa+yF@o;4IRaBi6Iy#z-KxvBT%9AY+FT z%8NZnB?%v}4`lYW48Mf#ugwdm9_>t*`Jh%ldO7xYcjN?@wf4aKoG_mloe9|I)w7LP zCZLo>oPso6R#-@{u1shJTK4v20-cER@&UF#VGK<%H*xq0qEu{!oW>#IgT|0BZSOPU z>ihG}7{i=Rx`cCo7Lk}pBwqk&Mr`;af4I1Yoglr0-?It+g>UmtH2~hS;u9YUT5{&a zgl)h8-PgjHdDSX|(&w+d#g=6AXMrRXU#)>lHKD$|_XhMGGpo%SsVj*;ZunQ3>NH=q zDNa#tD|PCts@Hi@+fcnMPJ-uW54~Yt#&W`O#TwFg@a3fm@{_N;CQge%sa`){Q+Ego zf=+meFGCo$19^&6<*r2X4@+PY-#q_~&$KOi1BzK|0VEb)Z`&!6K*Qd~)QF<0%pg`i z>%$}EqXQ36bUITZ?mVO8SC5E9(|vl0hOb5KIqyJTg{MHkeAs;ey7C)*WNGQP$}%F|%(LeSwKV&p=qV9mYB2#XUadqd=J_ zo8=vl!#DsVh-K#ftn8{av1*AVi~l5n_BN;BoNJF1Cw1UJkrj>_(5&tMOahi^xk|U| zFH}1TRG>DPlMBFXj%T4OD(CQil3u|DhY6pQflz<1+7D;G4U8Rt-%g=>GfhRn#iWLE z0~t!7B5@QF1r|Tvb7l&{th;^5vg2#t&3LtXnGfF_82FIM0!sI0$_Iu5NVgIWq&;dI z3`ic&8Q*l#dy^j>x5Pj8htp$K?WYmYz-+`f>+$ZIVB-9{H(VO3m-BiiEXd#7>w3er z8yNNLm!}7ynaKJaph2Kk4Mguiq84pN`|%ny_OtMj4DSoq6jrCD91+%IG*%uDsO{aw z%p1gw^D|ntnW^OB#i`fkqT1eMm-GkGZADp`Vc11%)a<;=dSOTrK_OKEc@&=w{KR@R zHZs@4_7LP_zr|G#M-7&wBqUJL#r(ocMaRIxB92NWCP=jk3PFj8I5rKf{H}*V`+GdG zP3U&p2cdf=^Ca2lSI~@4M{(2QufCssyFdGWS?zVJ&BdZgx=9(hJ7hDpzfU5@A<`fI zDgCR4hJPepmr3IjSX5m<^b}jNp&mTuKhlmbt^bh6b()z*PUh;e`3&=-8o13-4n+Y} z0|Ng=H9-I0>MQ)0!Ug{qIR2$@!GFc^|E+KVoIBA@ttcz-gZ4d}xT}c_+b`-L$|r3)Xs#JfroG;oeX+o#GpYc8#pP3l4!W zWgdOe0De&tjG_qHlc$F*ssK%f;c7_%RGe&6A5jyU&j`jwP}c>TukbN58#Fgu49O`m zzkkQ-N==$vIi%H&k2dsi>#VVroIH(htkgu~OlY>q?d8>))$YQmBdNbXThy>*K&c}? z9?J{)h)4q78}quXB|;{yy_(k0*75w&-Dck1g0~E_;I@*m{aEX}w(~gBeDU|9cw|&_ z+I9ZN=~6a_XIfeH=j-TJ#%Wbf$L&R$5-oBZ{)1sA{&CM*OJc5)?#PrVWbK7&=cmZ0 zU5)M#V(Z)ezHr$=qm$e@3gIGFErj4X$Vu>5qQ^%uiDX_yr zZbE;@k2IgqG{qJHgCQ_m)0S@v$-&};%PdA(ZyHTl>bd#apKhpS-(Pgh`R2>HND9dD zoX3>^0eOt)<@^j2oIXr=DfnOKtzg*WQ7^M*lqv}?$HFJY%eitAo$q<6d-g^2e_&uR4xL3P-(cJRM3BR-J5 zoFH+tS|%t`ZN2NMn012Xt7wJ}%`VV!GvBt{Hj*E5I<8%Se{WPA|Fs!TM)%xat@6lg z&^o<smrQ&)+>eyMMc20P`o*x!lzJ z-a&;ITgIZL3%Y9#vZZ6wsR_A6E#BbTp&Ob;9IWaPGjBChCe2_cIW%M8Vr}b92k&lOc|cBv zWiMA+YC0hk;IAbf6-OKk{KP~x#xs@H_a!7_ETxmbGE^7WA&|BT{`;NMq}z!Gu7mJY z?)yiQw1?WQ!7IJgeZR<`cPsf%;&S$P?e*!=Lj$&iu zheGM!{LuoPQh0ctIU<(1#y4F<5YA2iu=$A8(9ouVL2xjfjUp}XUIQy=mfM<`@GMw<>)+*1<vBR3_<@p8AN7n&Jyi&s+h!%(tor2bL0-SBdl^*bbE8 z9g!qe=(zu}SXhgl%3kHk2ppHySJ=qF8+ShJMaxSFhb_1J5I(v zT+-{|SULyaB7(SD?}kOd=VqRBI~a3(3|KLfo(+c9w#)( zNx1dTMy8HtV5syHNsuhb!nH7f(-#tPob#Rz%x!z4C&)et@X#*{Ky<%0J|E&Bb3pJL z4x(hcg+5^n35cYjX&jVkH7fE_d2${W8J@z02kST=xs)#5_jdJ(yhCx02TQTGJSi@M zcOrh4Uwf8Y)QvKgUz3 zI7Hy;)jbc+TVKXQilyRj7D$;s*lsk$kZE*BV)*)SRB_T`FF%^573ZyBI3M!gImz{Y zTF=Xh8|A}vQI~=kUpKX;@lb_R#xq|94pVetrqK{TthG3ImkNuOKyOEQ^uYG_I@}MA zNiN1at{a{*_d={l&6m*0ry7OiRatry`I?Xu5`m$j15S1ne7L5sT^Z@A;h=XMEG))w83H9(wifB?*KI`}#ci#lC(#1<2%n1D>9<8^J$tuwt8Ynj_-xQsUoxr6o`1+=COF74*?kZf(uiSo16 zLagQ@Zo$2Y>wQzHaGKhfCnO}Vg&({$e69yhX| zA4|p3@~ro5$BNOxe#iT^jK>PsuFpv5ELPZHTEgQ={q?yPy`&l4%;z9uwo#0hxHLAp@-?n2E2lH|%iu(nFtEUVw2 zoRx)(Cc4pA*WK&Isnpy~cdRE_`Kgf;hAa^_FI|;W}OvUFZ}Bn-egxd zNygRe4y$AtKRZ;GL30`IO-HovJG|CrkWkUq{lZ8*AXAO&LNSqiEIk^P22#B^9wi$1 zABj@-4j@UyKSs@0f)^6jE-jux5b3+MiEi)k&)7}3>xZB?am?_`na~mN#}2BXY1u>0 zsfzv2P7>mwsSQ^-ZS&ea+UMHp%k-b?)7E0njaOz-S{4`?&5slRYx^fxk|L;{T(IfRKKZn9;Y`3Ef0CyfXeo>& za9@)&ceK@rY)x`!Kk%?8jBJ_rbeFLb${*lavf|d@EZ3wv8(8*eJUF5ou(!5=-Rn6$ zdy+%maEWQ&^9B*2BM&G1_^KeDX`=sl)~HzJ{48?`osLL#GI&R4i9oJ9D3wf0T9)tX z$Vj7@aaQv0KADt<@vMaaxF&i~=14J7&%DNjRzPD&04Wx!6BXiqDR=c8=L8?54xbTyWr0IJTOe8B+zk(vmG%crGa; zr9B9OqU@J4S@hqq?$+0N1}vlvdp;I47|1Ni5O7ICo^Ik4J(abc@uhpipHn_YAh%4= zaPAJ^dPw>3^Ruhp5s}b6M%c5-oFLB3VWUw?NkxdysxtG>nYI(?>0OFOQIYdA_nQQV zhf0RBix`nn7>kKgB)r#;yin_B>M=2`Nlqr*)3=i`aMF~LG62Gt+zn!xnAn-A*oD$e zSbHc4vh-6_4oI>h!dpd1z>*Gn^{6tHxXe+P3H|+d$$fvb{WwiyPA90N>@R+g_b1G8 zodyN<4Ec3^^8kK8NGTXt=rx4U7C@=jQ`mbwzp~2s2KP~m7 zSP5adncteVp}dDpcKW|vi#<^CP)ER)5tBJW;9YHcMGNm@R+9*iK|0G2G~lC?UEVq3 zNO9k*-4WA@T0&v**3Do2u2kT$kI3^IxH~zU`CsnH;810DTa~RB7lxhMbQ1qq-x$IzirsOp`fXif^}{5SyIjgvKiCjY(@dXrx*lCsXyRAl ztjI)ZAQ9LlUFMxLUwxFsuZoMDrpub{d$#=a#Q_z8i#yD4H(iA@y&BotjI{W=8kyvZ zA<0lEsiNJ(+t4?u)Fx3)mEBGHmrbjsjTv08-7IJ9J9N*B3S@?(P25zBZCp>*MJD$< z(KWIAx5vxm6a|cFe;gmt`hXJic)*Pm?rkr}TWpIV3M6Kuhb=EI}b*gfW z4hy@7K25y#;%-4)qnM?8Bp?4N9#FB@oy&9!JHn!An3mBOttDq51!-elcC8rUuV<{x z@6yzx?~4nN5ak)Q%khXMv@fON6PPT2wLCGUa)yl+XRwW>zH=qXK#l9{O6?s})~qc4 zyPmPxz+Ja~>8w1Ob41_0ge=q}y0yt9)-;++fRCdx`(ek&{a`0H*F8(W@G^>bSaJkA zOKpDl!9ua2>T7C2my(jjQkqk@0{<34$YjXynUbQe(3MHpDus1BXO!#I?)!ypAlAQ8 zUrSBU$yEL~RqN-acr`sl)ZH}mQ9{VQ-+R*|Z=co=be!`Qu{IE@LW>ngzX!!co@JVR z>6&^2cQz9{T1cZb)qSWf#Hktg!&wENsXxJ7V`XVL#{*p1S^71CBr-v zm3^N_X`v0)M%qHAEtZ3i#V@r3nCuVU1GNKUnX}VPnxg+$ztj%kPY@AF0RD@{7TRzx zO$eC`%hjY(4a3(mvH5n=9gdFb5C5wDw8e9|oqp7`T?3~OELyaX7v@6Q!O$hbyRH{4 zGg&WsZtBxGz(wKGPa5?gnWe$>)ue1;OtEj`E8&znhx7){;hkND#R7uP(yJ0yYYCs+ z%7ePRNJjC7V^}UNuIUnCzYj}3+FyQ?>nXkyE2J-#mrP80a6ItHc_|*?04-Ddu+}63 zw@H`in)FAiD2VeA$|)KX{VsMmv3S}eH)cDWbA`H#tJsD(7Dbv&QIjy#UieLlm7?{U zlXRC4d$?u(_ml6D&%#9ut7OBtie$p0wm_1 zZjL%$Ge~!}w4t>x#6rVAE=8`8HC(Ejx^-=MlZLNOGkrcP4JoIO)c8zMn;$VI%zayC zll)-FfwZIe6fDZSHH#c0bL;C-oYypB+W@QLS$Obq_s#EFN2m9@4))S@vp; zMII`~`q)Uzb=4FJXNUfIyHoRhsaqnVpJr)E_2>9ksiW2J1()PzYHOdDDAY~WyE<0U z@=TUZ<>nFMHng&x`*y;iHT!6f=`}P zUy_z=ne^q$8^}Wa#!pZ&MMT`O7=vj5rkG==JlyZm$zc5y*-+I0af%+6$1My8)4bpy z%uOD$iYmq6tcmsfm(l@*2g!ZdT)$Idzi#_+y)W;jmJuI9khE2DC#)vr3IaNa`b$B< z~p^# z=+6u!L{QDV(U*7(2$D)*l(&XdYevoB?5IS;;iO4P|yqrpdlA_iu zy@^GmF#z7PX4B#{brQwmC{eEjU;Q8m0OM)Yk;+2ISD_t!aJnAz$}ec6$TF^!sOCRczXqu*!D%YkVo5Rsy2*m)&|P zRC9upJC++aQ~u%|aU#kvl^8w}G&$HeWpgPIVrhvTsg67V7go8SnXDX<*Pj64Wd90Z zW&C6!wk^pv{7!+CkY)v=7|TfGan6;m5*j;mZk4F{wp9FACGX-a4`+e9+!1`X#$PB0 z>OqYJtmT(3j#E;*YKohSYQT1 z`nRYIjSJZlDsWdK<$fP)z1;AzhA{2FWdjU5>7CX?n=l|q@2w&c8;{Rq{O-+sqLPaU zW~c+sQ7o?~QsNL>kEz0q5XBb!n}o4?xuIjx;A|o{!#_>J6i6=+J#x$0D$<=TmGLXD ze*WzUU7gr`y}s{O%0d#gW!qMJQx;G+ATy9&ERlii8$i*;i)tR6md}*W?u<~-J6Ibg zn;kOHJsXtyavF4PtmJq!kZ@wSYVto6a)PU<7+7{gipc%?UkQJ&Wi+oQEjEuQ{3Mw+ zSpE60C=QO15l^m2jPxK=l3iH0?$Q%;drMWwd;)74TS2SeR^0Zs1W8Da7a2t9Z(pfy zkB@~ZscH0hbd*}6#`_uRR71VlftR`gY<-AC)j4m~v~EwzgjCd2Yzt=R2RcrJ&3{|% z%s495kkZqV%?2AiW+pNmhc`Aa9x z-*Q|h1*b%#5G$)X*qmu6PN^CZhg9uoRKYx5*|t7;(PxNSv9?DM3ug#lM~}MHOlXI) zpkv)|tFQ@rHWR^YY`l$99bQSxRQ|Sz-u8FGv|*RYES`Lg5Z5-Gw9Ho1xmes5_?CSu zzhW9Ei6nA6tn80?0Jnwm3E>|qF5qn#=xT!iZ}I&f0NWdbpyKT z=|>7*>IMv`y5CBVu{$H5_iUw%;m1sE4OGD4urc@-F52hq=A0q@@nbu~($E{ADHVDf zqfnaF&uRic66uvPbw3MM4;3^98vYqR>T>!5BNd487Vv|2qZ ztm|2cR^qUoGZ53CMb-CQ1BS~*sjF1IvuP&ZP4vl>mrqb|3XeT2n=%Qnc&Qs;ay5Zc z+t-tmn|l>{kT5?OpUh6p#zxAZ%-(m#B9=-jPRuN>e`X)hlT$d6zr74fH**H~dA@uZ z@E+LO_>j&1u%So8fOx--Zc8QiSaVF+=&JCI6C=Z#&b$W@*VNyX#hSIZg@yZau5Smj zOhIg-Z_?j4CxKz)tbmq5l4>Zi5@f+v8)J6=XEcCX=j#aRSkw^uZB4e5oNDl1KW+6 zxSaWs3&Fi@dqk7X?15JvciEp*$3ARSc?td{^ark01|!F=fCZezyfq9OQeje zfUMBm$jlA5(36mEw1*B31f(3N0i{!?Ud0D$$AG-ZGoOr~ z8v#n=2|-V>$GwV0t;t0oP_$~=4<#&bT3)|A(bki%RWR-6ay>MWAszYKMAf?!upkrn zuo)hpI>di$!EKG7h<}C;BMMuKrlZ?{6={xjfn9=pXmUCMNPt=L$s$nGZyH|ia|&7K zJ#dp7>1C9w#8b-^5^7UQiMQ|2mG3=5Q|0xyx=V89%g*GgXK_Je)4=dh;wu{g6h8kk zNH!`<8Pn^_cqRe}NRCa1?B#3ha1JxZYjOj~9Kw7lVS+_#>&d%1#ejVGT*&#bsJwtI z;<2Eh^UXN=qIcs=ziGf5kRaJ28}2L3HSim+fr5f|zy`7+P48r-+V5m zAGOU#Ak+6LA7XXKBhvH+ttqa9(91bjgIp}%1Fu#Q>DcVNEHL=-cEst_#*&1KdC+$Z zsr5wHT=gLT5Aa_xGVc(8;aIc;8Y>m8wa}!-LRi^}mD}(jd#~jP$+TlH4`$lK;ErX3 zX35eyF|UEkeEm@#B%Kl7&g%$r;U#N9f=gE$pM|l}QEGi9xM$)I*!)clL{Q6RgCs1X zQYyzq3O@f=p`M2X6)U~E`702z@R72SGHkq;8`_ZiiS#oFEoKRgH%S7aJa zNxn0+TnH3uSg&~nqnW9TK%6Tt<2qmPO16FapQ;td!T2f0)rXVEVQ0IgZ$NB9J9Hqy z6nO-gMbue3(6!uPA+=p0!>ES-7aNJJ3Kw<^P1Bp{>z(a03r%4X2k zR~negUJ7HnO|O)$DSgx#({WjI087ATC}QmYOymVLJz?~uzz3i)i~Pt}`834Py39UJ z`Qvv}HWbNXh+jsLrrvlwP7kN-+eWKd{cPl4&7k~e9)rT#Q`A|!b4ARNKy?@*R|qg( z+^YwOK(>0{<;BTYQ%WTylX5K}K{l6#*Tdf|^GK`(W{~*aQFl)AfaLfG%#;FuqJiY_ zwz|-#2F*hVU$AYWhNZ?*v?7cmB{O_Jj9X^Imm+|93pS&hyav(ky5@gnTR#C4CXVCt2$sswmDqUzD0f3S5qwGKnZBd*sxWu<;5 zbFIX!$!B1KMXn#{3vOhFuk2D9dgba`0f|=RuZWORM||~h=>a7Qh&712 zRZA?12{a{G`FwC93fK_HQ6Q5dSg4nFkWoIKyvCky zH4C%DCCB$s@1ZoBb0G)SE}b<35^6(^38;(q9=351(y_dw_c?qvb8)H zHqn}%a0l=>Ldf8KC#G(b9iraJZf#K|!oYF?w7g3i_^QrVYu3n!J|Dsat-f`+1JoPb z^}WtT^{^deg5642L%%ZtdfTRmb}OW{y9T(z{?9ht{a#-QXHh&@kk?LTKL$JXLHwI= z$}z9H)Uo_zSw?jeV95cd-5UAIE)vM(hXXZmn3qN-DGZ4E5M?elwf&=H}QavXPUK&2gso~iZ~M}iHYbKcJBfDKF|`IP{%+2e?+GmkH|7G z8{%N%O_DH%p*E0`t5~xp=&gT>3eDZ^tLrjRG*Gd#b24y2>B)u^|->uZmpVYiqO9)|v>C_RjeAW?ny3MOt9jHz12N)xGqg3rulln%uO z`F}Nkk>C?y5fNY!&2tq^C7UC$8Fhyc7)_)wON$G;IUlx>V1{{yi#O(3croBLo_oIMOL@S!703o|{n zw#)WKR+D!yw}uxv4_E|9dpOM?Unp&&v-#9^m{qiQ4+EzM=l2VXkt&tDk`PM>al?^k zy3j1gV`(W)Ukr8zQOWIaxRWK^Bn$^-2ixEs&Qr={11=za`%qc0UiN`0yM+QYlGBf$jxjKVG!V`#B6;2h zMOZw8w%~J=Y3lD{y-=l`?9a9&1I3hYHoJdy!i@KR1G=}^5%=$E9K#a`nx)r)&{LK9geu!%K)tz9EgfB z>bTO1Zt2Lu8}4MKg(Roea{Q7QM7(yHPv8z$L0{rNH*!|Lwh|ItPP9)IOcLE6sY<@- z{&>$O^dW*V!vUBga8|9RlXgK#Q@%a-2Sp4dOdY2d>FmRY~JGy-yz#(L^FoFOuU zUX(FByPNl6H;u%=6pNF$QCJF$tauvR08$eik&QSzsnD`9EN(n=vr6YTcLeFwZpEJ% z)`=n8W|OQ*ldf}1lb6)q`hqpS7rP~CUds*Y-`dAETkTwtv<=|qQA{-?FCI95OTLbJ zzZEDhrd=<0@g2vJx|X~uQqi(r zV8`?TM*D)0E4mn!Ct{fSL)+Uo;=z1f{(`?%+0n(PD5<5fZ(5YtP9p{h|mwq7>8Jn0(zV;{?wdqQna7To$Dk1I|kE&)n$=J|3GCZ^F zK2lqa`jn)(_&O@s`7&Tc$F@>(vp8J}1{z+E0cqDDH8*lm+=2gluT;@nYDAx-pq#-$ zN`zj^EUiC4-1uQ%SmW9?<=QqS=wTnh&VzyZCUiWM|C&cR#=MFI-eh)`DC?Hoc40g` z`Od&%@$ZCgTh40(>!cMI)$K-n58j2sX$}Iyiz3H?2daqXgd++JEAS%O6)k)_3QUK* z&!TOVb~XFzPAB8X-h8$j+o^p>wR#FPb7t7+Ny(gJ_8GmJo1&2IP?aVHojd7wSE>}u z=#}0|?S|SHi@X-bN5)N7_!Sj%7k;Eqc46wB5RsC0f&3YS@Iq+*%t@X)xb8`4I(S9- ze=zsfQE_zPnr{;%!5xB!5Zr>h1P$))?v1+!4W6LEoyH077A&~CySw`o-*?WeJL}$= zJ9nM=6`WX;7$z zB)>i=#zMMq5ZCd5?0MtID9zMvd0KH5yL9d*WEWlcUS6`a3oW%>VbLEiBkTw0c+o z`71oizA{=_*Tfzb6P*Zgp7ITpy+$`R5?st(q2P}T+p*|xTrC6)r<^-TAGNiE6a8r( z&_v6kJL3ylQWF)ZEv2bG9q`3>#+}1=8O}+m#z=B!7N<_*??HziSa=Wysz|QgM(l@Y z5)|sR@VgEg!uXTRBI*pFg5l$3qd@ZY?xZj=F(6s8=cLN^biY27B~9**l-Fb>@iEh2toH&CW-MCnX4RP{lx2nl*- zd?YsEv{WVQ|DHn02LLM-^93)`C5Hm=i3;;)G4-L?dSKmFGTk4iepW$974D&-2E9W0zo9RG(aAttuZ%$&dyCN{>;ER0O-|8;jFR;71#@K6pBgY_{N zj*cu+Ld=dzHebA;m5hWx*jSod>Kk{!atx8zWPa+L{)PC@i4>+UL`@ZFv_A0nKkt_B zf2CWWIZRnOj7^Q{4Oy6t=vfT_(;`D=me2GYEKE!WTm~%6?7%uEz0dT_^vwT*m(f1~ z`X881&M)m}8X_X1ybjk;S{X?30!gC|0i;!KxquPT96VyvyY4K)gy@81Q&ZFYgfvst zWDEXnqdXILWlM=@_uOp-)BJo@UB0B`Co^?@Tn#ikTo*Helf*xV+HFZIgOA$}4@0&I z&l1p`qP7*WvvOVsomV?g7$S^gT4-hN<7&U_`{QD=qesO>_H&Gy>xMH}S9T~$e;c39 z@`$)+Q+=oJ;dVWdLIu|%qql!=Vx1U=@LA7xIRwfNt9U7lruuJYj;)(a z&*WOtm+TyN+5*^dq;K#3W~vIuWgCv!g6-WTWZIIUl2_?t|Z%oowf|2FtOu?fpy~mr^O&^kbi|e6-Z4+Jxx=%*ifmzTlvezkgLoL1~5hF~WzybsuYwDND zws4Hr*LOelqWEg;Zftf=R`PGS3C++1;C=dSdk?K1PiA@T&udafLb_YlSxhfvq;v(I zZuYygB0ZMujc2-}I+httQC{r(_paUsNAbmf7*t;;Ct^bqE|3!j(e(O9&%7S|}>_NYINr1lb01(3WS$B)7yHmP` zd|S@79QP6n4ZJN|oawth+uuAwd~LN27RD8O;DxqtzL>6P>IG<{YoogiH=r2W+V`-? zy#VPhfjEJkL2ByOAorpCT-FFT_lV>8Xf{1wV*L63m>5-QEWO+HcBG6VMdnC8E-kXd zmfzEmLh=fBLk|19qB&o-f_kNM8OqIpR*n#G3wmk$&h7>oaC~N_s&-Zr6%{8 zhek%86~&n0a+!yk5}RfC^#R@WMBVkipo{E7U4>+eKi}K7a;RY}u2I?ciQOEMs<-Pp zdwZMf6*N6I%8|9|bf{b2yH&c%PvcnRzWAep)hqQdu3Im55NDQX+%w#(HrnJK)jTX6 zHqZ4M31cr^(`;FviWzeeU;1?_!B6~qQ?9m_xkL9FmNi%RH|cvdI{1rgNzgN;{1i3n zSG%}o$Ldt{4K8|8$@JFt=hk-7} zqmJ5=Q-4KruYHhh1Can6yWP_4@L=_alzKHJdgs#)aZFj%lj{J!?c?7XIWzX*6-Jwg zG>?kDY`Hu#MZf&k5(++*>?@y0?PPhQo_oqHFf;z+Ec=Zo8|_+EYkV)_b3h0J|p zZ##9sRk;*(UmETclWRhC$xONqUn6ncywC2R2S|xl!<}-+?PFw7J>96MPGFKd_Nbth=Q@MaoJc68aTWt@)%JFHL+VkOKc&?uJ zrJl~9fF#L~P0d%bon1#%MjFjV^Nh>A1bY^x2muMBdD#V@rz(QnGL{9vf@DcA&%QX; zl=plHGTvjKL*mh&Jj);ULPFlv+`tH|^s)h1% zXPc~tM;q(q{fzXUYk4ArQw!I_glE)O#K9BZ^I5)!yrCl=UY*Lbb0ViSvB$%1=n;;( zwkz+z8_C+|=<0T8pQ>lwr#1KtO5_pTXRcC{{sNik^v0b7RB1k(v>~m_k-VU~YQE1O zP34!1WE~q>lQ@oYN8)lG!h>}(YA)JW_PhZB_lB0uxIn$k1SEfM+Sjz}$0Cv3-9c}+ z$c^Xj)UIvD%2e)IN)-xN>oMO{HU*9pmEYxM($$BRJeqZ}YG*x`qn5~x;byz)Z5w`* z?*_fx^NJ3%8a5l@lGPPQ)lTiRyJNKKm{ECJa=B;rP$|#ON`u|m;R|0(i+*m|#V-R- z8E7*0!;zD=I-|8Vqrruy=KIl&?O<=u!`sU|MqsYKek*R>@5OU5S;@jSQv1N6C+%hB zCS@_N#!U@AYEfLtYaO5FYRN-Md5-72$>9rC%3BUq6L^r5fXw~4br#e&Y&K3Ip$XR@IFfy)-_g zOg6S7|3qA@(j$5q7x-fBAo0ygTEc=fm!g(u@-rJGEgJ3Os83BxYaAZq;ky!!k6WvT z%3khE{z`HJ#?R#RkMHVi>yJYdOm_ajN7kFY95_B6nnFzWR0iavEk1i6UGf#LYu%`i zs=kPKPUsks5G0-rcGfkgDt~^w!WE{3v^ma{>t1|p<9WKhC5smA>4_ehtvwu$^UY&V znsauLz2_?5Y1Q-Q^q2`E6IiMvBY){$hz)C{Hzye-bN#uAJE3h}XL0vCX5*n%Y_Sae zrsemYg$y>q=8|Q!Z@rI2`**viPz}V(d&~@9v@8L1uM{Q|Ur5uF4B`v*ii6(#%TzKO z31b_@<}Yw(G8=50IXP&|67y?XxwA4RmA6fAy98LAzoFLQcx=q)yUU<`hZNg?)yr1shWwc%1 z$~{ki+HQWBFA0kmGYMTIYYB=t?De;a;e5Ew4o5DM&HgI8KG~Q4_Q}dE-ZQB6H@ENQ z=$GJPUzvHQ+qv}Mmh!t}y|W{m;9)t6>cjJ6!OoBw^K6Pb5@YqtuCce`0!)8T7f*EME&a-G63E*CE8cGzDZ95y6X(#?vJtD1?{+j8$4Ilp9Hud3b;>z_s&LhrC z)A3P9a7B1NqlC@;ioWKl#-OGC zh!&TB8$Ra_bSPK-#5Z_uKI(`p!JQurg0OQGt-tvi&scrHbHuHOVc)z(ete42cE7hL zkQ{F+*PuN$h(Y|@I9mR(s-rfM!s~hL$8&HVsZIL+i?ppU#3ZB^2m6(qN< zc3-%i?jvPp!1$K4Ui;54i*~{S!o6~+(r!;%?>l`}%ZDW9Slds#vaB>qwZ~RQWo|n4 zLZ9I9k6-qd%IJxSEWdlxY<$zG>C2r@6D{3jeNJWZt$iwz-Rq{(zVTgRXG;)=sl7x% zGRMn`{yruy9z4*N)0uOHZhzeqs{3s|2DS?UGxn)L zL<|5a1>uwZArs0O+!Hf$aE}8ASXzH5|Q=9;Pc0QWvI52`tWW-m*NMk zI}4r$*6Y}}Qk`E?`j#bsCBN@uHsVDi}fMp)i#MEofoPa#b zFjjqA;892AE8=~kJG#>o1I^!YFnE+mcp7}(dozRm0$%n=8G8@|r8Lwlln_?8q4^Wnp{}pWBuAW~kb?ddzNOv@#u!ENe@fenEqmK0eIN zThZctYbbtjI)`>9y$45pVoc;qHOcK0-)eoOMAj7|IrE-+m^RJRr<$|S$xTsvjsmAU z5xH#FQ0Jz)tMV#M0*jJ^3U7I;)cSa&F`S}wOS*ov^=1mMp&?(|QQZe@pqtD!cuqZc zE)jtt(iWu)rAD3@Av-WMJWThM;qM^3E7tvfTs+l;mL4;@a$&4V$PT5;<7-eGOV-e);zMblC99z-8I96_}YO&2qohJXGd`Yj2`7fd=R(YNg zCRdO3(jD;@gx9rNO|YtZ&|jeswxRFb1^d_5_l{y)Gm|h^K_f1XSaJL+grRPFHHUj4 z!2l_TdV{me7^d_2=7~smQxj8@8SjX>h9E9-tz)aX{Az{iq!gULAs>W?e=PDglwf(^ zQ>bHr%>hJIRME7iNXNDIz@Qv^Qq`t58~l8xpw4PgZHJLlww``dQwYk364j&8a>|0#i;%&{Sy}SEsyDk+*kp5_G|5JY9>p-zY}5^S=tziNyUoGX~}mE zZkolJ>pQNJbjjEyA3nqIr(~ZAT_(S?q3;jBXb^nJ5=$p;p42l#)+`l1FPq1+qHM6J z8|afcAfrI&5KqyPyRRe&8Y=XuZorhg;s7VgU1jrkL+&em)#X@TB<0RbUfwMiI1f^{ zO7;ah)=@!ja|NyDr<*clpg5^N-`;`JQqdDKn?H%wRgIVSOU_(klLf9W*_6=MEb@k{ z-G0Ar!*|y_Efh?X!LntcItg4lb5Z2U&UGm&pOty%qh%&jy4hnLYh!!hKcih*Rf;)_ zLOW;b8$Csr1$E**F`<#yIwR>@dpK{+03^mP#k>&>Z?b%@=o3y{^81X*5lA z9%_HAx&@OqUS{~Qfr~v<+9FZEv_#$Nv*btY-g$~jbqQlp0rH=rN2Q^0Jyh%x($*RJ zrKjpvUnDVJ*PkWDbiIVd*3X`ef9gN#?v8?=mW-{T$or&VUt)vd8UHC^CI6E>y9i}S z1v$s>urXzO%LBG~7JT<%7%9F%Me10kiaX<`YnJ|bxh0BV_>Nv4jec^IqiZ4Nlr*ko z>JH7sD%I1tv_j5l_J%^?t&?-emJg$VBNwDwnZj5`ZjrD@1B*|AKCvB5lPURRrAhr*YHnTCMQhJ-x zaNHAgw?1h-)pKPZD_!Wc6edj=DnB+viRP@|mz;FqM_OP46}$|7?x*T(sX0+XMZ%Fp zkyZAc=P(BNX&jJMm0!%i3eGrdE#9wv&laJ8f|ER{a<-K-i}#3|YgP(LcUnblkyrjv zST@soc3AdM#btf>Lq#l_7lp!HXcWJYeyR;yr((_TrK~00`99>$In0tF%xQK+lMouf zXDCalA7u!?y8peRHj`Ge!u@@j=GeKjBq_Qd_+0}4_)=9j3#vLPXyV8L+AMf z^Wa>MpG~RQTWTm_`NR)W&ZGs7KQIUe00HNRxQJmAh7bvpi9h}Md1lE~(m*=@m{_N= zI!!Z)H-o&eDZl}ts&X^>&Y?Yqg>H9w_)GpGt2?QCoJ3R|KxOi8qyi+h4V72eYI%f} zLRR#Bq{Xx(&s@gwCfEtInpzs1y7u(vM~jtF{p`7)YRj0PBoj)%_$b$|<$bZS!C^r8 zVpWw%{-b2*Uj?U3Lr|f{wk2I^ic(6DRCb#tqh^ybt(X<|OiYrJ{1nY$pfQ_(6#X~V z?yd1Ehe(xta43X^;TLyNd#c`)f31RIQ1?K;ipffRDM><-=^(x}o?3HP=ajnhx3s}# zZ8787k?)nrk|SP!t(Arse5$ot6K$nXXtlh3w%az{b7&7!xae6H3aoeS8mxx~)y`|B z>UGog(gEFLe*aS~@~ec?I!7@AyI1ti`ljd~CTLxx|x2PE$A(uY#|w z+=f-;Fk+LP*w2#5S)x!j`^NEKER*EYe`1-^{=qU?4_4HHNi(+s+#LWclgUp1KnS%J z@~P?{2GUHMz9OY62gj;-ttMWrvn81*Yw2x23clmRM5lwHrC8e;L@2eLZDd52{<$R5 z9lb1SmJ&D80cNPGWw2wwdPi<;#aKWJ-1xaV&s#z<{_V-mxhx8~QjITsPtuzN@hU*flUfJiTp! z=g(!r-SMwnCf$FyOxP=NP=Jc-6TEnV=?n+3TK!Z6%8#Z&HHU>wMfr;TTai}RZa4Bs z&c0M~T_}*~lN`+tL(nd!FLF`VNrM7$2q7Y^Ru;U?oS&2zu!B2?O)nZIPA=Y+Rq@>l z;%?%cEQ`v@#*}1REPKJL^8xu`)^|+M)is$kadB{xgRmU05N@`lY!j(%wfSh*Kq8nR z?UaE<%(QSE2hg#g_qUvoMPuIg-bQ>9OGGx3D4-%ysJD=Y&CT@+Xu^`ER|BE7w8@KB&Dp#-rdISr}8$KKh#ImAtZAtP%}fY97ckxE9l`S#ybd zsjdYp{a-K>%Va3;E0~Ev0aPz^tp1~MgB==VXHx&r!Pg*x%35dD;1cTv>(?l{_Ku^y zsc(PVqn4NUpufX9_CX_@*bf`mHI#V{`Cg2(#5CMK`yea2Q>% z=G`2q0C#0swR{psz(c47t-w_z->+(Hs;TiWbvv^B8noZNe3d1E37fWj%Oz5t3A)|; z8a_9m?k&6GHfFZ+po`C4=*mCB!7p*?iGq9}j&}ucHrt?8G@5q+#Ya>=8%Y1RUyVHh zlEuk%Ns`f`Oh^QR3l#WksWYPHJs`gtW5Fr#m!x$pU;)2`NyksNxPn&szpH^VqsDMh zt`rVpY1%Fx+srXeGLeK!jv!6@ljM0E6C)$ zR*joyp?0#3uvU@cmViPGf{`OQ?r-1wDJi#ke~lCcI7EWT&Q0TGG)_*1k`mpC(&wL@ znxV91_-0-#U}61|vr;ft#|i|Y{bChH-+)rTF{uBusek^Z_QmQ=J6w6YrnRo{Hoz5a za^qU*oR+k}?vs>{zA1>jf9Anx)`SAJV;7yIH0+NiLyfE{9Q`MmY1vJA3~2@+GhJni z;>&K#W=~8#{6l6ceikO%DhU)*%R4$3=z_6HYd z9YKSmgt{Z`elH?kiGm8mS*4)&6`!n<>lT3a$=A7kg#m=V!6Lxm=-Um2sq1tV1^G#3 z*Oe@g(ZLXQe>t3QQffhuTH=pKtBfs;}m)@=PwQB{K%3lC|S2z!9CQTH$cgtjj zR&ZMdlW)t9R)|imPFchFOpFu2y69Mf>V%PCIKya)CDQ$$0h#9{oGW00wn}%&`Ucca z2ABlxXYNDtFO_??>{y^d1t?Hxi+(vYq|I-kKsADInKPd`%Snq$a0Imy5jSiEo!9GM zM+r37mV)wby*1FTjIg96;62pT&$H; zKkhCo4nJrGA5Xe|MK_FA$9$jBXDCj%Qd2-DW}gTugK@?I0f(vE-&bui=(ocayIDF7 z6u^#w0)RyBZbTwM?<`jNcX3zg;-LLJ?7fT8Z-fPH+`!)(w*o2m7L!-ySq1)tRY4s` zQlSbieINuVWbGNqFn}K1l7?;PxP@oY`#}fGz%-A_Wd9d5ORAZG9s4j7$nOFzvx!jc zMUk`}d(;>jl4%0xTw3H9<=0M*Mgt8}F8|X+${S^eM~EP(V?Ou>t+}w-b0E0<6=oEp$4^puDNuC7vJ_=Ee zt4W(`V6E1?6Sk;V>6lKQ{*}!{3$U5`>m@(SzyfS0*|BOKn_@?R%|xTtkXMM<00?Fm z;UZnqgEo*eQ8dB`lYc|CSB)tGi~W*+&-h6|@U&4@=%GM%JiNS6t6!Cqwf(h7;Am9- z>7Lx`bi6B2K_KLDOMk!zLuV3Nj;nh$d3S%!htm|astK@}xc`IAMEEb8sS;o_HM|o~ z{ia>2_R40$h3r3k8u?>_E$F&eTQV^;?=wM(wIZ#Q~k4>0uy0+8#9upp-IVxg+RzI5#$^p2DAFlD>x514bW} zNnjD915KWI21Yr;wOrnIw*_GmPDOf>Ak4;%mjM`2R%^qX?&dwfU9*>g$O~R()aei@ zhz9Tln-Oz<{~A(x4&|3Fe^=wx81o0x^Ka~!IZF8z zxVg3H9cTIUzHRUM=cw#VQF73Gg6gRyJwV`s;XmZMS|P=SS6mhl;(qD8Rx2(Q#9EGgfzN`Z4)ukWPt5$x_)( z+qj;887mE+0MA4=JSuuJW!><~f+B~Dmtv7HFPbGuF>(f0*Sr`L#_SREPvZT%r&ze) z7lBu)1kOc}7zF6rG{dek9Za_?MAY!_ZnnC7ie|}oqJ;gg>;QoAK z&2|~TST#QmOE{49<2V}8hP&O}<(97NK5KEvlXKt)87x*IL{iy$)|03 z?3va|Rx(HK_?OEvZIre!Zk^ggGSv46LBZw z^8i$zc+S&{=7q;?uLG{SIr^+fuXL}woOB3dZ!2fzk+{qY<7Rlv*m8=VfWvKkAIb9k zT1t(Y(uron{+~=-%|#%YTBw1`W5fH>x56L6QnOdzP(H#v)WrE&DD`Rk;Nprh;g(@)#9eLI%h$spmnudCcSR16DHnx6QF{5 zF6ON06W9gVTjI}h{xHp4@Ta+zJXGFb&OCeRo6OYe5hUFGx(LL}Djf>Fe@4WAu^Q@s zhBj?MXaBgi6ROo!mXl7hQa7(PBlHe4iZSEKXhLxzA)Q$!r|*&G@1f37ab(rX(a3X@ z5Q)KY-W(+_MlLG(2&1E9|Lnj9f=|CW;n+ryLNu&}&mF({^U!phCL;VFf)(D{x#H$H z=`r6iONu39l*@|zDb-!ll9k6r+r09siSe}Di}YvL5aI~-_pHggsE*U@<<|^-8m^~T zjt|LPLyi18*MXO`k~@Y~-{^}|oX4j&NUhJ&R9o!y%Pxw=d30Fr8l70#zRjnE2=JWt z7Mpe12t?j9w0oA%+KU#W24Bxni)nk9b>9hRga(X9@m=X$3}xG!Y~S%6=wiP#H6OLX zZKdJ793S0npF&!39#~si9Y=alyW3C9R-6xZXSzs|`ObS~8IRDKay@Jg9k%^%uix4{ zetfahTx5SZ4NS`z#I@A{PUM~!6YHt2Wmmc$y-@ZmO}cjz#m!SeY9Y-eRqM_5w2v*+ zEUIOgGfo8Q{2OaFhE>ANY&kgiZKLTRhEPZmjpQ=jhrf%TvkX^PFwc zYYm4jE7&O2-X7xI481$GW_@sE+B%4eR`D)=yFwCsr{k72YttFiDe-bI&D!#;GwG=& z8=etV{ZcsfVAgF@T%IJvyNpHl;%0tz+GEzO)>Pd%;Rpy1PV0Wo5r5WTbgx<++;knB>4P|Jgwi36I@FDf(!kTcKfs^3l~Q z>$bmLYi4-)I(?SA>I&87rCEvZPzis+nRP?>gyo%xS(h0({^`XE7s%e80&+V&TK9EfSS(FZG4IA zs>)R^yR_o2dtwq9K%&R@0NGI^_T+PXB6D>JQd)lvE&#i31VOIU9-evjb3WKE^;DGKl~|+^F&`#%mo7MC!$oLc`OX}3 zW!oy$o9fP#ZzO^V8=faPPLk^d4oURNhawK$^S16+AjfSRr)nFH$3E09JK>l(s(j_L z#+F^h5=Y&M!5p&Mx2d|#m*;}^`&+Hqa%(K1l-WK9%gq`;>YEQLySmz;LwBNl(%z4I zN*iWCnkb&{`p846ebF$zZ;n4d={g#KAvO3a8uusR`-9E*sH!*l23Y96{xb~rS{1-yS9FAm-0 zjageUrG%^Z;vpOh&1D0(m~q*;_{XL2+fAu&W~cwbK*sEHb0X(q$;{nJ>%zjOua;36 zwj0s>{n2%mUYB#joa-Lnlirn;mtd8_>Tg+5aiuV5SBs#!1h2bPTdswVOT~}6*2@*W zk6$~J>$VAytp0GgIVHysg6l6K#puCoi8AA(m!@z5YrEjchNX)M)$U-tl;_#y8P~C% z2B_y=pJDdARJUE7k>PUDy0Le@52XioR>2Q|(O*+j8`p-$eWqleF8)rT22+I&-;`ima~TA&lJ6uR}_Iq8PV} z&=$bSJ)?`CI+RJy#%r#;D6yiK?`Q^f=x}tX{&b72^z!T}f`r#;Dd(sxtH(YBOu*`q z0Xb{;1-$HRR$6*aS~Q3DYP|w0x|IZ;dp%u9Hs+zKny^stExmR&ulB-CQCP06uCFIr zo^m3^8(legPq#m8v{%Ct_)lLlwK{B_Kg^SoMG!FLZ5vyL5SY2l*U+6$wDzPBA3J$u z8wA?xTu|FUv{-uNzN7QEdTiTmjN_4|9WD=1nzIcg5~F|(+mfgGU*;879ELAh_bijI zX4SO94TE5b|Hz`Q6&}6ac3-H>6LGOLvp6rE%dU8|zKm9Zh#jwZMB9V^{5GM4L*Z~q zd4Dc*T_qgZD}H@kv-PNNBtMU-moZwy(rW*>SFeS;WWvg~*m6+)1m8n)k5%??H05R# z8v^}Spgyb3Uhr?7EzwK=fZM>dTrk~jP1{+(Lq?K2sZBw}MSe@HTrHvX<8?RAf`=IL zXRy@W>`R8iQ-p8Y`D#sJFLb0v5MH>c3Z-L_>-=Ah`=<~AuLGT9XUeIs^Dn$Q!~~<` zWbJJxz$Il58WA4DQxr-xyGR!o0i_$vZWs=lq>MECCxZU>BqzW1d%u4T=(CRcjuOd| z)G3@}*g2GCadx(swPwFP`<+u*(<_i#JDHx0O!)qN26{3vHaa?fd;$(pvT0Z(Qe=1p zQPUQcHo4p|wR3wG;4>M>qu))|i@)TiKjp66T<`K3=XM6> zyC+dN>J}&y@}TgPEyg8sb+CtYzU^(P?VcLXq~B!tNiF)VoYg!$Qi@Io%U&{|R)502 zlHNEL&PE}n(^!s6HpZtfD}F<|F~u>_^swda$fT5axXL8c}2*vTp4e ze|HrfSeMsru-wgVK;}BBZ4x_L_kfXCp7qvkVZ9<`0UH%Oj)=<}7M$q%8+8P^67^+y zj33~=*>%)@6K|UBJ$Y9Vo#E(twjWIpSd!h5wl=QQTO0KAkIKTLl~I^k1BMo_8^v(z zu?l?mLCS(oy&-}3v^wui$EO2A#VFmT8o4H>|IjEL-vYiZzpj)juVT^i(7kxvZ|fu( z@YB*$($X}3X#YsEInLcHPgMITXY1X0>2*<`s@;nRt{wVeCOb03LM~vQe#og|mcq-< zR{o*t0}F>ZKF#|LPARDcI=m!}?afr>2RJuLrIdcHIK3aqeAC=@t`54#T?9&Jw4O&Y zi090`6u8pFT84G_T9SmlOuEw;J?#DBx~xOI+<-=bwkC6PXOZw=hhm#B*83leDdAxp zrTpqKuDxwzi%@IFZ(NKkjRK|TdHq)28FkEW)YPfb;JcP*j4 zkRi!PG#jS=%ASYwi|6ioq|ziR+D7PGwTnQ9_zxUHBMm7^62N1Mh*|QPdWh&~ZH1Sw zO5efZ3%Igc2i6qunA#=zD0g;gOS5Hq9ynQYX8MoER8QL&GS}iPbBlRIHTy`!BySaO zRr7&v;atOpV_5CtLZi%H>=6qG?GKcW_te4;7Zg(Po>y|eDHb`|7#W=%@}@tU6OPmj zJ{rezN%gD5is8P`M6I9ZcMH=!MgD$$evM@3A&Z#aD7eI;xnys18mDjD66@C0N0pZE zn72Iv6ZjYKnELy##}uEkd8ub{$d7z@{=6D+at}}Y-InoOtq|i9Jm4`U6p>$bQ4fju z?;cY`!mz$e0-gx&$*V+>hlSh?V)Pc^voV{~nC-ZUlF61fhi`!!I7Eqc=BkVaHLCsZ zNFJDIaK_4faOkn7aEq>Uot%_t>v+jwfLl!QhqPaoA|` zJ1fm4Q&fwdRh^;BXV+=gVW}Ykj%96mt%IK~S_WF1pPDl8DNJsvd4jr$lUI#=JUXhV z+KxmU$TJX~QmV%uQR9kKOGgtR>;O^(NWgbNB<~*Uv`Ln%4KX-;7r}u0vB~E zc|U#LQ+{cV-co*1G!qzl?C0%nn}mJ&FgZ-!DmNUM1h`CjmA3KOJS>gG9MY8@|58mR zy?tW$pe8H%;;s;JY@7$#j6Xi16*2cGv9;1GyxO+VSnF(zt{$&m6@~H$p;|ToY^ER^ zX}K<jca>6A5GR4?c9Qi1i;`i{&utex;I41%GW6q4H!D}-I@%2R<1ZPT!({W4Di>DsV)ALC(=Mm`sia~wOHhr zlZZ$?I+~~&vN=Z#-ns;SQZEjizRzlgzLp~od+#2JOV9Ip$0bCn3!hGNnp2`qidF_m zmngq(io&KX?`rvTkH!~f?Z%HdOJrT<{?P;_;)1f-BJV6E%=5Q2dBN|V_gq?PJSyVN&77)? zYU}^HOi{wy{KQNaP?z16);<$moWmHw6Kjzb?J8UEg*onOHIH^M`>p6$K>R1a4AB4mttv2rizfm(d|qSiodTzV3sSGbz=l zuThbCBjlQBVPZScGBR~?h=9MzPm5vZt+&n(6T-E5X_NiAcO?=})Kq3(stK1&dI;(c zVsDp7FpT?U5^-Y0k_Pb#H(oUgeO1!P)K=aJMarcmqq@l)|gFP0|p-Pj5ErKOyN zj4e2OE?^yT4FW|ht;7@`ya82Gf&O_rEqSf9AbL`BXPj!dm?@1)&Sbil-_)*Nnh6aU zh&$WLW^38I<@4dHj8o%dntL_I+gNdR2$ZB6Q~$V3c{D@-7X1#dE>n59(ka^d!;H3H zdZ`L0*Sr72WolSjZV2C$$V|h~l3Z11^%Dzi|97TgmTb2GW@gGHqJH<@XxV|A7@IzW z?&6V=$8c6#r$TQQh?yy*pMYDHMvLWQtN92A{9`+D`={x5YS%U*;mk@`QXUnC9y)u4 z*4Xf>W-Nf!bcns;hDn#)6yv^eDQg}nS>$Lk=O5mmD71ClAbhFl0;m25|1OOFv)PW? z#{kK)$)NMX=2RLYFD_~#Kz=Y-$Td7&dS=VTB-h{G{QFSb;_kD1>2tcnY~GozUiE%; z$>O5*-893HX+x95&&a;RlZk_9=vcVLB^LMkFB>Zi9JWsCMR93HxsL?4{YX`S%~X%? zaFHNklN}}>KC?Z3o6gsxAPqJVPXN~RMG8nQ$y>NOE(9Y|l@|$b z;ZcR=Ln-5<`$=Accaa4nsq>wIbk|j;A&@M)Km80Uz|aM<%m2Izl0&Y?XRcj{V3(v8LC|(tdU~jOBXk2r?AM_=i#=@Hi2w%Bq+vp{#Fw z4m?nIo?Xs6%zrAZr+@wz)})Ml@;rn2pRlH;R7n8V^!Wb-YpVGN*5v&Utf{+%EbJB5 z)C7_j@Q{P@J3sAy%~Td7?@t|L&rYJ$GvLon2A}kONn7i~+=nH451J9?4V5HQ7F{Zo z&;rEtwiuF7SAkmiMUkto8NLEkOAE3rU%=Y~IU1)WI5(l7>|5cULBYL!^&&wXA;^vo zrUc}7UcX>O?4bf=cth_~6bh~1P$6ywq?rGv6x(m>rNLFP1Zt}1EW!j5(Wy1KDzm}L zo!4;vE}7TLt#8rQ($}Zv>M9c%s@HNA*bJUGFu84lzBX*aiR=-|KAtYo0F(|Qf_uy zq!zqy-0ep)rS-3DB7n6b=nc&bI00pbE%N)VAH2FY$@6(h0G)IWXwCMslJ^{-F;Vcg zSY;GZKDuFQ6pxI@t=kum>=&NWH3A8$0W_3MbLAueQk8j4$$u54ssKU|4W|IS628>2 zf?STrhEN4NGuLo|EDCZq5*7Z^_C7i)36wJQ7HcLB?jLg-gqDhud4v?Q9UBnfAd_!F>4u9HVj|!KDo_ zaO`Dj*4*fvRX^sGofW)^x|PE9Tcf7;*n{w=5;A=`4xaccLR;!8EEW)Qg!8 zVeYVKR}hzUTY2s`CV-9RoB0DMwRPPm`+W>1sQ|lgT2YABuLde{{wNzYnc3p#lm1ZM znbbfgoa$X#vb@YBuutr4o#6ioD4cLSlA!aMTQ8pIc9?@+W-#pcg zIXY+1;mhI`6o+W&tD*-;1eZ-F!%s458iCs5sVe2(gOs85)ZT&o@GrO1Q8Fuw9)AU& z!-8nq7jVo4EqR{Y9x5DmHRN&9mw~xJK{e;2NFd0sl8Q1aQ&(4I^EClGI$AWppP!2V zOX%kpLAFuMDLhRXV6kO{ASRHY9uX!l>QYcxkWv7a33jfJ@MjXT<_UG1zkdT_tuN}< zM4q$#AqlMw1EB#9qIv{924$~sl|B#nqh8RYSP~Ite1l5AI2n-G4a3qSf!@b)e&iOX zhMweT-7Ce2({bHih5=E9SDxtGxq=Bh?JNMvN2adw!x8s<1XLyl{-bS1z1T#UP{uRI z?bz!)o+(ycq_E%*y8}eej?ZZyCI;(48`U{-8!7RNa}ylr$hR}gc(eIKC)6u==+^8yH*ci|lFCR9O%rtOl$(>%=54OC*s76Pa&TagKZ zPu!%~39J0=A4LTSKjAwod>Y zq{%+ER;?j&GzKV|{7SPZtS2A@$Nqm&_k~aa$3A%(m>zinV&z8Wekx#M#g$2?%C3x( zSpEK{=KBG>bN?AkP7s{mLe2*#5{C20VUVAu7z7xHp!!ev%%Dsxjspj=Sh1FkQ1nbi zu*-ws(fZ{#E7D|O02c7#MqOJigTD?KQ8Re)gN{SPaX?sQILpyH*q$Iz>H%;C0@Q;Q zY9;^_)@Ji~H&OVWO~?-(BuG?rnqvHJOJ2vR?u)4MrHX3mF$lyls1hhl(7{(ks@5H@;3Ly@nzM zk)^3{5)%t%Rr3-v_fQrQF;~{qbTUt-W@lz)VD3S)VixU9`9sF`?A@Ec9Z6uz@NQ@r zF;#zfU}SIz9tjmO8VM2gFqNI3WdO@uG8`c~9EqbVC40h<4EUkY*x9&CcQ^jHOyTzF zFFVIL(qUsssUr-#(iYvE!f@Z@Q#Re?FA~CkUPxYL1U#;-e_ws3wEvef0&yTZ|KA)J z{*nxZF1^k#0Jtr zP3G&_eDYWg*AiB1vq?vv-|zco9Oc{0le22oN(l3kb0?k<`d+&pLO5E|6Uhn-^ZoTg z8DlK#C%@8`yDoY%>u#Vv>AS6pzx`qz$&ok5so1}os-CkGJ>Ky>H&-B3Xptn7b6@Vy z-;?O<4}#)U2q8T?o}L4fgnLO++5sfhonIe)W>I5DQA2zPVYdn1R*(=MrRQuC$!*o?K8*dfB{Dxq zY|tc?H$fwyFff`%ZGeizo1VGvw0V3bGjyO z{pizz`$i0Vu5Jv*Uy!K}4naHy|Bb!34637Bw?-Eb2=1=I-QC^Y9fG^NLjnXRxVyVc zaCdii4ekzi<^9e+wRfFwzxSLgb^qM{L3L5R)?7WGo{!DZV}NeXvpP}b(>S)nmV`DV zM#v&|lzp!roV)j!NNb*;;BWn0S!E8v&zPF?!?cK|neCo-xcsH1FN7XN9j{`zMm%rY zhpOAY$$9X~;5e)GqRp+fX~yi7&3#EROR$|!`qK3oR~q$J5CUQiY9~8=`!(9$`FdW z8O!Z%Eu^S5!^T0KK;nw%BFb#wQ9E=2K|%?#j20;2xrt{2syu=c?2Lkl)E_8l^`*NN z6olL_m*DekJC4w}!vQ98UGvx)v=S?!Ql(ojWgf4bN^Ws~pw8BS^0FM|roWt7q;Y{v zHoC5ZG3(4HrkYu2c1XH>mR;r=3_)HM7f8;=HL4<1V%LUx`r1RA@Z*6qbMehntmo3R z!ds9=v4g@A1C?_-*o8MKP7_-y6oN$uK(eWM4eU*Dx5}?|PI&J$Ga{QMdvXF;lW|}KL>Zg=%A`@c3tTT&}}rBf)%j$>QIU%rTGJCaHJz;WJjgJ z|M1|5{?ahTa-AJ3AIlt@85_Hu_8uFXn;99iO?0C^Q6yl#^I&S2k)rl8bH#9t5j6mD zGkth?LxpwXW;UBhf68JuTNEr2&J6-=V;f{wo1Yc(r;@?=3i*pU^&53#r%8`tVFhCf z%ajv#Xm{NQMbp5Zmd(=Ocz)1+t5=wUqzw`=SnaX65$*v(nZ*Nys&5uFQ?RhUGmU?L zWf5gQ%!F^Wab!`@$U(d3sdX;wY>PQqi2y9vDCcpx?3ek?Yum-`TqK$ z;k0=`05Kq!>}zJQH)OUD-Qx2*HKG0Cb3Px|~%Pa|2Ao6clKPlHN)maV0k00sp#A3 z28z!!fBO;UCj0U-SKAcM9@uAVLvX3`nEb;}XX;}#iErfX>Z+IaP&OGAGtu?e{ds30 zZ@IobTg^vw8YsVpi@v4B*9-AY>GgLROGyXE?jQ8+@9xV3?2ET)ti z{;)cscR;RN2O!irqD^-{d>kH74cl`|Ss=n{BJQr{CajDh<2GBYJg`k102maiX_TTR zjwipX&c~F;Qy2^q@*c9^lSY0-DwS5*Qf|gd_0B+QmZ<;o~@9wrQwzn`aI$qCT z#mLhiCp{?b(U%#K!FlpCU8JVb4QoB8RPR@D4r*I>*cl!HnxZQUrbL(OE<=?GRikLB z^JMad*D(yBTwlcY8n+sYE4Oq>Z&6Hn?)oTgYH!8&znKjWLDJD$va#kVj=?+xPihvu z89S<|SZPaf5sGqLJw$+m97+#+f27Seq{2VrF{dpdTM+|=20Ly}_1{(Hr959nd{Yhv#0y~^r~}HR)K^q>-URM*0wuL7tgwH7KbNi$GBEZ)d{U0MM!#=D13J)zFIu| z6ZpNcFYa@iaZ8rYS>5uN9vy{<4ZELg1ba$*c&pPN4-YI41 z-Pt&S4|(I0qdRlIEOyTwt}MHtbQg_4jY6Q*>`;6)8(6i$U#*pm(Bjz%9VU(r_i3#f z%6b{Fo&HGM<}O^k-|7cn^m!QR>)lcC**J|{^rkwuV%upflpGYPF={=|!i)|D$;E}!Y+)$`F;+t?1j!F8nUa3(!0IvD)+1)e_T`D}8v z1KI7=!(%QMYOpV3Q`X}4Hk=?Vr>2vYzuhp>{56l`S*??E=PKKPwqwmhM2E$9$xU3` zn@BH*8KLT~dy>*qd69q5%i$t6WoOjSk2*|wg&%j}g5R)|@TzO5xOeArs^*8<#(l;T zRlfXU^SpTflC=J`HaZ@9>jlA--_wH8PG|^m_*>m3+9<@w`^6GoRoJs+%$aNYbJY@= zXjbgwj$BRt4Nl#ydK>wBN13eDqRVFZKt3fmI)WZmWtPvYlI)B_&FRct27mHhrn}b& zx!Wp&o@a#Z6f}+*LGpah-88~fn+5-K@3-aQU!NvTqs`~1i>1UxSAF{}&dvrOKDgI&?Oix4eD~vtl3Dfl zYl&5mhVe$9cR^)#+LaAy?n>!Npac&eI;Tz5eIn9HOJT(8#n##mQYX)nsYuZRe}xnb zR0a}@OaIzD)CbWv35Ii%(M&;ik2P*Ii2eS~7Z4M&`<)IAtDa zRAxv1@M+&-E5TIU9@)UF&$c6%E&Ko?8B;P))>7WW-jo+ z_vyKy(U~X^q-Qry?qhw=``DOZVhO$0v2!>*XZlu43w5eAmnMA<+KYWsYnLmx#^&v? zXJOM~JQzcd93yfMT;jaFxgxCVHFxm z92OYG7`#t9Sn#UFZ@Q2#NdN~Gro+_jH(f*86Ia9*O!vF?Fh?>j@pXB5^W)6hU`ifk zl^5qj+D-PLcu`4wnNCQVP)_4vc>6nuDt&$YZD=h3sc@K5yUahjBc~F`u`B6xFooYjmRlaC66Nf(35UtCu}ev$x5tK2u7d zGy3gR?Lm9t*jXF;p7wHoYBuKn^)S2>@5HLjO75OJxFmGIO>Vykcgd6`cn!P#Ve2Y! z0RQd%Ms+W)H-eO2#>@J>kZ|BlRu89^{)G0bVdH}Bx&Dq}_j2~bg8w-9e83E+$+Kls zxHmGDS@;J($BSys$%b3q2I6Nde(${%E3tSTyVL4+5(WwGuI35vdyAa%0*B);Gj7s~ zj)^6?-TEA47^*~xP_;YI&R=?VUucgZz6{g#-8P8t^Bkyl_vYI|+96}7m+t$l1 zdf)xv-O<=1KfOoc`*;uCOieJ8FVLBvm|L?L#gDuz@|#Q_B_puqn_DtkCunhzeZhIb z)AL$A|4uou3p490o#vD0aGudwzFG4^w!2fWjvR#n$AIr{rLA$(Z86}3$IV-BZDh5( z812@~d!ypFd%lSQr~fX?zxzP&xL#9kBEFkLiBiRjmGw3KEgF6Dfp>0j>HIdcOElvL z_U1BY?wQv?S|ow9kJT26>@)VE-YPxWc0b=he^NbV`mr3@FZD1jX^ZsC*30Fto3${I z3RZQu(*eGT(6U`op_>}_3t%gx8Nff#pWnYo2gSZDV|MKjq}f?b7~f7qh8 z47TPf#UV5?nL^kwmXL~$j*^awnX4PsBr0-zLT)I6xj)S?f4(-332q%0U0KD{JC~G- zj%*g!gvnT3ynnDybH`{0rH%^% zf-+~Rcq~~%SVhR%xt?+u6=zmfi!4J_MMg!&GvVflsqnlN^Te5BA46o9r6k1La*mm{ zVexL}!hrV$7VEtJOFqS-HDwqRChWxQ1UEazeiXHS?{T!0nN9RT!SFR;-NV6yhekH> z$5KLLqj>R*v4{OhTQko>9NB0uM^QhBqX5|(kOt?lVnmqKqL#jiwXbdi z-tufV?KOxv;Qm-smP&1Tx!lmvGdyfKjuw~NuBF`Y{*-}mlFy@Dwtwwz8!Zlo4rAhR zx#+ObNoLaWA~`e1O-8;|D=h7F)+7V>W>SRnz>Zbr)K@RU-t7tBC@5&(swx@-pCT(W zL&G8DC60=W-OcUP%U7!Zy)dK))$iqYIup&xZ(f?=#F}(_^8Rh*02MveiB>lZMmoHf zM%UV}VDh7w zZUzBo7hj|UmTTtjliJ4QC(xZLou|Mx6P+!>y`@77ikDpZl2E0eRKC*5(N82KX$_#7 z?wbdGeS$o%-HV!_tRavKkmSQ#ok%rJKu6PrmYNwaM3(+hXyLJ+GTZDe7+U{upcE&# z!Z6s91Rq&|c{4N7G{Rrdu=^&IX0o3cAx}>!lN43fMgPJ&}3(sFtlm3s9hazf?0X91;fM<;L-?eOqX@`mt-vpCQRa z3vx>IsL;yAtNz*Ru6vtq2MMFjIdurTq6q4Q<}iJWTwLawKGL?IVOwdPyFExI^4)*I z)jV~JVX-OhZs`Ex7~ZnLC9c(iNOzETK2UPThkodIq{J4REb#ZiJyICUv+R46itbuo z>Iq z!!R?EPL@?#M<+u62Nc)(^GRR__S=MMlu@zPl}6l$a+%klyq2P0qolj}X3Ky_VQx)9 zV(jxuwDSryHxb4%@0lrOE5UE_vAfx$q9#ZT(9@^0;*p9com}m*gv66YHUzg%6;YJO zD!OAXF>BtOzn;JXQ6t}tVZ?K)+!Z*`?&4^%tBb2TG^u{SeAWnmE*!<}FZXB=^`13( zwsW~t?rR!JVc?ae+o|itwcyV5l|$P)Wq6yJvZo}r7W~C4=2kN+d@biz2*rr~R>KyR zv4WqufG0QSlar8mOkja>j@r*HD@DHT*bzSW9dr#Nqy@+jkf_3>vFFK zb2mYy1K&b$YVQ+GWwwo(d3}W7Tt(Nxr(IchQg zYXaP8npUYhnNNW(+i6u*Hov-yGUnG=EM6p-SANF!X zi{;Fa(JV3+*PfW}ijyw{{*=9g;o%BglB}_mVtc}+%yixfC$0cy#o

    oC+U`EdSjq zDbiI3_@TG5}C-@f&KIIAMzoT8?=Yv0T*s3k;iTqTuhiOLx~e0?G@?dPierc7hC{(<4! z5~O4hDeKB&k5{3`c~dswGEvIEU+!rhlKynE+Z%MSpOUpZ^h&I?4(2JCHW6X&q!at8Spp@`;s5x5^?7M}v~u_!a9a&xYs@XjFA^l;=I^#D}S<#sh?Oo8;N&W*0X{KUP9U})}q2iRb0jl>}h`wgf zURP!}47wH7Y+vJAHiuhFKIA7@aL74Q3zS=XzvLIzM_&^~3M3b~Fq^I^$jqDN21k{FbBXEUyKuW^mz9wlmenDu!Sl z&)q}eWMW_eaV`VSCLo_%TTLWRm7NVE(eyymvf9BQfE#BukH3lCg?_$tT5G<24?RBJ z4uejpb|pzkJ6qMI^|VcB)K}MR%2##f% z+4@cg_GwF>sHq+Wv7?TeNI!_l;f9Sv>0!o)vu3CIx%9bG&CAZ3SH8<`S$Y~NHTxzB z!4YPO?Kmw^!_ccE_ih1i_aT z)-jwUc_D3;orw%pe4V*AbgqjOZc+MqGk@0dn2RN3XHk+awR_6Y=t`Ga!a(8tL+>_@1EeK$pot*`4 zH3I;o#}~(7fYRJ~qLY2#iSc({AAkZ45hM%bhSMVrXjZVW51e}}llTtBO)=n+1r%L? z6A~5h$fbXle?5N#0qDZIpbU5xiw8=5Wec>W-jT96=Ww-EK%Bbk0XyvAJ9!70Jvi)J zZAUVIY@IoUVw@fUAdiCP-J@u=cTu^xhw@-$8gb5VF8?qPjg0^?F5qrCxL7eLARD`l zB{+7z^d{w;6FfIK0NTUOD^UwssuxBHP#n23mkJ$f2FbTY78L)ri=E+I?d-_w2G$On z2@c@zmWP6kk>(~ER*%_^VFSef&_QF9+(lqQG$8;39!XES2Qwk*DWg8#H)a#G7eA(z|c0QfE@>o4^o^3PMHAQlp-hq z1AJEl2v&-Gx8-Z%Wk{@ulDsAoJ%%LQ=CeD#xLQ{1;q%(L1W4gvL&lH+iesBtNK`_{ z{!ICAJpFPY`~dn2P$k}04K_7}=>*!PO+jqCqdf}P?7IBPMF(9C0k^cx&tvXcSYrX~y? z_$d_txDYF%0CX)*xd5>^3U-zsym)Oz-yX^i`jz19FzimtcU5t+jd@Z)y2_?;Hnu`s z{y=;HmxS{^ZzQ^BQhll*=^ncRpG2*)Cp=u-ByOSwe+}F~Z+xGD?ayYQLc{>~$1Z6F zhj>JOZr_47GNTXV9z3)k2ATV$%K>P^V8v~dT+j6~d4E~;Uo@+es)13n~0Ra75v?gyke=Q_`_L5CD zcrX*-MDujW0WluH6NJaGF925YAEyuMU^f(U+{~HqX#a- z`s;@L=CFbw0bA7Z)3HUIe?9^Hg77FgO8x$D0l+1PU1=B;#g^`4rokh&Nknonn5|Y) z8|BQUDyq#R-3o?FQL#Ll2S1gy&L?j)s4bHwkV$xi+y+LwcXOJVScep>a+OWzzQ9cEwwog-sxnysY zHo+xfDYVS*KDPVTxNYu5lE-XOD`wsC%*`s?@ zAs=LNmoJc;g1tjB#V6_#zl|CM`%?ltAi9*vQl@}DqU@=Iyul9+brKSQp>_esT8a-) zz;F1`;D0#X1P5Rh2|AutSz(10C@??H#aD<>;D`JMMBcbtbKrZmUFihTZ-KFy1K_mE zi_`vEzR+C{^S$g>!@wuU-k@!ygkSYK)_$zc91@6jjh2sEB8o485u*SDw0*L%0`QFq z6D(D1-WMJIzJjn7zj3hKoZbr3V1;wvIQ(@mY@d)iH0lrlvi0qV4K&;Kled2lg9vyt z=8-WF6Iu+HOAZ7Jg+0V~$g+aH0{{*Caap^c0Rm9wW`Y?MzXWWiwuHWa<=~qOK?JoN zAg9~fU#rLs1v(jlius44Pme%8A0RSnquLabA7CcZJ+kMt81!xcG-0!>?sVOXnAP(w z1$i(?OV(9iUPakNm*c(VLdT`H4h%l{t9suXatwbP(d~4daz4X=V0J0 zWP8>p`UJ}RVb9Zg2cn%{h5D*R@4RtmOYx}9m+A>in6UD=X9`fpyO;qK;!+#eEC9(T2J!hJ!s%<{vDKch2n& zbg^4#bJH)MztIDK9yTEsAY0H~S2~uwkp8A+Cjufbwfh;Uz)DyQ{8qStN|*%u4h0vS ztBImZMZ6IRf9$+yAma=+_k~fMdLO>X56CU}Qrh(IBdC`K=X@yf@B3BI?-PVAmmqFP zeV`pe-O&y}tQV40DMTI+C1a0p@tiFo<(CvV0(@bNdIrQ3b6bh zB?<7i%b~^sY^8bU8Kv3vd}Ghy0kho2nbi7YgG;7)mu?0~6am>1lS@3^3BX#P1U2v} z_&&(gmZ7D!1iVBbA1=CSQDEI&QC8LkfS@dE5AO~xW(s!csE1nk z0zW{Joufbq^pse5-2&rC(x8`c-dxZCf6p$kd9NwdPk=`{y)OXUGDxC!Q$YM;dI(j_ zOIUxw6`yxLsJVN?TlE(pgb;Z6*D?@69*U|%0O)&<{s4X{KIt;?Ma6gmj3a>XWdU&P z%l>Z_x?TJ_z-I;Eu7)66{8L%Oe4-)$@DM#8L$ek-eC*gwYv6~TNl4^mgmitM2o`c2 ztfGU1q=RD0G*lslleM+0tB`3rO=0oaqlmPMj!Yrvi1AQV zR9JLqUEg3Ixv2>mnT62j2u8F>+J<F-wgV;8oTG517Jg*x7E&T+gMT9l$KL@;rV*z>ik6}#$YMWs(u-!7u3fR0PeKUk>E$2OKpb$J3W}#(0VB5WsjzjwtaOm~F)Y1Uug!H!9=nspH z%M)n?oUZrXesSU2UzDbWPY0gLmbYduRlep1xDKw>bZ%ArWM!={ar$)1oU~U{mLgrE zqIx7NA>LQ9$J4)81Cz^F?2@&Z8-0=lrBpIA~t%cV8pNBF;8Jisfi(~Na zUdz>U;2xhCBKqo}Bl${&-uy&!(%@c8rfe(uLxgsHgLbfb>aa%EhFPH3zQ75yXP6A# z{SIf0Fbd0ivOp<=g}3FQ$(ixyMww4-8wsF))xekcc}UX;o?N~A5q z-XND_Wj6G5ejhMJus_k%i6z}z*lL@=dsFIK`<2W+Vmq_OnZ^xzXtk;&HS&ehqyW-d zx1eOI6-ZEAy@}_5ig_Phx&1<2rKw~VU>VO~!R$T|j?@TA7!2p@G zlQjBr=7*AHfAE&IYV2IzkqQfn8p+A)tUqdAwj6j)7;i>YY|ppQK%@1~A|?rrn6M=M zB?4cLyl#yQv{xlZ}-p|`(4Dy)y7x6rjdk$LZ&MNjL!lZQH!Gtc9=a%**Zpyc(?OUx~I&MX(0c*C#B3}JC?${v?2GWEHsoWo(G_m(#Ugz{bSgisq zdZoLqCa7MiZ5JR`>tKJA=)QVNeEWf-SF#RUjcl&t*y180E*1=(HzD0LKFAvWH1V@eGAA<0s6M*m9`fd@?{r)oS*15 zi@*}lrEmQ$kc;SVy~IDDS!ZgpR-lZiZ7EI3hXr6Ogh+2sJM>)4YO|xtm^o=*e z)b2akLtA|N?{%(FlT+9|unvB@&`O5npmRD_kUh^k1~Mv*NnZaf$2)m!HKn#;xxlGt z{Jhy%O>C;u_2uRZ;T@hig|CMMnvlwuQH_015looy%*0ru>Z`R4DKsV;)Pd9=m&A}E z+sbi@fiS{g3E$UGr+%w^vdkxvaaK1|_Nuk3Xe^?UV2n2;L+~W2rsk^-FdMGQC-1; z(7D6Nw(lWL?nV{{B-dQ-O32Dk0+ zuh0lHz*U2B==wE$)z+%1g(J^@8eK!yG4Ss*)^&D1BJs0D*iXq4+-_|OR?UuoLZ2H8 zKkm(@xoc2=Tf6|TNe36#H>fNpj_6s#h#7Fm(W%)_kwLaC4~26Hf{r0`l~)@~yT0`A zAB%n%*W`3#t;LjSOzZT@aKp zvJW4RC&hFQg_zk%)R%Q9YdrQJD9YHNkvHP|SJAU2riIXbftJC7>m1(p-p_B}-xYhW zQuB0*%Aa%{in(pz&G@;~&-&`>WNcyL@w`$M^kz6Qe2d=txSI` zw=3R)F{5HiNlQxqa7tVXnAnvaPVPCLPwyfYvm~AhH)q z?>6QPiCu9MbMag$q|}p4X&@quN{7lMFpsLTf{v*uRR$A4LZjrm!S$oEVEGS3E zyvLx3Dku3FDr0Ew=PT7zVAv4$oQWl&*iVYzINx}vP`V}<=J`}w^Awe|K}`J9Yre`T zfPQ>J0(@#jJE5&9o4wJDpyzbdVvA0dfIO^^f;YExmQlWgl#3QMq2p&bb?Rnat&MD$Z z8JLT07TV<*hZF5PziLZZ1{8@>3GFU9E-)|Gam-^@@^Fe-OU+_R_(TAS26kiyEQn5Z zuVcsPN_@U8=OlbV)aq^6-e%L<{nmLFt()0ks)c1P6ZR=mQ$pPde*$lk&6~uzCd5N5 zjkdZa!RvMze+FBH@s6YmV`w4%De3FmZFAIz4b5(N9K4i4mEJPq5>eKcEX~ zJGdM^V455-LZ7MwBeAM2QCse%#})4W{nXKqLWX;h)3m^+#YQ@ejZVL-e{I67t>@vS zo*H?@{f#cMo^(zbb-ikHL4_kkU{H+>4Dz>w>R8hw5JDfeh=rxbVnqD@$KT)nUJB zH`!vmb-g83mPi$hgUinZYCM&PsHu!;2*^=l2Y=3|Ew!nzp|;w=Cf2R}riBijq7X)^ zJdp5pi|JHpIB3UR>&9$(VJCdUjEOw--ioc&Uo<_9V@ z)zIt}f@pB&hA6thXw;8N75-fGBA6HR8*AEdZXe8LpV3#li~3#pto znGi=b)YVHS{UPBcXYcau)LIQaYT;3mWOywTU=hYR66V?fe= zHN71JKB~F1qmO;sTHX#laqBQVSdPjNDcfmYGaG38r2arcRRw!;y>sCeFTS)a$4zJR z{+_&7V-^ZnTn%>$pAbdBY^5VvCEtAPzqL{%dO?DC|22Fdq5zunMwzao}b- z#N9`R6228JD|g{^d(eIdK9V?Po8?fe4dW5H*WxU{_}s_#7X+pK+Y%19P)6s0{j3o_ z@W_hb(V=G8Y)mqmA%gq%unZF0L0w^p^qL@;uA(K7H5GfNnJMe9OprrRY!*UZ-A{6d zlRxb0o4j|vXEQ^-E{19ZSH2n5nr6u(@s3@X~FvfC*{CdlT<#k_NaTN5(MNLS5L_|^jTY1H1$eu5Vi zuBxj-!)yqAPswsTrn^#%5S8IVQ|b=vy5>^P4&T2Lg?iMwOZWYNC4z<>r+4w>8#Z>g zZulJ`;Ab!m%#q43bWg#j3i53H?TvfSuCeaX%i{}qJ|>{+TLzi@MWvzj0o{Prxavaw z5lfwu-gt6(Y}z@cr>~b0J)lo7ewdVW_j3f1elpZJhqs zw1DAX^MZ9v=|6G03nB*>~erUxYGPs`n2Nu|OK6HOt+G!G{x0v7aw8u+Ft> z>mACoL+Bfax5qAb*?YH7cc}-spxEWOO6E2Em}Ww$2Q$v9iSPe}>A0Be4}LS^1|E_u7D;{#AAw;RXjY^qz6DI9pOUj0tk-~7FTeo>AEv0q@J%~Y^O3tWlOi@2f+ zh;ca7E0ru{Ub0Pke_y7Ah_LY$N9mIjy5;Jqq~TIhM_5rftz3nmr}G$xZ@lt78uh;Z zVq9UV=RzVUVD|t0G7`OG1lJ*_dlA&pjvBybplrkTguV#ifkH@!WFOTT_UmUuLXb>e z21aUWUdxL(ss&a^mu# z$AJSxFH{(3OjHIeV^DICm&9;}Gu0UuZ#sWA!x=UDZHc==n9>n1$ps2VtUZ~n$p`*p zT+}Z!xONgl?7;{xEF@4E@8=TZ4O{X)3{q`@vr|cO8cd@0S0ld2Pqe9$NI~ceKECA~ z&gA&!Y&R{xJ6*&|wNpu7J}$v84!<+_CA|I8aw)@pd$tvgQK>w*$0SS6i~(Z4%%jwF zu|$`@2Q`nmsi?kr2w_14m--I!5axhKD;eOVt@u!2<`c8dY75kLy*;B7XAmY_AoUV& z65HnfVzls?fI0Cj)Dr(z#-&4sCX-8LA&HUZG+HP`fquZH#$J{&=&U^LQ3~Y3(j+q| zB0ET4!OZ$SS3@+fIiM;5VUwU05`$Lxi*N{q3*t1L;Vjzn!9rzjL4Z?u0ufU=zKdV* z6Cl7w)OB>3_4tWqKpmH}Bvo9N#-GC`_q*gc7d|IfAi_!D+;0sQ<({kbS<7rBFjE9mDYKs=YbC<$8*Dv#jq%0-mlxyC%FVS+S5mlRz$eVyk~V!j>us=Q?n1k z@Qf+&G+b*1Zs7$Zi7q=yT@%7SME4**(%__PsJ2qHs-jfy@B~VC;L5j))}deI6Eih% z(iC$*o0FMy#G9}S#K*I=Wz#o zx_%odxo}OzXj7(f6i`>*)L{5;^4C)-kj4=iKh)m6wl8Xe1|#?NOx<+?W86XxkT2D= zn4E4s0buAPiq`H+zcH^s-t{2Fnj%=^FZ%1Zyug&&ey6%O&lDx@;T^I*FWP3H`wYIR zSC(iM&?ZF8CdYW|iQrV)ie_clv%n%qP?eS}6po}m4M-%><|GCQkxf|=M%<8MHp8~E za4{n1HMl3*&~`k&CGJ=mdCXI2rtWvGh_?C$ROrH-95<$|4;lTYbvd2}6!$7D+Bs*&}t00WJ{i`q7TUd2n>~qbHItsPC7JbEDouvgr3?IvBcWy~dF#LFK=Vh9kZFI=5s8DtIM5 z+D#|r*F6WxC@iwEhm8=+e{ZDba2|ef@{eR{$amp=mQDJ4;VslHuZ*S z&wpKjG|#;J>czWtuw`<2`%rXheBSrH2`p1uxwifZ&GK!@RxM*kx-!@?bzyrpvSTV! zNB{N7(l{OzW;du-|zm< ze(--OyZ^ZiW@2Gt;QW`e`#<|7DzegGjELNH-q2H^7etCb208m`(rArfP9tStI6 zX3|#2e(a&E7b^7a7PcOC2jhEJLoIP6B_5PwiQTaLl3s{%c~y~sd^kk6 zs8KxYW;wGJ96jn4Zi#XHe4TpnDH<}o8WXeN6FAo7S%Gt24_yu|ci@jNgu(LGqs%Yk!A-)Ktlad(mcKlI-H_gbI9U=A-|%Y9U`%xQ#-*nm zD*srrw;IusaTBhgkIffO%ghiGSzsfXpUa}u!N?ab`55xZE6!zDh0LYzUs$7HZyIMf zI!eNpit8I+(q+>!?H+u4&*bL0oRFJ}mz8#!`m6nI@{C~A=6dGp_)edZ-}@?lnDY~6 zV&ExK2&bFIW6G+h5s2nvi#l6`47q4R((YpJSU3?uVaE7hZ$>|HiXe#tTP3@|jn#i| zGvJ2azqi?cdA~eTVbrph0kQ3n8kSRAWMY)^NWnZcp3TPG={IS4K?5<7S0qW%<3$;d zj{R}Ymp)5ZQ#Y~RiURD4#UrB6+U{h(xiRGZNi?~Fj{vHs}P|y9GXvOk-5nj>+?s_LDcma`u z432|*2pv}F#Q$`P!Y9REu6qGv``~^}Pn18=B~(m(2)2IC-BswM$GZC^h0A^62eUn< zPB0a@_BV`^!J;ISEF;BORdMT9j+|3wDhfrNnD(X?*Ov*eEQ~;KHS_McH~ph^%|%kb z0oVEAkH_({NrrB8^OT!_AEMNVbt<%`9_bG0eQ~0dJ2a!S9Fl@^Pl1!l>Gw7e%-2V$ zN|(kw2gH24;oa8dO*?-biMCX!@{5vTwAsL>cV>+lfj-iV zrh-ZHwo=4CMn7CD)7%j}Cs{ND%#AWW^UF&a!Ie=j;*wEwgCA7erXNiu(=Q8|R+-~D-{fm8Nyvp4^ji!!geQE31j-zIGt@6AV zp&2o{6dJ1sc-(~poPR@hj)7v`7~01~Vv*X$ejSw;jH*ICYvE4el27Ir*A6!v?;k&rr>!D6bM?qZZUOLT=t zRck8i4bTa*Cj53Iv4N+t%d01Mi zCTy<=*tI&1d9}Z*zCbwy=G1ON>1q&$IFC5nD7Sw9wuc{9vId$OETgn;NW-hNjL*1i z?A0eb$3Dh8?bEZJ6?Ycii99!8eq zRD~2320RBP1v?f4UbZ--Tu`?V-SCq~K?mVb3lqpGtn!&~12;Bl1DXQF5Ejh9C(=uK zL=b_m+BF-`BlUU#$poD`6e>v8sGH<)bW|g7Y zlh%upaLMSBR&ukK80b_ZS($*}kJysyDBKAsgeepp^~(Qj%f*zUJqo%R%@dY)V2Ugu zbqE^pNW}>YS>N@bt^<(^R|y=b*)KP%HFGFs)bVhPSc)%X{P~QjskPpp5-}^Xr1igT zKlz>hciT_ByjRQ%|IvO5|EvA9c#S64V;jOxMpuFfg_ZS1|1C>kE5u${?MK%U3Sb{W zwh}!5jFv=mftNl*wERo!8Ugg|giDmBMV9RS4BF?Fl{Vn(EE$7+(nXkKWp7hiJaeWSygNvCUOYyL0!#jJ-w z^IqCjO^Z6o_`n;UUBbu@au4$^q0FxYMF}5rkLuRjHCHW%aM3qLd$_euFqer|RtJy8 zpTV$66w5yvRWWOlu3~;jHS$3D>_i?mvmaE_Vp_Ed( zR{v{DG8K_JOE(t{jT!x?a#-*-hz0%peWi`vLLaMGi!r#HX7f;>5={!zIWz=aFI|ET zMN|>1U!qOB_w);GQq{nT>e+}=C%iT4$~b7FB0Afw4KFh*?)_g0GW7NBKN4iCXPLr| z=vn`^U76)GMpyc72AUo>&4q|%11NFd2}G&JhJ)w222(j4s}%`RGeF(hH6*fw&*H>N zL)SsBAvWj(?x=;u+s)czfcP&zLCxB?u{`gaUw1?ZPDykX2}$nB(ahW3`RgxaJ zqzM;ur^?h?JNxkBA}*@K4vj~0I1t-*{1kug0nG1(&~U)uL5X1qUz^R(SL!O2arcpZht9oa$>k~2(s_~>0jf>>cJ=6xcvCN^V;`O2tH*9ZgtA_F*%?4 z`Kz$zC-{30SDPw$6RmDe=U*sd#b@tnP^&HD%!+??69l%O+F%;^oep6_$9XiqtD)cz z>U=*$Oo3jVI+^l4ajQkd9c(!*g_|x?+za16-%9uTgsm5#T{H0$W~g8;mH7KQUXVgp zn;GRi!cb9S%^Fs}3dYj>Qv>?N4}bu6ElsxazSa@=ep9Bu zXQ0>W1(+$kUa-8M-*wC{BZLWf_X*YRnP>Il@#^(T?x!Rjs6~rb_$=Fdn6tb6`6u(& zGe>7%4-dYhHm#4(eUmhxBVx9ze~3MHd50$nmD+aI60db|4+ys{@dd* z{snFLr+umaYvcX53xNMR-v3_!{C`vc{H@LYwkJyW51ai@9!cT<-4Zahur_g`|LZU9 zpCM`gaELH6(lP>tqW<*&c4f`~Tz8h}p)fm)Jm^P=D5yb#$>5(0d73LB5+Xz*^o0ah z@aIs;Y3wJVZ`+7Hq+zvRMhL%vpeoqL1Mt&{#D3ZjF(WJf%Bo(gbfhTIn39t!tN~9G@M3^wVpF(~QIR|WgCvV4Jj@!*uETOp_S+Ln zu{*e9h`Wq{l|jda#6DAh*Qo#%E5sdmTk!u32muHA>Ache(WlhG!oaLyw^|O<${>_X zQrf*O*<|u;q|L%8rn3Rtn6(%?wrV()V^!BxI&;9-*oWzIpNZNl*BI+Os?Nv1Ol?JQs; z>ALi6dpxEwGcz+YjhUI5nLTD^W@ct)W@ct+Gme=#{{Br?n?$>jvTBvoRjMxCZr!TB z=RN1V&(C@B{@=UIla@Yjd}jhK@4Iw1B_-N^{D6WF($V*+G z!&zd}+Ki~J;GTtd5JO#tXFLJ1w;tZF9Bjzu7p*P0azM&zH<7*9?F~EH=b>%~cPx6qH~xrix${ z$!od(DLg(W*u16jr9fkY<1Er87KHRSWXbONRar=6Uq1G|sOY;_EG1t9O6{WZu_>fL zQXfqP)FH~CciDI9b-lm0Lk$0&YKuo8;ENVm*%u*X46V7jw8lzT#hHmA>IGlnxMft5LvDst~M#f(B1FLUKjfTBEw`^RIX z-`tyJn&3>amp4D}25o4rkgKR?xy}wttz&~_2~wto$BcJ+w#V?H1C6`82beV$qDp5we@K0E|vNIre3Ig&*Ze3XV#%@I;1`HvbboR;RSXn z;vmgz^rpx7(>C-u&Gm_E>CcP^+63@SnwvLE{g(1F?8x$ZAqS%sFjsyZ2Ojj2i8cz; z0B1$52GL$_iZya8*j3_rOd}k05OtFi~xtcbxdSh z_>Ymqt~DRwqQL4md@eqr`E}?Lw7(qMH=9U@b(Y!V+_xT(iF3+4Pv`_TlQ?IFJ_!cW9qXVYjN~wa_ zmf&;Dt!NR7NM?+&t+qb6*=0iR#_9c3whu21j^m)SOXN1uC~p7+^Kdij=|ghp2h!}2 zI;m`k*KxvwGh5tq9=w&83pzp^=8`!10L1%anJ>%hez;CQi zwF!D{z;kt%XMPkk8pG9{z-%hP;gOACOBaEQRS&DgqUbn*Nu$`lNH?+dESGI_KyQrc zD(T-qz*>jNnyaMUeqGG%1D3g;;%q4H#4;{h{hP)cEc5DsRI@H3-$S;k*2)0kK0dnk z#MX-Z*gZ{U@r@i>r%7c(p<01v)7T5M6brjdeB-^da)mrxO(&vsz#CbP58a40fb4^) zgV@JU=5B=_aCzRmaQg#OHLsB22Gwr9X7 zXMUVV3|c{(_}qnk3dosy-ouuhu;@fktk;$+*Cx%n0gi`5RB?v|*q&`rQ~RZy#UB&y z=`5N+MGy4pr?qKKa0nLAsjOXvgXt%;41$*6s|ddQfJtCT|Fz*CO|>*K-I1{;UJyz+ zMDM%Qovf9IbAhNdyj7yJawjG`S~s-En$zdq2vM|@kDN)HX{s3%c{T5>aozT`@ml7( z)dat4I$GSfdEPU)Q0YQp_SxXtQnR&z-LmEmz)KcivU{hsZaO!SUOv6VEWpTA-PDhl zn`mm%LJQC5G_lptcr+P#K;GaO6imCkour25$Wx@s#}^xBX1YTyR%O|>t|KIx^gr4b zTL+*;9HBfQsy2c6Mve?5G#Ca{JdpC=Z22<|hg4!*)B?4wwb-j>qG8ehDmwZ(ur68Y zuABT3kj|{pVq|@q$e^P5@rV#i=P!`eOFh3(y?ry%r5S(LX^_1Sv0NMmXU*;EwvdSe z8r(!+lVrPWv&w{HcU>R?e}PENYAlg>wmF)Y| zCFvkGCb%WDhV;?-E@Ez>MbrIFyR2_?y1Uv(yuw@qzn*u^B8{$9DJqRLpUeAj*O`l~ zebuAvut`U=Eyb5RJBh0~BRhs4mmlxFt}6fKu*h~XZ7Qv-$z1lqB@ba68<+j>TH#}Q zaH5Y-#IM=tBXNx2;?#p9tUuX5NLbRmeB_>;N>ylXo$$Rszj+_ENe5Q39Uy*|S3BBr zb0DYw(7(}RN$fz`i;wZd1Y>36c}|o-c~%#&b}wS|z+S-P&6NU+dU{IOmgXZT3lTGG zKfB4s{ET){Pw~7_2!%r72?PQm5dP0KVSRN z{_l&w|7+1_Wot_N4>k1rb)oS~OX~D3CoxdlfhIqif|a-^)GDJIlVpoR*wkd2S(M3f zn8UOkvuW}3T?yHeNJ7#WjgLS+LzCP%ZX?`C+px$=!^QFg;5a?WT=c8s{p)MyYT@k# zJB#DkE$8Wtd*;b&`^gQ?kj=+5njykwxuTa3b7vkz74C@96>);Jpa^0kIf{rI`nX|g z6;Vy!CCe#-nZyy)| zJaq0Cf*ojo2%)?}s06-3D8Iy1EZ!svL2dw$u#l&}H5Z=>q<&uPaBXZz*bsSJtk6&* zc7O}7CLxmiK0I->ME@$mI6_$-0x&gFoRy}8kj-)&FNivIA8)w8@Nm$s*uAJcS&%&< ziM$lZrP!{hJO#h-sKDzQ-(0axZa91Rr->Ekd&CaJs@BbD^2NmF=+d9ru0)B|(m*Bq zAvkV-df+M8x_BHYjdF@yDS~`RLyj(*og=Og`R1=+R#Y_^u!}xTA}^q%DE(FKL`k<__aXe z_&tSn-9kGR*(wZ90>#D9li=Z@{`L;h3zI?|889L$#6|e(LFV9w%H^aafs!rH0a2Qzg5KZ2BzqnCX8DHDU#+5WnJ+Qi3}v;f0^U-SF=z zw4_V0fkH+UeOcj0gObO^#!BEIQwR}DB`QTPBp$cS z@lGr?^%%PkK+ku;O|rw6V7Vw3BN#_we0K$h0hKHwOoa3RuQqt~6#5f6z>CStKna<9 zd-FCj6Q_Tyg-xz1iB(=DO#A?%7mUnoQp^e?yQMNJs{9!~rzWgOBe6 zDHeDU%$|=TvIL#SXcxlIfmm-DOP=(LmzUpy-~u@lT&=uaM3lxX2L!iE&N+NWIk4|) zmlv4ikpg}`oGauca;#TR+wfq)AqRxGpv^Hcda?Z)Hz5e|jaV^ACxtU;^dPmGfl>Sv z$@jf>%CaOA$kJ&gSWZFDCMb_v?cRs+EeipcGFbS^FA?EAP|t9~Mh{u=g@Sw<2xz7V z2ungv!n%r@KMTaj*Z1&X`o|vlqk4lpZgkfOcCHawsQK;}OaRJ~fThP6DzvuTlx$nT zJN(@VY|gfbvcK&bLO!ArHIiUsrb9!>Ks@~*wnJc99>2JN?dJ&(LD!E!h-`R-0TVgm z)iA%H%n)%x_)H^KEFj|AUn7O=5>uqYkh8n2KcX#sGHx!N^Ox$k+|#TY$eXl*)(h90 z&8wVjTuq^xM;2iXHusd-&kEd70$&F_CE(5i$4=6_qEt_xEA(FViE}Yx5Q{8;g@L#0 zOwNu0@={}esU&_%gvnp=j+eywokZ?!=fx^Dz)eqRvwHiBtr5Lw(DQ`wH_am7{tS zMec=+5b4qK{G}nC_rS*;ztd@$q<@4?6&e-08=+}Hd2%V1bBDSD=(Yem{*cQ5yQ3vC|BL&7FrA$mK?ztVA%k7hGbI|nw0hAhWsFq+4 z2==M+@zDZde>>l(X(Y8TGCV?lqIqeiTN<`ky@aSg%XP`ajn2w_1VypzM+EB?K*z|6 z-n|C_j4-P1=Pe%bdMVEk_+lU?P!wz>IyNtxB3_!o@Yy6f%gWE{ic0>YPDdgkuv`YQ*GHNKE zt_8CmdD#`?=wr|tFAD{Sy0+JK%!a0tgme}K(Ws-GGqTSZZK;pvSLxd;lf)`6@8xy} z@mbw-tQ0LiCungIIARvsDs88#gDBhEvEK&|DGZ8{&cPV!AuHSDbtA`%o~mVi2FBT| zI79Z?$4sWt0Ajt)pRJJ_>>(^2LmZv6t}rY7lGcuf(qy`Aq_USr4^tH_?V=vFv*E8O zi!N*Vy&dKjP!H*flFGp%waC*t5N=@J9sN!*vF1@|Mj zAHQ%AxcS~ota|)@Pj`V!S`~lQ*$+fDz~obBgaz9I-eAb0CDY;DQ@m2(eRUX;5|kpKI8#&f^-vyIH-gm>(8;xqH@Pp;%=aN1#CO@I?t z)iO0EQOnXC+rmhQh;@v)4Q_(#UC`q#R|)mya^}~(o5sB3;8&nXRLjN1#)fVbl~$Y! zOVj(gw)z^%+vCjoWYhu$ZlbH((Z?7TR;FNgLsqxN#V*D)0kQDZPuQqip3EB`c^M{l5y=p@i)Nc~NiEs+3} zNPS)b>)^Txa{&y$m^kIM&u@wN%<2vxTf=B5#k z`pf*-U0l6feUZCt`jO)LdZkI0Y!!~D#FDGi)H-|fFeErKP7{i`8ip$~{h<5qVEFCj zWGUq@u2Y!Df`|;cPo91qYaX|OyX#edliX)|n%wYYK7&i7AevdUD^}vNDv^A6^(0&a zywu7!c$v!M;Av^%YWf^&PImZ5MMjr}#~>dYuLiGCrgq~!Q@*V5`y5-( z7xzhjPw2z~VN`@9jHy=Chj*_olBnwu3E1$MN?*>`?b&bJ&7GH1V_TgXyxTq5W=R)+mJoT_|`zI=qYT7 zwIC`cd(V90*QZtwnZ<(py@nq;faNh#+OuxvJ}Bkq?nn#_3~djy(fV3dgAH@4wQQB` z0jyuRS{IQ93rRnjNQaJfWtw|-4cshDAF7A8Ulb$e(WYP=yylNn zcXaMw2Zk6zxOK@S5VEo_VZDm6(JRsLVPn*6%z3L;@sD-{KYqxQMK7iWlX!W2UQ}20 zu{u66n?_Nzxf~{skqnw6f#^>1*5)1G?_wzgdS&eGc5Qb^zn-GixM$a1A6kW-yX6eH z+#TFuCwHHasomy)sjm~*e?$%gd7m9V7ZT4DQjug>>rOY@oV>ZGZ%;N9d9|pT$+Ua5 z*R~Q!s^}|T8^|A`R+t}V)O^Mdb2#&>X=KE4eNV09-x=OCW!!O3jn~N=B$M>U!{t~@ zd21=Ha*8tU>Zb+2sCKq>dk_FjyeHmw+j(&*5)rdtTUk`qbTy}6$K|nt?Gqr3RYme~ z4L;5u_X9NCR*9Pbj`yqDE>xT7FdtxZm-el()nc@-kuB8`TIu+k8)BQ5sP{i~8D6wT z|K{5U*v+h5s}Ej;z;HdSo8;!db_{t_MLi7K>q2iM61CSqhnlKrFVg*W+6Cs4{`F^O zE1QPzG?8IfYH4rg$?XBpMlaRvei{Xo=8N=VwrU~iPIk!}-72-4u2BANVzfjsxmvsw z)t;;tG}uWaLxPf3HEZK$M~tL}xXl7I!l{Mx=DY8=sYWkja(S(|EMrX&fbNcCP_EyX zUe5Ausd+dAmA?4+g`Ah#wx>6kN%6T=YDoJ8L?#6dWryAgwWR0Na=E*7)+%m4Y2P8V7!<+3o)PQ+~LNEa|VItBZ>IfQvJs148pas@0Y>)TF7Z3@bqY; zdU#7ZTF{PS2vUe9umPsL2Gc1H@{a5R{%OA$XbFM*wIY8AVdHDi1r8uUpoI%T@>2Rk z@Pbe<3wqOQ`uYNMBO}<^7jNyU{S1WX4I&5(fD9r>5Xco!h#u}prHF?bb+Z%j5ARRy z3&$8ML%^j*ftTV^5HeZL;>A`c?ca_NSRanHBYy2FMH)Z^A(4{;&4&2&kRKELDF*Ry zd1a=^A}5ah;>tvd@>ygDYDMc~JaS)zXGFr6fe&wg;~B8$HG#M6M5(gGL!S zws>cb-46l<*?oYNgq46AEDNEQKYWKW*5jz1D%F={PhZ;sQPfar=NJ9B17;81iNX%q z?{a?;ds87_c!(A+YODcJ2x<#9T|!~}MC|CjMb}WceR!B##HIY_?TQQbQmf49c_`kA7R^JVd3I9$|Zy&g4nUGN>!u_FfzRYH-L;#5)cU? z&_X3};ada*9AlCRqThuah@fD?AljD|tnfYOV%IQLUDa7Ydc=Z`AqVoLU!N`dr&01we}93OZl z>=^+ac<{S~jX)xa1rh3kkY4vp@S`Ye)5i)v2;j?sh9i-eSiZ*PXKRxzxEO@ayBYQS zi7h2&5WbLy$lmS4FvC-${*vSTyVV+wdxtwW2kT+`99T;3J2_&D$0wSe^^2IlO=4XR zmxgc*))ymwAysD@6wHB4=vLwrB6K)N4kSDiB=r~w`U^SaT^Jw18DlRL7Cnh7aI$v` z#5gQ;`**$(t?L)iSX85^9j;!*84gZco)?7cqR1K%uPQKDPtcod;c5uK7{w(FYB2!vUN?jvGkMQ}j3Td*}e4j_M*zDXq# zeoZLB4tjk)#t&RX*h&z;wm~Rpo`L&@oq2mlOKuYsGr@qiBY;_v$&%L9HGMgMjsgKs zl<|NM!eurbFUgej$NPR0KsoB3_`o=cVWDfxgfl$^YQ(j2Z_8coPZ< zm&#EFl5-5{%YsDo!#j>5kd}auKsr4aOb0GP!uNA{3PEK2j@uyyV|9Qf4d*8Y^;!kO z_tQp{qW(R$L`(T7M zslQ9~g56Jihkl2(JR5w8Ej;9@cuRM%FA!gU7a!D?FG0z8u|wt!)j0G0KxS^gCfVK= z_vXulLX?a`;!Bt2&~sh+kB4kQ=aFknQY>6Qk)?XAnV4k~(`*q!6Z|RYq{ygVfq8 zCuo0MI@rfJu0e(gEX5)Tu{o1@aj8ceiFlb_@ZqTA-^)8W^Zn`lGX2_F6s5$^qDf2> zrK6R@$0j|TLbN_MJOq7(KYKU>65SUN4#Z_o!QGQ2;2DE_AEcK=541rfx=MaI(Q`D_ z{V)}e9m9Y6;VsEAh(&A18iGB#mnCS8am!Wyg=gyk(1eZU!ctwO?72bF!+uRiX_c85 z{?;Q<$vywkBZ@9T>_!!Ax*Ni01>O0Qq(&0p;5LAvbxp+Zj-RR6!l|=mKK2 z(+?bfKUErXXSH;}yN#PYTZOeqRK+B>fxL3I2WndJ2HJf?eXzz}t=hUcQ8K-{!~k-_ z;e=VX%!`AIxCtBqv$=t0$!lP4nJ3lfTl_$xxqE0JCQ5TMy^@IcX#}X`xgK!FU=4LO z>4sDbYsgQTruw@2+9JTeD-yDxx;hN#7D?8(%Q1ba(x0L17FO}eT49arUe&=x)kjfX z1^(eZ)KghD?fbjaZSisLIh!N`f)9e}=cFibvR#hb=Eudpwjx87v#%y%h*O`7bYDa%9nP_m`MI>$1~Pj^ zS;9fWX(Q942FaorNhKoHR!rO^w~tz<1&69QO(C|{TPG~D0O{b(G_J~Iu8MkNK*fkK z>33+h+MU_wgw<+Zv#@u)ScztN(cc_(r?IT3&og7OvGSp_EohZB-A;YZu z<|yz|MZnigo#UiiQY=U2C^T`kGAvWRW~Ng%tT%du{brA)IEh=uSYfq?=$|7>C|J)PKxSaEbZrt0(ymeXI>)kk|gnGajFJ=|Vv$Np>N^^~AvXfu2pODHrJxY^)BYrwYano|q zz3|A@;5OM8A;nzALAGIPDShx!aSR8x5B^}0B|02sz3_sw%ky!X)HvFUUvhJKd&=~+ z+IjlAM*4gCo3eDj+z8pW zYjNA_vUtRX(^3w{1}Ts5drZZW+s4w$aZhTikM#Co<84w$Lp{|a%GB0y^aAAzd!5O2 zR#`?yWavd!`mj@FmB4i%(or^>fw`#)jAiV+rD2gWaFrTXJn!A`dDKI5&kB&5N(}KaO0-O zSE`Kj@wc~Obs5sQOisahY@X-F%uw(d2yg13NmV!F7STrBIVZa8$!SgsSB;O>Q}p+@ z9-)HB!oWZ`daq_){;fy2jsUB$kSaDJHCEhvjT?PopB?6@a%y||FQhkho}rmzkcHQi z?S&u~l)%iHXWhTl(chflS4A98$&}oXMa?*F{oT{dRQ{{n=?qyi$aNQCTNBw{ayZmj z)|uK}Lb)4R_?}17LMzNSXJnOTZ$bJLol@M?^5%GM0rwS&w#=r|=G|$b=6^n01){P> zJ{aIr?ejN%mE%W;?KXlrx`& zM=J1vCUv)nYOk*Po}P|R+q9pU0}hhYx-gH8c(iZ!DFSFDbt<3CuJ;oaF=QJt z+$EN5?O#jWX%o}MlMEZbmZ~FPDQL#LR>G!Spk^m#(mpT|pNeDBv_7S#3Y|uwDOl3II36?dd}J8z zCncdH6RHEizi#&~`NR4a7AkalHre;qsE)%wIa}uMC2>D;kGm@W>cCdGyVi>vTCT1I ziomoKe@s)1u}L#4R(mVwoGO?yN9o^M2^Brv7%z-q;PH&jC8Bh%w~|>*h}=u$$<=%G zr=8xA@G=QnxbE`dVPPnF+Xr5(*&gfIGET^LtW4^pXFA=^pOS8RoZXerlC`C$krZ40 zGSwVHTO{7lb*bZSt8ji|yz<8w!jMFxcJM0nid;eOx+#fI9mu9b8VXhMaRI+D2M<2~ z2{AyVqM5#tj6R6nUgLv*_8bScP+Le>vEl54#q8CW9iqMu$EwqA?toyw?%#`64`+48 z=XhU?{F%G#>RR&zM=E$Pxy&OC;l?X_pmA}xaKXJM5P~uje4YGNA)fB7=4$Kg2*g%J z%1bfEbUNQ>?L2e5#th?bUcq*3b(;GQ2sD~G0P)5cn}`{I`Fw9*j# zxk#1Cam0fUr_*er2AnG9m}lE(;eya4k)>>*=M2qqS4kbcIu2h?DA1t0n|d;c2eJ<@&o7>-W?&Kt-MRbq^YCoAHiH zN8#-ElYZZ!fhXlajZjC{WY>KSr6#5a_BqfL{nKc88RdM7@$o5wrUUslpXTFs`Iktg zV927{TQE+zV1*2qZvr9j9^M3x-rIp1!i#wvcPmlSEPKT6|-Ia3~ zK*Nwnf3swuW#pUR#I={U1pn~suHxEPyEv%*`0RL0hTT|(SFy?JnwKtX4opNdX1hHw z_fWSGuiLHN;btsYdLWAN@h4k_59*8SVE6-M6Ud_x*i7kf!->9dYvtqa9*@F!FZ#Ce z*uGZn4A+&J5N=g^=_B(Cxs|PT*>61p!>Zh*FEx)rb1G_gNX=;J4?dR60 z<-ha@h;KdOPf$6-tA_5mrB_wWLsR$i%755-;c((oX|I{P? zn=wAazs2~9j}T5bN+!NQWcVgR{rLE1zEIFGxrmwklaYn+(A@vh9Q+ZgI`RL}8~#hp z;UCAm|KNegp>JryYOJqM!$NPsM#F4kVo1YIZ)ifpXkfr@z{){y#AamtuUc3}b{ZxY z-Tzo2{%;l3-{yJWW#NA=w!Vpahui-U^+2!E8wB<;4Dyps9FXXZP;JZP%go)%^z_m4 zTFEQZoGBX_W74+%nyILmI5RPEl}yfa8)u)TJ~dVrm=q@mn;cbba1GG@C5-6rZ)a-g zxhm01-C(+#WSV);G0oNvDTawb@DH2_`lCNp5!1QAE(`(h|EFCN7kX|@+h}Z(c^K1h zKN7ObznMDQW*lzz15lya4l&R8o2(HrK5#0+lm?^Vzj%fm+oQUe&$bUxT`Hd|oB-lo(6)50 zO=EKj;NBcwuamMp(4LjLSUtCg%2l!au+|>I==FOah2uJ%2npi*dgeHe^HOsE% zYCasE1)LXCdfi+ekfLD#7&Cm_T`ulbxwc$3xbQ;Rf>1Ae0IQ}_GjM>7 zH7gORr?;w%D-8nf+G>WWARWLFAc&0*h39LIEjrSZ*~hx)iP7rm`)f~Cox+iR z87dE&Ze_cj=4JBGbu$0aJz4c?xqpA315D_!*Hsf6EtGGkL~cJ{+T!71uJCj(X!kh1 zpYh2ys&SnQ-N!8&KW(nm(cAPEe7)Wom5h9uG2ZV2y4}a`U*IyfBh$AXpI~f}wO>wR zSt1-KY-G__Zp-?R90u2VvN;-FW6IhTWn*4FS8IA=dR)fxI(2D3E@yh{*oFbE%>3RIeIokvLbW99D4LRD)u09D)EVa7sc|B z03%RV^DsCfBRG~AtM|AHA!yG>-TyXXCv&cwRee5wbNqs+^j4TBD%mhS4Jc?KMvvmu z?mrHi$nfx#aoDzY``CFcwz;#JHO_JBS&x%ZU$E5lNO$4fJq6lr0NOoPMIG+wD!6ZO zLi&uaE~Zt$b0>EAU6Lm{c%mx8_uiXIv{#XKrTxA$JY{eNTXWHISo~w}Chw>H+}rqI z?Qq9-Ysa1H-c3@lDvj{lkY85{8nvrshwI3cE#PS!06>OwT7QplyOhnJ?$p-|d3@R1 z*d11Vty!ls;bnf?TFwF>gL}m@)}40`7*t?|SV6y@^VDV78MRukY227C(9vcnJm;@o z1$(T|RG;_7gpTL*Jl@U>;s_+%b3J$r75*rBjC?VRXmt@zA;o&dMs;Ds^KxuBU}P}+ zn?U#Ry!rdawkTtG`}0!2Zlj%{EoO?HSm$k1l&o5}Lw8e$+1bf#hxF#I2G8?3Z8Tu7 zLv;JIv%yskxx+d0qpVCytG?}usY;R;Wk^`n>uGNbd~g6frTV>X&V<1zSMUALjD!71 zu(9mo%g*t`G*;bLa@JnlFp=ivs+@)6f8vD2 z^*u6|E>=;`-5T4$rMsBMB@#RY=`p6}Zbqvx^p(dU^79FBA$yVIw*hc3s%WT;+OHfo zAF^avu7{r1D!Q;DYgLyovu(_}r)+|N6l+Q(%@EiVjGnz(SLWLtDnAaqPQs^Q+X8AA3XJe=y=RHb^-XpDFrwCr^82bhI)TZD$^H%O6#4@+p-FBc1kF%p-j> zbHDJ;f|~B0rY%oDPeM*U57}Qs(xuWY+K;x(bAI|fNv~?teVbEwZ0$Me%c7y{x>`Kk zwM=F5+AtRqvFyQo=$^9J_H%h`r#qh3_3UZjnAzXWk^NACZLNRG?^4ZzO=B;t zUMueD5(QKcetBL8lL?w+opu&%ZJPA^8rRgk+wQ>;=|*NhJRohET+9R5-IZrN=y1J+ zGjaejwmxfV_noWB7t1x%9ChMSEO6XRRuz}ciwSx9F)Z0_o_*n8k>{HgmM1G%vNPD% z35f*1%M88IlT{gCQwB=tEI9d2A1YkHS+mi1(lGzm36YIoD&MkXUX z{?50XYK{RMhh#tFXsSO=M9-RUQVP8liji`#JR)4L^9;X<&$=&ZsADK^HZ&9*bVi>n$>4yp3$V z2A$&1fR@)FF4W3Sax?u7Hs@gK?(_U6AEu86qq&bTrs}Mx5j2|Dr^&7LlUcq{QL5O-cT3q{wjRzMJBRIv2f%X*R`9B^KE*3&=>Fa$&{@|lSQ>= z_k92sHrlgwSFJdm)u+`Z*5v@z$cvH5rMugmQDtO!%gQYIT64Fk?lgFbn zNsu1^a?5Roo0M%oKeDT@(M5tf%gURRQamTT8r*^wo!gY|DW``1fuoBa53Qr@iJ{^R z=%2vt2mqc~?qzS7bjX|ja#^^)zskL{iV9<7Am+Q6^9Jtv9t)ykaRFcIHwQ>bF;_ch zBk35|X6_9H?J+s-J8!7d#HgJw@0lhx3Ftl^v9v8apX4dTGCbGoIey1723?7RbvfXF zE#*ftxbYp=n@wetVVJAx-(Ong*2fo(BWPOdyhFzx?<>L4s^ShSS80XEO|( zFSzE%na}Q9vPyIoHX4myHmTxOwT#yn?jLqL8UP4y(65gZHZhURUE5y)A~{VM=II-S z5BmX^SjO&OEB=dJ8PcR%X$S?ES#(QfF5uO8t1hvp`=_cpyz9^y)U7 zxxKZ=cXNV!MAq!JmMZy&_pz}_Xmjrlofg;K=F;Kmx4$>4(biU;(YHAyTc)a7pNF>5 zzWawj`X0X5FHdkeRyuJ!$_~nWqAnRc9)Aw`e!Nw_PSuvL=`D!z0NNgRi+@O8*0IM-Wv!uz?|-l(j)5I1y6ws5R#y3V(MNfkpzoc?;$ z?6JciflwS>;sVyQ=@n@H$)~}~V)q5K0Iu8fI)W|BIM$%BC?c_c??2=iFk?>z)-p4;pRixhV1PI^m)zQ#nK@YL>gdm*qCrE(F|j&f#v*D;Labh}Z`eRR zXJRNL^+&*}lZAklfP#x^%&A4%Ac~>O9dqA(m{{p$z!*ibo`b6-Wyva9hTeokWaRk3 zNR$kt!7<;kp+3Ieodk;}4Arvn59(Z<_?$f?hE088cZVHMX85ms$227&UY4_r?By$q z)2GqyAHgvW>hKzwrgf;ISWzM?<||p6;deGQY6FKcl%^LD+eKsHkG|)7;dZL&m7$Rp zRu>Rj--px|oa}`m$fZ?;qG%%m4pOCFNka6iHI;>==8c5m589Kk4a9m5MhA_Z?f4w? z)(omM?Y8pGUzA}ilpn%_J)1skRXJ&&Nd@~t8)#jLPG@RxX}-2Rbu%aYeeWA@@v0kM zi;})d<5{B1HPS0fhqO__qdU8MK1$A6QCKpW)#W*|ZLzv!J^>6w{q(}43@76yk0(Jq z?j!f<{Q9c3hVCK@Ktu7MUIC>7nG*5ztC=R5;o|nGdtMx9UN4^ccpv{ z-Q7ix|CpN|->1<|pYCr3?xTRaUOODWcg;<|P(>zn&p+VM<528CsB|n z6NFU8GQ6D)(F*(SvR{o-; zZ2a0<)iATCZTcdi`xV=u9ks=3*h6IDe?N;IZIsdO# z7Ca$dAF<_jt5(Lo9n{qA>8i^zwTFRyUDIr`KEp<(c6!mVs*y}W_)k{zE1E*ofJS|r z7oB$IID^UNE!%h>2}?52K$e;52I=Wq6l1Xw;|2)?L4J1^H*q<&(!>mmmW(?xjR~&m z!h0uuKhs0Ig4@Grj`?kWhshxoH{+MZfT^FtNWXAIaSiy2|Bg*3WvmxU#!?xJ$=?`J z?C#tnd-O+54htMmIEq4E$3%TZtN$bKasMc| z5ER(alQkY={oAl?K4Dqrw~VTc)0h~+q#Xb#PYtb!bT+L#!(%ZU zPa3Afzli0K+4A9UjW+(SyO8A}5hnw`TKS6bl-XIQ2+QKH!w}_iHmi9_%kKhg#9d64 zRvF2Z;$RXR+F>@Lp8@)F%KiM>i|t3#hNU_;s|j%Qm9ViY-;GC@2&|>u_BwEzFpO^JP@mrJF`?3>+Iiof2yBxe=7s7zCgkw_EB(DaS2SwF-1JF&guP{TQ8`C4~!4 zOe@k3Og6Fc84N;VOTrhnr_}FBaL_4)Jv~Ke5%2z3g@ipeVs3G|>(lC+8jEU+Da{cY zmS(B>pUvOqtTo9IuZ?!5>(eEA!osbJbXD*yWYhxV5FU1yp-WqT|8#XOB%KOW(ptO? zO=qp+M2->v@q2ec_jA&Nrg6pSgB5N*lQ zaXV_{8k@w={bv>j;H>RBcS7MH+AyiB)IF<4c1=bH!tXOk>(yk8UYZ;Ti?Xu9lm;nj z#S@)n>{&fCs|Yy4&K$K9)M&`;8@a=nw5MhA$JH}W0_&tF)~H3YDZi~ z4dp*jJ-U!_vfBO=hltgH;ai?<{bwOMn#lESru=+m^&8FgD|N?BW#WQDE`wM4yp;RR z=qBB7svc!tDUaP8)}w^%`D*D;282hG@~$aLyskSsm!|N{{VU=acUN?gNcL=4J0vwQQ+4 z7p5y8nQ>3wi&4Qf&?MfYPBb1cwIAPhC!vk;Dus zH><`(3a*e}@(QK#p41Qlvofy*SKI5Od}Sa-Ky4g!x9Y>-{lWJ>N`nX|XsNo@8l zx2@XBFnAI}^3 zH5Y;$xrerDvOj9Pd22)mNbB=0w;;;0^=D&SG=M10SCK5SijT1^HWALY2cm-n%`I6@ z--ZA;Q=(FC$b>$^suaZH74a|e4%)8^_k+8`k1TWop)+VcSY{!SAoQcBl=a}<7kxAi zgI3(5jfe>MM5AMkUz7Tp8=Dvu;LvehL>E)RqnB|Gk<=gBMPz*p9Y&oT8*P=MhxM7< zmfc0f^-rr@RL#x{oC(VfWGt-pb_U+v_oiXUwaC#Cooyb4MdX_6V(G?pN2z2-8-yMc z6`M<4#UDv*Wtc`G)5{70f2%E?SMWP3ty&i?ZIgCgAG*Y19N%~?IN?RDyr%ChSDRs9 z+^VG?~NwL&FWu=lk>yWUzRnjw3JT9hNDkr_N93wo7@{!yy)D5 znjJ6trNva96fEEx%WYEXte4{*txqsXls7u-b&_VZd0*|_){fp0 z9^DQ#HtR7dF(zOc-5%x;ssS8xPN3}Ly&v1~w1TqF!Azf=-_`BR-)j_cW0 zh7%rsTXiJh^0pk*l&wTRK|@0W@Z4?w zK|LloQdIOnDV6|3_mxIe6H6?sz7Sl5augXV`wvP6$cJHi1w7a0ho{#!*e%8#p8w+h z;o;i{42WMG?CllE_spzm^!4Q%0<=c2iwA@p0Rf`Z=KYvnx`ax}v}yY>awsMkW#QH%brW%7Fj^sy9GX)Z3ku>4yl1Z<6h)t}kKSh0`rOkI zmd3xhdd+i?QNN@$;|OU)F7jz0zi~;w0}A~R26AoA6b;N^D3FODeckhHe)wWCETV+} zRt4DsScI>YVdYY?O3w0p)^E}5PsDNPh#i3La!KW;8U4%Ow zK!AEmCQ&ir$iH!M!LSfJ+|S|Mhx5-(`l&FvCYUw3N@1mFK#5`B*-L8Mf1~PI7bqeF z?bx2Q|6{XA@AQ=oyRG>9&BC=#X<|b-BFm}M;zwd68558hX4kHcXyw0nYrc48r{`p+ z7OI5RP&*P4KM;_oB(v_$E@O}I0vuwFQ5yd*s2 zRzsoZL?`X*lQma8GP%9hwc&dnDn(x-I2~m??QPSR3qAS*?f_(jSB%Jkb*2vmAN`f7 z;o&X3z6NGJM+>mox8J|?_asnf?-f09{w6FT{>J}4!kwQAtPSHkjq%K*5iBv(N;SjK z4G$wG`R`0U4tUq30*h~h!crRP2RQRHqa8Qov>> zG{rx6%{P**Pjft2u;e>m1O!B5zWRS4^=5(oLFzUAkJhdNtg3D6Za_pDBo2*;gdp95 zq<}OiAuZkA(g-Mxpn^2gEl4OJAP7izcb9a>;r#n}gkyBp8#fUzT?l3eE zN1An!xv32pIa88kMq#mO@(4)J47w@4!nA$AUA6>5fZsW|Jd^^Uy$A*X4;+mJ*EmG9 z`dTT>BjS2`SBRaCcFuGMCuD%M^XXE{Ff$x1cnCJf8`Ng`xqtxy6#1GYh1X2g5X|c# zG2d3zZC(a#SqL0@$}3TZ%@Bmxyo6=*py2{_y7Xx@3JxG9_yHkHe0UxV@dc(GG~kMQ zDNG(+tY=~D60-eBJO!E>!JKCV;-u#x2jbA%tB=G-B?H3HeTr@(a5|H2ecz(eXanb+ zU1iuP%9b!xQ+=F|q4Cr|@LfT8%*nP!7s7t%W;gkK#d2T(ifa#65nNEutyWwZInOZH z9QWa?R|Tn>x~1tebp_Z*xGKoB`?EqOa9k-m%QwZGa@UyBh>$3%QW_?e)jn}vGs2pC z(j9-dyBVFHMCCT1rRbp#7FB9gGOmF1*AKBc)7Jt3CmGBG6@zOg49k|P=WP*1X(-so zDRxd20UhtHjCxhOure?JWhqT;YTRccxu$mwQCOYG z1emV3@EJ~SMJEw^dDSE06elTV{D{uzfKQa)RSnS<8PwBD&gK50r)P2WyPn?C(DVg8 zy-m2DUKyyTH}t!n9)e9 zxXk7;hd-qc+t2W~=#nLWG$)j;bM9XpqO5@pCZBziC?6MEV-A?V`-&Z8o+6d|)e;#L&1?jw zi81|^>cA~cNXWb{cu_SHfUtHq8!E8H;z)x8oJD&(TJFj{?!Mm2k-~kxoeyRc0EFe_ zpa+V?0_HMI&o-CsuTE1*XdnP7L;`P8oy^Vj;1?MCyTh$)}0+DO_9+ zO(Z5X*&v+)Af^@o^==VZY97C@KtFnJ0e|<@Y8pD8eMs;fx4}@q>iW>%Z;yZ%VPSTt z>f2I-!=?rgOW5uT3i1iwWu)yPvkD4nYmsUX=Anx<&@ShmQZiagn7?Ddk~Tn=_&U-s zEIgv4>$Ug|@(1^MoHbVG;D^o3$Yckfb*zhe2I_$HU zzx+}d%(11-&q=H=u=(uA#)+)VjlO>W(-htXOf-dx+1*uyV;ttFVTn| zfwB#_;ouP9zLrtWe(|Cn>BTj_;7#i&?%QTEMUfNphfvchIcFDV6BCnj4t`XzzQQnE zM1S1+{ra~;Z;G(wU!Gs^?Fg)I*+@YiBFzy@H& zYUjIsxeGFqETz>YuKBDU>LJfMM4h_wF`s^~lrYm}rnKLz?=9Dxs3DKz{vKuHRqd&l zDG#%Ku36@LtYD>;Q$`E3da-4n^4?;*Q*2Z^+D6D**efkIMX&3~QnH-PQ?JAN4mEt9 zjrV#@+g!SxZxcoi91M1sBz5EWJ$4%Sn?BpG7<)DsNkZ4=J8HXX7k6CZE#G>x>&?H5 zFAA+)>d%~|JC8LanS!=9@15UsFFSh7Z$1=B^JdIio@C4Y11956?{mRIazefpk&sV~ zF1Q_4>*H*0A;$ZKTnapRqn$(=@7+Ewe2oZjj}~JdtTp32-%Q-+h|#q>l(X@^p+lMA zJx$@+RjfVqxuR<2do1%|Z0$#<*Q^!>?{xDLHJLFfCRb~Z=VBz7+?LDIs3}{ur%pdR zwrN)x7k;+e@@$DH3_nV5nClXXR5@wi6+1+U81ohz>Y_MV`^G`XLx^mpJpr9E5cFQ^ z!hBBa98^0yC^Y5)?V;`%obqO0HC+uJ+q90R96C&eCERmrZ18@Lk$AW?B!ig@t=@N< zXfJdxaxX|}7DF0hZWCmu(bM0}K%zAXa_|;%Fz%RgF`Tk{x@Xeh1+(bYW}$37E||bJ zfi{N)Q|{*`JAOj+zm^(d`yHA_C;IS8ZE(Qea|{cxL}_lkmt@6}L|p#&^|APo;K%4x zQb+nDPz4k-1HH@wj;fw{Tpb?Jzxb)Bqy^7TrDc#OFk_A%n=KHV$HZ#=Y;v)I>%1w|Vd!uapq zHa>i%Ypw0MT-wC4<}3ef&BK++-&F6g#H|*kicEXapO{3hbMk?MBDqT^(-$Tsv$qy> zdyqxSc(v7~UO~e+I@wr9kCFvL)wE%q2wiWr^u@iRml5UmrD&A-qgx73$_na*Uat9Z zdT*?_WCt-7yXvLben0*obXs-m;8{PB(~;qpPc{qQ-bt_EY5f`J9EH1~-CAXZ2*N`h z3_>sH{E+u2T3V~_+BrJyt0NEBoNkOOJgQy@n!35#H+yf9&SRGD>}0lY`}L&G=%=_A z*n=2Vyc|qH5_S8=?lk@W5QD`I5_e@w7ncZP*w|T)E9^6IF=fY*yZ{>~3$r;4mo0xg zX~^2*v@G^+(L@}Shm7#l$(y=pzJq+WbenP0YkP92QB2KaJKk;v_I3EMoTYvAvs?lw zAatIi>5XH6eWd%EbBn(imicOy@sN@bX47wF5e(TRWL z^X!a8fdbz^<{->$?3lE;p{aOe*M#!=bAjWTmYFr|7yM{V1*iv83|NWV^I!b7GiC>q zP1e_HPD!_YHwtVhYv+BAb|9ZAY}EIPhjY795pY{-Up&XIIoT)*tHF78f_f4mGlc_l z*?QS#4py_STngp2ZLr=poqO5gV$DxqsDXE$Y?8Hsx9M%{414Z<{2;+@<)hGtgKFBV z8kwD+W?PcKGV`qRT*>shRm18zkNv#aYe5xV4#tnf_YRzO*+dKwfg zaVnr}tF>iW4d5l=mk$8N&`=%a>|iXu*8$OQ@iK6HB!_lR9+#!UU?$f)4R|iUg=## zkc?)=*5b8=URr;*;JsH%jd%y6(1~vxY&hcFH!^;cRSVjgi0;;*AxC z`<+b!yY|&A>o|^zqa@v!GWV7FQJQF7N|PST^OKprsd4+W$6D!} zj4G%FsEer0iiBn4WYXF0({Jd^cP&PU(3)QWkthZ<)ysY$k5?08-@3l7IT&ELId*JP!z zH-2X_estl3l>S~z&g}9>mq!hSQqts+8+7un9D+`>Gfn!2^(`-vtW;)cM1$YYg;&5# zXG!!7rOTj^9nY>}U+dnxt?gp)AW`$xExi}4u>K+42TW=o99Fe+#Bc9+b(iyX^HLXd z_|}Dbm!bB5SMXr(*4dVj?r6x5p=8M6F3wL28T8_H8%uEC>^8TD-60t+sTL+1LOY~{ zI(nu_@Aj8FNiU{)deK7n>s_Y6N;%9=m-4HyI?p`2T>>l6HnjRZ4qkn-AxA&cpU6vw zB|6V`e%)pql!dMAjaFnPY?RMh&ezv4ja9qvCUt3+9=VSPZgo(~dK($63Ch$;xmEQ* zWU%)>S-3w9Q6T+1U-Zdnv!>u6ulgaQO#ORPwa9e$IhPen58=EEzDijI5 zA;EKyDCFEUB|779$Q5F=T5TYr4GCU7->_)SAGjuzb z;yz3}?8$A6xbk|zGi9wXQ^o{3-2U=@%bEg{ZCKU<6RjL-x8c%6nuGLg zggQ&&evx2M$3w6n>UveJI@6|OEvk#P`)vP9ufT!K57_7&8bk-=Yqg)v?6JjMlM#4g z4-PqGJq6Z0p$WIdw)WZF4=AnlTuo;85t^kXNOZgm^p@sT)QF|ytL@h(+bbHAptVy2 zGhbc?W~^;YQV^1v4O3m29L;D}3H0VpV)a}WYER+g(BIRRNrJ$_SBFkN<((37w}0<3 z3pI*NT}#4dpTd7mQph-3qhHA69fulJzhrV?V=*5mq>e{>MVnJRPyJwzTic+y<6tYK zS;$69?zk(`y1X;d8EO`>s=I~fe1w_|<=yNTHyU|FrB;T{$um+SX#icBa@bE!&5Ork z^%&k?r`nC%i+RV*TL0Zo?!a{}gofL^TkCA~dvaIwLH$sGB&7pRi)`VEFEGM$(!D(e)$`_nGmT zA!4ldnu6vaf9t?_4tw+^FQL}%#l4L>NM$JN9kUwSyDR$4{xMGjv@aw}BVM?*yrsB~B7?sG*eJofgI0#V3YeSOofO ze5Xr9DnXs0&b7g?^dY=NZp*9)nQt!?*qbF*qNDbewzpfhm5A9ULX|Qp$Ekz$1OGNE%`+r?ExL&n7R|ygf zjvCAe#7fvW7<()^C1{p8D#aU3jEp2`RyaPq)8}B1V!s}}pJ`3&@XC=;3Dr(%;(D)k zXZm^7@`fX?5#;_{Y4!^b^L`6kjZKl>&6l4~bofVC!lvuVqCc@p7Y*?PQ}Ziac==D-wxgC5Fs~ z&qjriTPnl|H)2=Sew+##C5Dw1gXB)S8%Aqz?6*rOm#@Z^V${e;vbP@&ThptZ4)9E( z@a#mkxdk2B=3d)ev?ApDJj27=anf*{F=;kPNe~?UIrj}=o4I>d1ymAi)8KTaT2394 zj_Hkx&eE7f-b+|TEe`HoNyhm=^ZXFHvU>Fb&tw1E_jKJgXI^+zZ8ffo?{*q1;$m2s z&1N3EO=T>w6k)JT?Vq)^LL%B<4p-LY#hmiefF&1d)yZ%G?=_KHY&0bfc&USE<=659lIQVQC7M_ zHd%S5ltev`Y)K8#N=0@=hH591crl3Yc;%CXOk*wjj+HkL_fkCB=ID9U)Nb^L##hwj zj$WC3+M_qSEZHE0Bb`gLH*Y8J$ce9r^X*hEN+G`;A)$I8=$EsY5Hvz~Ix|nY&{w>k zZ)R+#m&!L4WzDY9z8;U&$<&q*S;BLtcQPK8%CA(f+=*pfEd9oA6!TD1s|TovBDheKfc`eJZ0>iL@Wsn zFoqKikiy)M8;=jDu}jDwEjl*0%H8Ru`SylAJ;k>2mF@I2zGi(`Xh1;I3KvK;uw%C1 z6JMcqD14PiD`#rxdjCi_W^K`w+COs!)mH>z<~5-?)eTN|$1v`_7pfX65d4?fE7K&e zl&@Er)>nP|bn~cXCh|-M@2O=v4$0HZh{)8Q#L}!>RJ5Wq)nZ!BUf#WYRSh0->C&B) z%wVDim08Gq+I$0Vhrq&sQwIcST$Om|;mVAkVEJa9R@W=(6g3{ne~`d*?|64*iJIYM zgsF_sj7ovTp%HCbmT&w_~2FhUTHs1AmA1>1wjv6|fT5Ypu}NLsF2rqC9^u zl8Lt=WX;nWIgg`bNM3V!^#)gF#Y_0_7un!Im=dSw^*?0H9NbE*M2yIdvYfhOa$0#V z8k8(?sJeTNsxE0We04N#>P@#xS?_6OuQ6`;xBlW=iPkhls!`*De4l!KMRO%8(gV`A zVtB5~i+f3-^AH{j&EM2z|E4j2sJ%*W-I04|^^|!4iaW|G4*jC^kk*po4rKrJOn50d zQvwsDG_|U%7p+oG&z{C4q$oA<@_>eBsUhi`FeO_mu;{Mt+OxEDi_dN^v=+%F=z~&^OoLI)o*Bds7~Mzf}}H>sEQGoSmaG*=GnJ>X2Gu z^y6D}#Vv7PX(v8ODf5QNRpLvUH1{AyMcaA~@C6arU*~w+P= zDWXr5$1H=#scV(G^{KB>Z3Z-HSK7#_foLUsm_znK|FX1!m71{pJJESlxn&irWxayZ z@C{p8{{l_3HG@{k&7-`MU{QI!FJ(JY--8{rkh5_5D`8Atsa+>h7U{fqGs7oy6OO?+ znhcy8q~~k66eJvb)7-z-(aJ6~_VO5|EY;;iYO{5qy5O0`=H}f^llban;)VrtE48SE zg>@vP`e=rxy6}%fwfU&WLMy$}q|2SoA(2%rJFs51u^5JyR}J==)ofbbW{xxyUGcKi z?vdJW6s24yPUf$D+GT>V&>3px-b&?fmPTDKq2SD^EmHZB z3bRmLQMxeeH2UQGr^0fsQ4X7RgVeTysS{qQAD|fnkH{L(!i%pS`)lbs38seappp|& zC4(G?1;{t9mS8%Ucn==t&4RJsyhXYSNx@GhlC5-yrj#o{XIK}xC!xjlK*i`%9qCwA zYAD_M=*igeH-&GfW9$!KywZ4=Cra%klG=RwK>O3y`1_D`J@ryMX+04~CF63wE}{dK zJk6U6L^n#@(+lWuMra;lxo6ZeI-aewK;+ns?@op=d$TY$Cs;59e;3?%Q&PaVA|tgn z5=8=etD6wd2vxt;c;gt4y4fX&lfY1#REp(oyL!{CmW-i`+iT4#sAx` ztx$lv6`P*XByJ=p$>sb*@5$#`99zfZjSze5H-mDvd^`@u@<~d9-BJ%H;vU^vvN|OC zTv}5aUMi|zwzN)$sN#v{Ch>{nLzB4p>+ytvj2q3jR5p*sCAw>zF?Tb2-*gkc?qBF+ zKxa1BL055Kl<$!lAbx^rG0#7!-^rLOa#dwKCwM2PGQl3Rh9QR>4b8rz>8$mj0M&x8 zOtH1Dez_Xa?9|k)NDsk&I0Ln6yk;$Q+|N(olUa=FhjQzOh6e1<1NMkHZmq8(eC{ji zv#4igq)#BSdR4-q*;=drNWRpGdW>5S{Y}LNLwwwuH-n5dY8p0=cDjSi%!1s^N-PS5 zKFkwD&C@uHzArt?$K5uA-(H_)q!HF>C2Yj(w>n9J@K@+t00l~5I#?$p zzXE(Q5_5MCLAdzEjcG!Gr7iUSe>Ro|?tKUtFz$F{RZ6ybVnIKX{rmJ`WJ%kH?^*JjJ zxbmTjh_w_BHLL-L<>7Fhhy1mOVIh?eY_V(RQIa zE-=m4pViv5{{{KW`+JI{${m1WIEtn%B-0q7wLTvn&6KM|^awuaes1~TidK`)-SRPt z*f!O!TW*o>9Q`A+`E7_dw@1|L1>I-20BO>3{|9)0a*O8O>$&$A;VUti;Vi1)@<@Pb z9Vrg>ci1N7S8ORf#FOmZj?47oKz@Oaj!4dRB^H1luM2;=Y@7h+x+x-1P5SH|0O&S! zgG3MEXF1SA$E^gk$)aEs8Ln#WYPo1&stmbV|sdz$fiLFb2 zRY6E_o4bES+@Wu}x)LS*xH*ME(j9qI#4J!>k%-_$2%>OU{{#|}2kM+oHip9wg257; zU~ol@7pR7CkyT4U-?6iBhG)BWa*WN!Oo2yFz;^Lx2Q-b1=v|0W_piL2H zF=-&;5Mtb@tSqpPfZuRxA-E~R2wyzQzM33>B;cNK& z9AhcfsNw?xb(pKX zI?kls&aOD0jX>Y_m0`mu+0RZ6OFlX%2QQ&wHdbBLS#7qWM(O8BYOsy$(kCKxs(a@A zk)Vh1RU7mkBEw^vM=BAK@6s(XCr;ndVe(sh);-T=V9kL{@xeFR0{iti#9BGlrJ?#5 z{UgYU2bq(W2Os zY*YnDAM~Iv9{VXK7+`ZOYHRAlvSX^WnkVI?JbgEWO{t5_6N zU`ecq3{&ClGq9`zph&n#41C-oxp9!S;}?_q04b^2^F<}dF`zXH=c{+$%D-ZZ&R_#j zEU;V1W?+Zr{mL=0y5h5Iu@z3Df(_6|j5cW0?GlY$^&xpzzh}lkPI5Kn;htj;KrdlR zbVWp%yxYx@GJ?ydG;a&pqwcEp^q{`@1RO266S<~{q9#Z|6`Xo^=al3d>{?{%*}C3Q zUjs~e1^Em@6f*^5Q!#+8>w5(VN37VMPb&0IlaunAC&nWd=lW|J)&m}Mk-H`>sdPrq zFmSz$`N1(@v;2)?5Dan*SiiwJ2Bg1m47zT6#+%cE90T(J=;QCC9x2LW?SW7uL&Lr2l_+rV$=nQd4N+y{}(!1b+&G- zIyE*TaQyk?_Glj93PR2L@6J%?XOkuqne5Xi)gtBI|B$PSNg~RSHz(qbn4vhBh#)Ly zK6q*wOBnG43yfDvG01zHl;RRwk$qLfH5K;ct>8j13c;4(D*Zr>CLWSbuv^-OL>X$G zXLf@pN&z?r=!lS~dEYof1nTl|qY!}c`n9J~rLi^bYr!1j5?s+BeJ@E=5q#gcZz}q0 z+Zx6F`0dWDEhU0huxAZI!B~*wm5=rLy3Ny-Ch?>GLz;{>62^Wa&fjS*?l;yMg}+0T9)PGtf(eU)Vd0Q_gh z#FO$&_bkD;r06_#KLN~xgL-O0EU*vfR=5;}bd(=GBCl&u#!Eob@qIf2R3|_JR%)K{ zq1i$hz<@L8-yq87)0YBjd=)_H+HSy9@iU6H2;j3hi-4$TvLKQ^BqGap{RVi}>DdPDqN0PS!E)yhG2KF=d!UF(^+N6N#H_7#`M3N3)hT3*XMe zu3t%u0Fv&8;08SIIAG6kHIqWbwa8_e|(q{F{;#J_R%oLqgP7F1bDA;NNwa_ z$USIne;_^RkC|Y2Cqc`y{Qd$#`CEPFJFU0B+2G-k<3Y&#GQ}k9OhSs|mXk6F;tBYxD1(5A5Bwq}%>+tGUwjP?^zs3+Be+;tIat`ZS^gaM&rK4V z`Ff>-Q5glJ^5c~7dz)F{qYF*)8gTI%>T~ijaT#+MGI1KRvoi5<^B6JlaI+eMB^nIb z*p1- zK&mBh^Y@g3KS=lamz?&+avi_;^P6Da&+WlK2~Jl(4u)U<{_>~O@87rg3p@iGD=Ra{ ze`z)T{dVb}{rh%*!TF!sosirtO#%Ifb6trq;q|-F?jO{{e}-rJ9gA-cY~l(50Nl9L zj9I{chWB%8S6ZH{f#0QoC*bepu^t!nuhy9LoLpI$9rUbhEsb;C4<{RVRuJw6NMwQIq9064?@R931^*8KSb2;&c?^vWnZN}Z113&AaIomJbFeb; zaImrI@#%4}bAzwJE7h;2_NbZ#BqYLUH*2j^gJey%+(IIe+0e?*9Laz@N>^*LL5SB!lC5 z@(&s6*7HxTU*LBBtqg?=p#Gx_{qxzcar=J&`}Z!j;Nw|*hf{bM%#&#_Pi|c3DX?TX zxH;svq`#DZKSYOr$-kd_o-n;yT^1Z{G2r*h%V*g7gTHF+&-xM$=(n*Xd2)N|DP?P7@KvJM$5s6P8|HIjcupIVv-xewkEhy|g8H)}2&oquKRkHNWM?PnWW zk8k*$!uJYM{cOV-K=?i0|IDMGZ5;F6#|6KK0dvUU)!qd!0AK*+FKk@&rGb*gsdP{yCHvMHECY8QiD*H{ShX@E=Gc{%r4}5CUA9;a|rP%b(l()0+Z6H-E7P z<7IcAvHp4Ue?#f=*eNz@OheaPhqa;IFO|kioMQ0Ne(D7Qye!6n@|w5Bwh?LM-tB literal 0 HcmV?d00001 From e3e003b6d1dd008c7afd2cfe04c8f34adad6ef74 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Fri, 5 Jun 2026 12:20:03 +0800 Subject: [PATCH 050/255] =?UTF-8?q?fix(flink):=20Use=20the=20execution=20m?= =?UTF-8?q?ode=20without=20rocksdb=20cache=20by=20default=20=E2=80=A6=20(#?= =?UTF-8?q?18894)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 6f81c51fae1d964989edfd634c69720b4569a5e6) --- .../HoodieBackedTableMetadataWriter.java | 2 +- .../hudi/configuration/OptionsResolver.java | 41 ++++++++++ .../partitioner/BucketAssignFunction.java | 8 +- .../GlobalRecordIndexPartitioner.java | 7 ++ .../MinibatchBucketAssignFunction.java | 8 +- .../partitioner/RecordIndexPartitioner.java | 27 +----- .../index/GlobalRecordLevelIndexBackend.java | 45 +++++++--- .../org/apache/hudi/sink/utils/Pipelines.java | 6 +- .../apache/hudi/table/HoodieTableFactory.java | 4 - .../apache/hudi/util/FlinkWriteClients.java | 4 +- .../configuration/TestOptionsResolver.java | 34 ++++++++ .../TestMinibatchBucketAssignFunction.java | 82 ++++++++++++++++++- .../TestRecordIndexPartitioner.java | 1 + .../TestGlobalRecordLevelIndexBackend.java | 4 +- .../hudi/table/ITTestHoodieDataSource.java | 20 ++++- .../hudi/table/TestHoodieTableFactory.java | 17 ++++ 16 files changed, 250 insertions(+), 60 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 85701fcb715fd..546ce10bdfbed 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -172,7 +172,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTab // Average size of a record saved within the record index. // Record index has a fixed size schema. This has been calculated based on experiments with default settings // for block size (1MB), compression (GZ) and disabling the hudi metadata fields. - private static final int RECORD_INDEX_AVERAGE_RECORD_SIZE = 48; + public static final int RECORD_INDEX_AVERAGE_RECORD_SIZE = 48; private transient BaseHoodieWriteClient writeClient; protected HoodieWriteConfig metadataWriteConfig; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java index cc32351e62cfc..42cdad7032c02 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java @@ -42,6 +42,8 @@ import org.apache.hudi.index.bucket.partition.PartitionBucketIndexUtils; import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; import org.apache.hudi.sink.buffer.BufferMemoryType; import org.apache.hudi.sink.overwrite.PartitionOverwriteMode; import org.apache.hudi.table.format.FilePathUtils; @@ -61,12 +63,22 @@ import java.util.Map; import static org.apache.hudi.common.config.HoodieCommonConfig.INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT; +import static org.apache.hudi.common.config.HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; +import static org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.RECORD_INDEX_AVERAGE_RECORD_SIZE; /** * Tool helping to resolve the flink options {@link FlinkOptions}. */ public class OptionsResolver { + // Value to override the default minimum file group count for global record level index. + public static String GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_DEFAULT = "8"; + /** * Returns whether the current runtime mode is adaptive batch execution. */ @@ -227,6 +239,35 @@ public static boolean isGlobalRecordLevelIndex(Configuration conf) { return indexType == HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX; } + /** + * Estimates the file group count to use for RLI partition of a new table. + */ + public static int estimateFileGroupCountForRLI(Configuration conf) { + int minFileGroupCount; + int maxFileGroupCount; + if (OptionsResolver.isRecordLevelIndex(conf)) { + minFileGroupCount = Integer.parseInt(conf.getString(RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), + RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.defaultValue() + "")); + maxFileGroupCount = Integer.parseInt(conf.getString(RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), + RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.defaultValue() + "")); + } else { + minFileGroupCount = Integer.parseInt(conf.getString(GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), + GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_DEFAULT)); + maxFileGroupCount = Integer.parseInt(conf.getString(GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), + GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.defaultValue() + "")); + } + return HoodieTableMetadataUtil.estimateFileGroupCount( + MetadataPartitionType.RECORD_INDEX, + () -> 0L, + RECORD_INDEX_AVERAGE_RECORD_SIZE, + minFileGroupCount, + maxFileGroupCount, + Float.parseFloat(conf.getString(RECORD_INDEX_GROWTH_FACTOR_PROP.key(), + RECORD_INDEX_GROWTH_FACTOR_PROP.defaultValue() + "")), + Long.parseLong(conf.getString(RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), + RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.defaultValue() + ""))); + } + /** * Returns whether it is a MERGE_ON_READ table, and updates by bucket index. */ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java index 7faff523e6897..d2deba8ef0303 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java @@ -172,9 +172,7 @@ protected void processIndexRecord( protected void processChangingRecord( HoodieFlinkInternalRow record, String recordKey, - Collector out, - HoodieRecordGlobalLocation prefetchedOldLoc, - boolean prefetched) throws Exception { + Collector out) throws Exception { // 1. put the record into the BucketAssigner; // 2. look up the state for location, if the record has a location, just send it out; // 3. if it is an INSERT, decide the location using the BucketAssigner then send it out. @@ -183,7 +181,7 @@ protected void processChangingRecord( // Only changing records need looking up the index for the location, // append only records are always recognized as INSERT. // Structured as Tuple(partition, fileId, instantTime). - HoodieRecordGlobalLocation oldLoc = prefetched ? prefetchedOldLoc : indexBackend.get(recordKey); + HoodieRecordGlobalLocation oldLoc = indexBackend.get(recordKey); if (oldLoc != null) { // Set up the instant time as "U" to mark the bucket as an update bucket. String partitionFromState = oldLoc.getPartitionPath(); @@ -238,7 +236,7 @@ protected void processInsertRecord( */ private Processor initRecordProcessor() { if (isChangingRecords) { - return (value, out) -> processChangingRecord(value, value.getRecordKey(), out, null, false); + return (value, out) -> processChangingRecord(value, value.getRecordKey(), out); } else { return this::processInsertRecord; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/GlobalRecordIndexPartitioner.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/GlobalRecordIndexPartitioner.java index 8b6f1ae22ad9e..258c3f4a8fd34 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/GlobalRecordIndexPartitioner.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/GlobalRecordIndexPartitioner.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.configuration.OptionsResolver; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; @@ -79,6 +80,12 @@ public int partition(HoodieKey recordKey, int numPartitions) { */ private int getNumFileGroupsForRecordIndexPartition() { HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf); + // For flink adaptive batch execution, writer coordinator is not started yet, so metadata table + // is not initialized for a new table. + if (!metaClient.getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX)) { + // estimate the minimum file group count used to initialize global record level index + return OptionsResolver.estimateFileGroupCountForRLI(conf); + } try (HoodieTableMetadata metadataTable = metaClient.getTableFormat().getMetadataFactory().create( HoodieFlinkEngineContext.DEFAULT, metaClient.getStorage(), diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/MinibatchBucketAssignFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/MinibatchBucketAssignFunction.java index cdf30c932eb75..3a44267c42a9a 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/MinibatchBucketAssignFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/MinibatchBucketAssignFunction.java @@ -20,7 +20,6 @@ import org.apache.hudi.adapter.ProcessFunctionAdapter; import org.apache.hudi.client.model.HoodieFlinkInternalRow; -import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.configuration.FlinkOptions; @@ -40,7 +39,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; /** @@ -152,11 +150,11 @@ private Processor initRecordProcessor() { public void process(List records, Collector out) throws Exception { List recordKeys = records.stream().map(HoodieFlinkInternalRow::getRecordKey).collect(Collectors.toList()); MinibatchIndexBackend minibatchIndexBackend = (MinibatchIndexBackend) delegateFunction.getIndexBackend(); - // get record locations by minibatch - Map recordLocations = minibatchIndexBackend.get(recordKeys); + // warm up the in-memory cache for record level index + minibatchIndexBackend.get(recordKeys); for (HoodieFlinkInternalRow record: records) { String recordKey = record.getRecordKey(); - delegateFunction.processChangingRecord(record, recordKey, out, recordLocations.get(recordKey), true); + delegateFunction.processChangingRecord(record, recordKey, out); } } }; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/RecordIndexPartitioner.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/RecordIndexPartitioner.java index a4597e00f099d..e1d2a5f203aeb 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/RecordIndexPartitioner.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/RecordIndexPartitioner.java @@ -19,12 +19,12 @@ package org.apache.hudi.sink.partitioner; import org.apache.hudi.client.common.HoodieFlinkEngineContext; -import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Functions; import org.apache.hudi.common.util.hash.BucketIndexUtil; import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.configuration.OptionsResolver; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; @@ -34,7 +34,6 @@ import org.apache.flink.api.common.functions.Partitioner; import org.apache.flink.configuration.Configuration; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -77,7 +76,7 @@ public int partition(HoodieKey recordKey, int numPartitions) { if (partitionedRLIFileGroupCounts == null) { partitionedRLIFileGroupCounts = getPartitionedRLIFileGroupCounts(); } - int fileGroupCount = getFileGroupCountForPartitionedRLI(recordKey.getPartitionPath()); + int fileGroupCount = partitionedRLIFileGroupCounts.computeIfAbsent(recordKey.getPartitionPath(), s -> OptionsResolver.estimateFileGroupCountForRLI(conf)); int fgIndex = HoodieTableMetadataUtil.mapRecordKeyToFileGroupIndex(recordKey.getRecordKey(), fileGroupCount); if (partitionIndexFunc == null) { partitionIndexFunc = BucketIndexUtil.getPartitionIndexFunc(numPartitions); @@ -91,7 +90,7 @@ public int partition(HoodieKey recordKey, int numPartitions) { private Map getPartitionedRLIFileGroupCounts() { HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf); if (!metaClient.getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX)) { - return Collections.emptyMap(); + return new HashMap<>(); } try (HoodieTableMetadata metadataTable = metaClient.getTableFormat().getMetadataFactory().create( HoodieFlinkEngineContext.DEFAULT, @@ -106,24 +105,4 @@ private Map getPartitionedRLIFileGroupCounts() { throw new HoodieException("Failed to get file group counts for partitioned record index.", e); } } - - /** - * Get the partitioned record index file group count for the given data partition. - */ - private int getFileGroupCountForPartitionedRLI(String partitionPath) { - int fileGroupCount = partitionedRLIFileGroupCounts.getOrDefault(partitionPath, 0); - // HoodieBackedTableMetadataWriter initializes record-index file groups for a newly seen - // data partition with RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP, so the writer-side - // partitioner should use the same count before that partition appears in the MDT view. - return fileGroupCount > 0 ? fileGroupCount : getMinFileGroupCountForPartitionedRLI(); - } - - /** - * Get the minimum file group count used to initialize newly seen partitioned record index partitions. - */ - private int getMinFileGroupCountForPartitionedRLI() { - return Integer.parseInt(conf.getString( - HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), - HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.defaultValue().toString())); - } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/GlobalRecordLevelIndexBackend.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/GlobalRecordLevelIndexBackend.java index a95e18e863a13..93a6303009f88 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/GlobalRecordLevelIndexBackend.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/GlobalRecordLevelIndexBackend.java @@ -20,6 +20,7 @@ import org.apache.hudi.client.common.HoodieFlinkEngineContext; import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.data.HoodieListPairData; import org.apache.hudi.common.data.HoodiePairData; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -27,7 +28,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; import org.apache.hudi.sink.event.Correspondent; import org.apache.hudi.util.StreamerUtil; @@ -37,6 +38,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,7 +57,7 @@ public class GlobalRecordLevelIndexBackend implements MinibatchIndexBackend { private final RecordIndexCache recordIndexCache; private final Configuration conf; private final HoodieTableMetaClient metaClient; - private HoodieTableMetadata metadataTable; + private HoodieBackedTableMetadata tableMetadata; /** * Creates a global RLI backend with a checkpoint-aware cache. @@ -72,7 +74,9 @@ public GlobalRecordLevelIndexBackend(Configuration conf, long initCheckpointId) @Override public HoodieRecordGlobalLocation get(String recordKey) throws IOException { - throw new UnsupportedOperationException(this.getClass().getSimpleName() + " doesn't support lookup with a single key."); + // note: always fetch record location from the cache, since this backend is only used for minibatch mode, + // and the cache has been warmed up by calling `get(List recordKeys)` previously. + return recordIndexCache.get(recordKey); } @Override @@ -93,8 +97,7 @@ public Map get(List recordKeys) thro } } if (!missedKeys.isEmpty()) { - HoodiePairData recordIndexData = - metadataTable.readRecordIndexLocationsWithKeys(HoodieListData.eager(missedKeys)); + HoodiePairData recordIndexData = lookupLocationsForMissedKeys(missedKeys); recordIndexData.forEach(keyAndLocation -> { recordIndexCache.update(keyAndLocation.getKey(), keyAndLocation.getValue()); keysAndLocations.put(keyAndLocation.getKey(), keyAndLocation.getValue()); @@ -103,6 +106,15 @@ public Map get(List recordKeys) thro return keysAndLocations; } + private HoodiePairData lookupLocationsForMissedKeys(List missedKeys) { + // For flink adaptive batch execution, writer coordinator is not started yet, so metadata table + // is not initialized for a new table. + if (!tableMetadata.enabled()) { + return HoodieListPairData.eager(Collections.emptyList()); + } + return tableMetadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(missedKeys)); + } + @Override public void update(List> recordKeysAndLocations) throws IOException { recordKeysAndLocations.forEach(keyAndLocation -> recordIndexCache.update(keyAndLocation.getKey(), keyAndLocation.getValue())); @@ -141,21 +153,30 @@ public void onCheckpointComplete(Correspondent correspondent, long completedChec } private void reloadMetadataTable() { - this.metadataTable = metaClient.getTableFormat().getMetadataFactory().create( - HoodieFlinkEngineContext.DEFAULT, - metaClient.getStorage(), - StreamerUtil.metadataConfig(conf), - conf.get(FlinkOptions.PATH)); + if (this.tableMetadata != null) { + this.tableMetadata.close(); + } + this.tableMetadata = + new HoodieBackedTableMetadata( + HoodieFlinkEngineContext.DEFAULT, + metaClient.getStorage(), + StreamerUtil.metadataConfig(conf), + conf.get(FlinkOptions.PATH)); + if (!tableMetadata.enabled()) { + if (metaClient.getTableConfig().isMetadataTableAvailable()) { + throw new RuntimeException("Can not initialize the table metadata"); + } + } } @Override public void close() throws IOException { this.recordIndexCache.close(); - if (this.metadataTable == null) { + if (this.tableMetadata == null) { return; } try { - this.metadataTable.close(); + this.tableMetadata.close(); } catch (Exception e) { throw new HoodieException("Exception happened during close metadata table.", e); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java index b17be182b9ded..036f724b00d56 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java @@ -291,13 +291,13 @@ private static DataStream streamBootstrap( boolean bounded) { DataStream dataStream1 = rowDataToHoodieRecord(conf, rowType, dataStream); - if (conf.get(FlinkOptions.INDEX_BOOTSTRAP_ENABLED) || bounded) { - boolean isRliBootstrap = OptionsResolver.isGlobalRecordLevelIndex(conf); + boolean isGlobalRLI = OptionsResolver.isGlobalRecordLevelIndex(conf); + if (conf.get(FlinkOptions.INDEX_BOOTSTRAP_ENABLED) || (bounded && !isGlobalRLI)) { dataStream1 = dataStream1 .transform( "index_bootstrap", new HoodieFlinkInternalRowTypeInfo(rowType), - isRliBootstrap ? new RLIBootstrapOperator(conf) : new BootstrapOperator(conf)) + isGlobalRLI ? new RLIBootstrapOperator(conf) : new BootstrapOperator(conf)) .setParallelism(conf.getOptional(FlinkOptions.INDEX_BOOTSTRAP_TASKS).orElse(dataStream1.getParallelism())) .uid(opUID("index_bootstrap", conf)); ((OneInputTransformation) dataStream1.getTransformation()).setChainingStrategy(ChainingStrategy.ALWAYS); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java index ae8dbb1e963b8..43d18e849b3d5 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java @@ -475,10 +475,6 @@ private static void setupWriteOptions(Configuration conf) { conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key(), "true"); conf.set(FlinkOptions.INDEX_GLOBAL_ENABLED, true); conf.setString(HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key(), "true"); - // set index bootstrap as true if not specified by user explicitly. - if (!conf.contains(FlinkOptions.INDEX_BOOTSTRAP_ENABLED)) { - conf.set(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true); - } // generally size of index data is much smaller than data record, so set the buffer size of // the index writer as 1/4 of that for data writer if it's not set by user explicitly. if (!conf.contains(FlinkOptions.INDEX_RLI_WRITE_BUFFER_SIZE)) { diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java index a389d721908bd..0293aad632dfb 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.util.Locale; +import static org.apache.hudi.configuration.OptionsResolver.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_DEFAULT; import static org.apache.hudi.util.StreamerUtil.flinkConf2TypedProperties; import static org.apache.hudi.util.StreamerUtil.getLockConfig; import static org.apache.hudi.util.StreamerUtil.getPayloadConfig; @@ -235,7 +236,8 @@ public static HoodieWriteConfig getHoodieClientConfig( .withEngineType(EngineType.FLINK) // this affects the default value inference .enable(conf.get(FlinkOptions.METADATA_ENABLED)) .withRecordIndexFileGroupCount( - Integer.parseInt(conf.getString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), "8")), + Integer.parseInt(conf.getString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), + GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_DEFAULT)), Integer.parseInt(conf.getString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.defaultValue() + ""))) .withMaxNumDeltaCommitsBeforeCompaction(conf.get(FlinkOptions.METADATA_COMPACTION_DELTA_COMMITS)) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java index 0ed8e61fd3500..a70e7dc9f7df5 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java @@ -18,6 +18,7 @@ package org.apache.hudi.configuration; +import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteConcurrencyMode; @@ -25,6 +26,7 @@ import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.utils.TestConfigurations; import org.apache.flink.configuration.Configuration; import org.junit.jupiter.api.Test; @@ -191,4 +193,36 @@ void testTableServicesGateClustering() { assertFalse(OptionsResolver.needsAsyncClustering(conf)); assertFalse(OptionsResolver.needsScheduleClustering(conf)); } + + @Test + void testEstimateFileGroupCountForPartitionedRLI() { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.METADATA_ENABLED, true); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.RECORD_LEVEL_INDEX.name()); + conf.setString(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), "true"); + + // testing default value + assertEquals(1, OptionsResolver.estimateFileGroupCountForRLI(conf)); + + // testing user configured value + conf.setString(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), "3"); + conf.setString(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), "3"); + assertEquals(3, OptionsResolver.estimateFileGroupCountForRLI(conf)); + } + + @Test + void testEstimateFileGroupCountForGlobalRLI() { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.METADATA_ENABLED, true); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX.name()); + conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key(), "true"); + + // testing default value + assertEquals(8, OptionsResolver.estimateFileGroupCountForRLI(conf)); + + // testing user configured value + conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), "11"); + conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), "11"); + assertEquals(11, OptionsResolver.estimateFileGroupCountForRLI(conf)); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestMinibatchBucketAssignFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestMinibatchBucketAssignFunction.java index a98a95529fc18..0e83ff81f75ad 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestMinibatchBucketAssignFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestMinibatchBucketAssignFunction.java @@ -234,14 +234,94 @@ public void testGlobalIndexUpdate() throws Exception { } } + @Test + public void testDuplicateKeyCrossPartitionUpdate() throws Exception { + HoodieFlinkInternalRow record1 = insertRecord("id1", "par2", 1); + HoodieFlinkInternalRow record2 = insertRecord("id1", "par3", 2); + + testHarness.processElement(new StreamRecord<>(record1)); + testHarness.processElement(new StreamRecord<>(record2)); + testHarness.prepareSnapshotPreBarrier(1L); + + List output = testHarness.extractOutputValues(); + assertEquals(4, output.size(), "Two cross-partition updates should each emit delete and insert records"); + + HoodieFlinkInternalRow deleteForOriginalPartition = output.get(0); + HoodieFlinkInternalRow insertForFirstUpdate = output.get(1); + HoodieFlinkInternalRow deleteForFirstUpdate = output.get(2); + HoodieFlinkInternalRow insertForSecondUpdate = output.get(3); + + assertEquals("par1", deleteForOriginalPartition.getPartitionPath(), + "First update should delete the location prefetched from the metadata table"); + assertEquals("-U", deleteForOriginalPartition.getOperationType()); + assertEquals("U", deleteForOriginalPartition.getInstantTime()); + + assertEquals("par2", insertForFirstUpdate.getPartitionPath()); + assertEquals("I", insertForFirstUpdate.getOperationType()); + assertEquals("U", insertForFirstUpdate.getInstantTime()); + assertTrue(insertForFirstUpdate.getFileId() != null && !insertForFirstUpdate.getFileId().isEmpty(), + "First update should be assigned to a data bucket"); + + assertEquals("par2", deleteForFirstUpdate.getPartitionPath(), + "Duplicate key should re-read the location updated by the preceding record in the same minibatch"); + assertEquals(insertForFirstUpdate.getFileId(), deleteForFirstUpdate.getFileId(), + "The delete for the second update should target the bucket assigned to the first update"); + assertEquals("-U", deleteForFirstUpdate.getOperationType()); + assertEquals("U", deleteForFirstUpdate.getInstantTime()); + + assertEquals("par3", insertForSecondUpdate.getPartitionPath()); + assertEquals("I", insertForSecondUpdate.getOperationType()); + assertEquals("U", insertForSecondUpdate.getInstantTime()); + assertTrue(insertForSecondUpdate.getFileId() != null && !insertForSecondUpdate.getFileId().isEmpty(), + "Second update should be assigned to a data bucket"); + } + + @Test + public void testDuplicateKeyInSameMinibatch() throws Exception { + HoodieFlinkInternalRow record1 = insertRecord("new_duplicate_key", "par_insert", 1); + HoodieFlinkInternalRow record2 = insertRecord("new_duplicate_key", "par_insert", 2); + + testHarness.processElement(new StreamRecord<>(record1)); + testHarness.processElement(new StreamRecord<>(record2)); + testHarness.prepareSnapshotPreBarrier(1L); + + List output = testHarness.extractOutputValues(); + assertEquals(2, output.size(), "Duplicate insert-miss records should both be emitted"); + + HoodieFlinkInternalRow insertRecord = output.get(0); + HoodieFlinkInternalRow duplicateRecord = output.get(1); + + assertEquals("new_duplicate_key", insertRecord.getRecordKey()); + assertEquals("par_insert", insertRecord.getPartitionPath()); + assertEquals("I", insertRecord.getInstantTime(), + "The first record should be assigned as an insert because the prefetched location is missing"); + assertEquals("I", insertRecord.getOperationType()); + assertTrue(insertRecord.getFileId() != null && !insertRecord.getFileId().isEmpty(), + "First insert should be assigned to a data bucket"); + + assertEquals("new_duplicate_key", duplicateRecord.getRecordKey()); + assertEquals("par_insert", duplicateRecord.getPartitionPath()); + assertEquals(insertRecord.getFileId(), duplicateRecord.getFileId(), + "Duplicate key should re-read the location written by the first insert in the same minibatch"); + assertEquals("U", duplicateRecord.getInstantTime(), + "The duplicate record should be assigned as an update to the first record's bucket"); + assertEquals("I", duplicateRecord.getOperationType()); + } + @Test public void testCloseFunction() throws Exception { // Test that close doesn't throw exceptions HoodieFlinkInternalRow record = new HoodieFlinkInternalRow("id1", "par1", "I", insertRow(StringData.fromString("id1"), StringData.fromString("Danny"), 23, TimestampData.fromEpochMillis(1), StringData.fromString("par1"))); testHarness.processElement(new StreamRecord<>(record)); - + // Close should not throw any exceptions testHarness.close(); } + + private static HoodieFlinkInternalRow insertRecord(String recordKey, String partitionPath, long ts) { + return new HoodieFlinkInternalRow(recordKey, partitionPath, "I", + insertRow(StringData.fromString(recordKey), StringData.fromString("Danny"), 23, + TimestampData.fromEpochMillis(ts), StringData.fromString(partitionPath))); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestRecordIndexPartitioner.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestRecordIndexPartitioner.java index 59cd3d93fb0be..e8b6c757ba771 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestRecordIndexPartitioner.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestRecordIndexPartitioner.java @@ -97,6 +97,7 @@ void testSinglePartition() throws Exception { private RecordIndexPartitioner newPartitioner() throws Exception { Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.RECORD_LEVEL_INDEX.name()); + conf.setString(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), "true"); conf.setString(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key(), String.valueOf(FILE_GROUP_COUNT)); StreamerUtil.initTableIfNotExists(conf); return new RecordIndexPartitioner(conf); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java index 3b3f72a9fcb56..37ac44d6ae06f 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java @@ -81,10 +81,12 @@ void testRecordLevelIndexBackend() throws Exception { assertEquals("par1", location.getPartitionPath()); assertEquals(firstCommitTime, location.getInstantTime()); + HoodieRecordGlobalLocation location1 = globalRecordLevelIndexBackend.get("id1"); + assertEquals(location1, location); + // get record location with non existed key location = globalRecordLevelIndexBackend.get(Collections.singletonList("new_key")).get("new_key"); assertNull(location); - assertThrows(UnsupportedOperationException.class, () -> globalRecordLevelIndexBackend.get("id1")); // get records locations for multiple record keys Map locations = globalRecordLevelIndexBackend.get(Arrays.asList("id1", "id2", "id3")); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 4af3598c0a979..610a23b3b0e76 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -1537,8 +1537,8 @@ void testWriteNonPartitionedTable(ExecMode execMode, HoodieTableType tableType) } @ParameterizedTest - @EnumSource(value = HoodieIndex.IndexType.class, names = {"FLINK_STATE", "GLOBAL_RECORD_LEVEL_INDEX"}) - void testWriteGlobalIndex(HoodieIndex.IndexType indexType) { + @MethodSource("indexAndBooleanParams") + void testWriteGlobalIndex(String indexType, boolean bootstrapEnabled) { // the source generates 4 commits String createSource = TestConfigurations.getFileSourceDDL( "source", "test_source_4.data", 4); @@ -1548,7 +1548,8 @@ void testWriteGlobalIndex(HoodieIndex.IndexType indexType) { .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) .options(getDefaultKeys()) .option(FlinkOptions.INDEX_GLOBAL_ENABLED, true) - .option(FlinkOptions.INDEX_TYPE, indexType.name()) + .option(FlinkOptions.INDEX_TYPE, indexType) + .option(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, bootstrapEnabled) .option(FlinkOptions.PRE_COMBINE, true) .end(); streamTableEnv.executeSql(hoodieTableDDL); @@ -3416,6 +3417,7 @@ void testRLIBootstrap() { .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) .options(getDefaultKeys()) .option(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX.name()) + .option(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true) .option(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true) .option(FlinkOptions.TABLE_TYPE, MERGE_ON_READ.name()) .end(); @@ -3717,6 +3719,18 @@ private static Stream indexAndPartitioningParams() { return Stream.of(data).map(Arguments::of); } + /** + * Return test params => (index type, boolean). + */ + private static Stream indexAndBooleanParams() { + Object[][] data = + new Object[][] { + {"FLINK_STATE", false}, + {"GLOBAL_RECORD_LEVEL_INDEX", false}, + {"GLOBAL_RECORD_LEVEL_INDEX", true}}; + return Stream.of(data).map(Arguments::of); + } + /** * Return test params => (index type, table type). */ diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java index 6aa6f20581c3e..a85d3d29b6aed 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java @@ -785,6 +785,23 @@ void testSetupWriteOptionsForSink() { (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(MockContext.getInstance(this.conf)); Configuration conf2 = tableSink2.getConf(); assertThat(conf2.get(FlinkOptions.PRE_COMBINE), is(false)); + + // Global RLI setup should not enable index bootstrap implicitly. + Configuration globalRLIConf = new Configuration(this.conf); + globalRLIConf.set(FlinkOptions.OPERATION, "upsert"); + globalRLIConf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX.name()); + globalRLIConf.set(FlinkOptions.METADATA_ENABLED, true); + globalRLIConf.set(FlinkOptions.INDEX_GLOBAL_ENABLED, true); + HoodieTableSink globalRLISink = + (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(MockContext.getInstance(globalRLIConf)); + Configuration globalRLIResolvedConf = globalRLISink.getConf(); + assertThat(globalRLIResolvedConf.get(FlinkOptions.INDEX_BOOTSTRAP_ENABLED), is(false)); + + globalRLIConf.set(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true); + HoodieTableSink globalRLIWithBootstrapSink = + (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(MockContext.getInstance(globalRLIConf)); + Configuration globalRLIWithBootstrapResolvedConf = globalRLIWithBootstrapSink.getConf(); + assertThat(globalRLIWithBootstrapResolvedConf.get(FlinkOptions.INDEX_BOOTSTRAP_ENABLED), is(true)); } @Test From e3b86c3227a658c4cf4e90f51ae274c6234520bd Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Sat, 6 Jun 2026 21:25:15 +0800 Subject: [PATCH 051/255] chore: [MINOR] Update DOAP with 0.14.2 Release (#18924) (cherry picked from commit 9ca18678b8f0add7eac1ed14a9f264b7591460f6) --- doap_HUDI.rdf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf index 9f56441f3f45f..89ca27d04c36b 100644 --- a/doap_HUDI.rdf +++ b/doap_HUDI.rdf @@ -245,6 +245,13 @@ 1.2.0 + + + Apache Hudi 0.14.2 + 2026-06-06 + 0.14.2 + + From bdbb493134c624bbd39f1be0e27d212c7d190d8d Mon Sep 17 00:00:00 2001 From: fhan Date: Tue, 9 Jun 2026 08:16:18 +0800 Subject: [PATCH 052/255] fix(flink): avoid repeated timeline reload for unchanged lookup table commits (#18930) Co-authored-by: fhan (cherry picked from commit f4816ce821094822684e80a76b7ddf7e35d052f9) --- .../table/lookup/HoodieLookupFunction.java | 13 +- .../lookup/TestHoodieLookupFunction.java | 161 ++++++++++++++++++ 2 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java index 443cba0bcf508..8768222463979 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java @@ -135,13 +135,16 @@ private void checkCacheReload() throws IOException { } HoodieActiveTimeline latestCommit = metaClient.reloadActiveTimeline(); - Option latestCommitInstant = latestCommit.getCommitsTimeline().lastInstant(); - if (latestCommit.empty()) { + Option latestCommitInstant = + latestCommit.getCommitsTimeline().filterCompletedInstants().lastInstant(); + if (!latestCommitInstant.isPresent()) { + scheduleNextLoad(); log.info("No commit instant found currently."); return; } // Determine whether to reload data by comparing instant if (latestCommitInstant.get().equals(currentCommit)) { + scheduleNextLoad(); log.info("Ignore loading data because the commit instant " + currentCommit + " has not changed."); return; } @@ -162,7 +165,7 @@ private void checkCacheReload() throws IOException { } partitionReader.close(); currentCommit = latestCommitInstant.get(); - nextLoadTime = System.currentTimeMillis() + reloadInterval.toMillis(); + scheduleNextLoad(); log.info("Loaded {} row(s) into lookup join cache", count); return; } catch (Exception e) { @@ -185,6 +188,10 @@ private void checkCacheReload() throws IOException { } } + private void scheduleNextLoad() { + nextLoadTime = System.currentTimeMillis() + reloadInterval.toMillis(); + } + private RowData extractLookupKey(RowData row) { GenericRowData key = new GenericRowData(lookupFieldGetters.length); for (int i = 0; i < lookupFieldGetters.length; i++) { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java new file mode 100644 index 0000000000000..8d336ede565e1 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.lookup; + +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.util.StreamerUtil; +import org.apache.hudi.utils.TestConfigurations; +import org.apache.hudi.utils.TestData; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link HoodieLookupFunction}. + */ +class TestHoodieLookupFunction { + + @TempDir + File tempFile; + + @Test + void testNextLoadTimeAdvancesWhenNoCompletedCommit() throws Exception { + Configuration conf = getConf(); + StreamerUtil.initTableIfNotExists(conf); + + CountingLookupTableReader reader = new CountingLookupTableReader(Collections.emptyList(), conf); + HoodieLookupFunction function = newLookupFunction(reader, conf); + function.open(null); + + long beforeLoad = System.currentTimeMillis(); + try { + function.lookup(lookupKey()); + + assertEquals(0, reader.openCount, "The reader should not open when no completed commit exists"); + assertTrue(getNextLoadTime(function) >= beforeLoad + Duration.ofDays(1).toMillis(), + "The next lookup reload check should be delayed by the configured TTL"); + } finally { + function.close(); + } + } + + @Test + void testLookupCacheDoesNotReloadWhenCompletedCommitHasNotChanged() throws Exception { + Configuration conf = getConf(); + TestData.writeData(TestData.DATA_SET_SINGLE_INSERT, conf); + + CountingLookupTableReader reader = new CountingLookupTableReader(TestData.DATA_SET_SINGLE_INSERT, conf); + HoodieLookupFunction function = newLookupFunction(reader, conf); + function.open(null); + + try { + Collection matchedRows = function.lookup(lookupKey()); + assertNotNull(matchedRows, "The first lookup should find the inserted row"); + assertEquals(1, matchedRows.size(), "The first lookup should load the table into cache"); + assertEquals(1, reader.openCount, "The first lookup should open the reader once"); + + // Force the next lookup through the reload branch so the unchanged-commit guard is exercised. + setNextLoadTime(function, 0L); + function.lookup(lookupKey()); + + assertEquals(1, reader.openCount, "The same completed commit should not reload table data"); + assertTrue(getNextLoadTime(function) > System.currentTimeMillis(), + "The next lookup reload check should be delayed after detecting an unchanged commit"); + } finally { + function.close(); + } + } + + private HoodieLookupFunction newLookupFunction(CountingLookupTableReader reader, Configuration conf) { + return new HoodieLookupFunction( + reader, + TestConfigurations.ROW_TYPE, + new int[] {0}, + Duration.ofDays(1), + conf); + } + + private Configuration getConf() { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf.set(FlinkOptions.LOOKUP_JOIN_CACHE_TTL, Duration.ofDays(1)); + return conf; + } + + private static RowData lookupKey() { + return GenericRowData.of(StringData.fromString("id1")); + } + + private static long getNextLoadTime(HoodieLookupFunction function) throws Exception { + Field field = HoodieLookupFunction.class.getDeclaredField("nextLoadTime"); + field.setAccessible(true); + return field.getLong(function); + } + + private static void setNextLoadTime(HoodieLookupFunction function, long nextLoadTime) throws Exception { + Field field = HoodieLookupFunction.class.getDeclaredField("nextLoadTime"); + field.setAccessible(true); + field.setLong(function, nextLoadTime); + } + + private static class CountingLookupTableReader extends HoodieLookupTableReader { + private final List rows; + private int openCount; + private int nextIndex; + + private CountingLookupTableReader(List rows, Configuration conf) { + super(() -> null, conf); + this.rows = rows; + } + + @Override + public void open() { + openCount++; + nextIndex = 0; + } + + @Override + public RowData read(RowData reuse) { + if (nextIndex >= rows.size()) { + return null; + } + return rows.get(nextIndex++); + } + + @Override + public void close() throws IOException { + // no-op + } + } +} From 622ad78cf45ac2bb390571c32f695343402effd5 Mon Sep 17 00:00:00 2001 From: fhan Date: Tue, 9 Jun 2026 12:05:09 +0800 Subject: [PATCH 053/255] fix(common): Close log writer output stream on append failure (#18909) * fix(common): Close log writer output stream on append failure * simplify the exception handling --------- Co-authored-by: fhan Co-authored-by: danny0405 (cherry picked from commit 782bf654ee4cac9da7e1244da03d83b298bbbe3a) --- .../apache/hudi/exception/ExceptionUtil.java | 18 +- .../hudi/exception/TestExceptionUtil.java | 20 ++ .../table/log/HoodieLogFormatWriter.java | 184 +++++++++++------- .../table/log/TestHoodieLogFormatWriter.java | 180 +++++++++++++++++ 4 files changed, 333 insertions(+), 69 deletions(-) create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatWriter.java diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java index 92e2e3cc53564..40a39542c7a71 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java @@ -23,6 +23,8 @@ import javax.annotation.Nonnull; +import java.io.IOException; + /** * Util class for exception analysis. */ @@ -31,7 +33,7 @@ private ExceptionUtil() { } /** - * Returns true if error message is contained in any nested exception of provided {@link Throwable}. + * Returns true if error message is contained in any nested exception to provided {@link Throwable}. */ public static boolean validateErrorMsg(@Nonnull Throwable t, String errorMsg) { if (StringUtils.isNullOrEmpty(errorMsg)) { @@ -48,4 +50,18 @@ public static boolean validateErrorMsg(@Nonnull Throwable t, String errorMsg) { return false; } + + /** + * Throws the provided exception as-is when it is an {@link IOException} or + * {@link RuntimeException}, otherwise wraps it in an {@link IOException}. + */ + public static void throwAsIOExceptionOrRuntimeException(Throwable exception) throws IOException { + if (exception instanceof IOException) { + throw (IOException) exception; + } + if (exception instanceof RuntimeException) { + throw (RuntimeException) exception; + } + throw new IOException(exception); + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java index dba850b1b907a..c2244896d855d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java @@ -23,8 +23,11 @@ import java.io.IOException; +import static org.apache.hudi.exception.ExceptionUtil.throwAsIOExceptionOrRuntimeException; import static org.apache.hudi.exception.ExceptionUtil.validateErrorMsg; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; class TestExceptionUtil { @@ -54,4 +57,21 @@ void testValidateErrorMsgWithEmptyMessage() { // Empty string should not be found in any message (including null) assertFalse(validateErrorMsg(exceptionWithoutMessage, "")); } + + @Test + void testThrowAsIOExceptionOrRuntimeException() { + IOException ioException = new IOException("io"); + IOException thrownIOException = assertThrows(IOException.class, () -> throwAsIOExceptionOrRuntimeException(ioException)); + assertSame(ioException, thrownIOException); + + RuntimeException runtimeException = new RuntimeException("runtime"); + RuntimeException thrownRuntimeException = + assertThrows(RuntimeException.class, () -> throwAsIOExceptionOrRuntimeException(runtimeException)); + assertSame(runtimeException, thrownRuntimeException); + + Exception checkedException = new Exception("checked"); + IOException wrappedException = + assertThrows(IOException.class, () -> throwAsIOExceptionOrRuntimeException(checkedException)); + assertSame(checkedException, wrappedException.getCause()); + } } diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 2aed1d7dd87a2..fe24bd600f1ee 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.VisibleForTesting; +import org.apache.hudi.exception.ExceptionUtil; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; @@ -125,68 +126,73 @@ public AppendResult appendBlock(HoodieLogBlock block) throws IOException, Interr @Override public AppendResult appendBlocks(List blocks) throws IOException { - // Find current version - HoodieLogFormat.LogFormatVersion currentLogFormatVersion = - new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION); - - FSDataOutputStream originalOutputStream = getOutputStream(); - long startPos = originalOutputStream.getPos(); - long sizeWritten = 0; - // HUDI-2655. here we wrap originalOutputStream to ensure huge blocks can be correctly written - FSDataOutputStream outputStream = new FSDataOutputStream(originalOutputStream, new FileSystem.Statistics(storage.getScheme()), startPos); - for (HoodieLogBlock block: blocks) { - long startSize = outputStream.size(); - - // 1. Write the magic header for the start of the block - outputStream.write(HoodieLogFormat.MAGIC); - - // bytes for header - byte[] headerBytes = HoodieLogBlock.getHeaderMetadataBytes(block.getLogBlockHeader()); - // content bytes - ByteArrayOutputStream content = block.getContentBytes(storage); - // bytes for footer - byte[] footerBytes = HoodieLogBlock.getFooterMetadataBytes(block.getLogBlockFooter()); - - // 2. Write the total size of the block (excluding Magic) - outputStream.writeLong(getLogBlockLength(content.size(), headerBytes.length, footerBytes.length)); - - // 3. Write the version of this log block - outputStream.writeInt(currentLogFormatVersion.getVersion()); - // 4. Write the block type - outputStream.writeInt(block.getBlockType().ordinal()); - - // 5. Write the headers for the log block - outputStream.write(headerBytes); - // 6. Write the size of the content block - outputStream.writeLong(content.size()); - // 7. Write the contents of the data block - content.writeTo(outputStream); - // 8. Write the footers for the log block - outputStream.write(footerBytes); - // 9. Write the total size of the log block (including magic) which is everything written - // until now (for reverse pointer) - // Update: this information is now used in determining if a block is corrupt by comparing to the - // block size in header. This change assumes that the block size will be the last data written - // to a block. Read will break if any data is written past this point for a block. - outputStream.writeLong(outputStream.size() - startSize); - - // Fetch the size again, so it accounts also (9). - - // HUDI-2655. Check the size written to avoid log blocks whose size overflow. - if (outputStream.size() == Integer.MAX_VALUE) { - throw new HoodieIOException("Blocks appended may overflow. Please decrease log block size or log block amount"); + try { + // Find current version + HoodieLogFormat.LogFormatVersion currentLogFormatVersion = + new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION); + + FSDataOutputStream originalOutputStream = getOutputStream(); + long startPos = originalOutputStream.getPos(); + long sizeWritten = 0; + // HUDI-2655. here we wrap originalOutputStream to ensure huge blocks can be correctly written + FSDataOutputStream outputStream = new FSDataOutputStream(originalOutputStream, new FileSystem.Statistics(storage.getScheme()), startPos); + for (HoodieLogBlock block: blocks) { + long startSize = outputStream.size(); + + // 1. Write the magic header for the start of the block + outputStream.write(HoodieLogFormat.MAGIC); + + // bytes for header + byte[] headerBytes = HoodieLogBlock.getHeaderMetadataBytes(block.getLogBlockHeader()); + // content bytes + ByteArrayOutputStream content = block.getContentBytes(storage); + // bytes for footer + byte[] footerBytes = HoodieLogBlock.getFooterMetadataBytes(block.getLogBlockFooter()); + + // 2. Write the total size of the block (excluding Magic) + outputStream.writeLong(getLogBlockLength(content.size(), headerBytes.length, footerBytes.length)); + + // 3. Write the version of this log block + outputStream.writeInt(currentLogFormatVersion.getVersion()); + // 4. Write the block type + outputStream.writeInt(block.getBlockType().ordinal()); + + // 5. Write the headers for the log block + outputStream.write(headerBytes); + // 6. Write the size of the content block + outputStream.writeLong(content.size()); + // 7. Write the contents of the data block + content.writeTo(outputStream); + // 8. Write the footers for the log block + outputStream.write(footerBytes); + // 9. Write the total size of the log block (including magic) which is everything written + // until now (for reverse pointer) + // Update: this information is now used in determining if a block is corrupt by comparing to the + // block size in header. This change assumes that the block size will be the last data written + // to a block. Read will break if any data is written past this point for a block. + outputStream.writeLong(outputStream.size() - startSize); + + // Fetch the size again, so it accounts also (9). + + // HUDI-2655. Check the size written to avoid log blocks whose size overflow. + if (outputStream.size() == Integer.MAX_VALUE) { + throw new HoodieIOException("Blocks appended may overflow. Please decrease log block size or log block amount"); + } + sizeWritten += outputStream.size() - startSize; } - sizeWritten += outputStream.size() - startSize; + // No flush/hsync here: append-time visibility is not part of the contract. + // Downstream readers only need commit-level visibility, which is provided + // when the writer is closed (see closeStream) or when callers explicitly + // invoke sync(). + + AppendResult result = new AppendResult(logFile, startPos, sizeWritten); + // roll over if size is past the threshold + rolloverIfNeeded(); + return result; + } catch (IOException | RuntimeException e) { + closeOutputStreamOnAppendFailure(e); + throw e; } - // No flush/hsync here: append-time visibility is not part of the contract. - // Downstream readers only need commit-level visibility, which is provided - // when the writer is closed (see closeStream) or when callers explicitly - // invoke sync(). - - AppendResult result = new AppendResult(logFile, startPos, sizeWritten); - // roll over if size is past the threshold - rolloverIfNeeded(); - return result; } /** @@ -237,21 +243,63 @@ private void createNewFile() throws IOException { @Override public void close() throws IOException { - closeStream(); - // remove the shutdown hook after closing the stream to avoid memory leaks - if (null != shutdownThread) { - Runtime.getRuntime().removeShutdownHook(shutdownThread); + try { + closeStream(); + } finally { + // remove the shutdown hook after closing the stream to avoid memory leaks + if (null != shutdownThread) { + Runtime.getRuntime().removeShutdownHook(shutdownThread); + shutdownThread = null; + } } } private void closeStream() throws IOException { - if (outputStream != null) { + if (outputStream == null) { + return; + } + + Throwable failure = null; + try { // Persist all buffered data to DataNodes before closing so downstream // readers can observe a fully-written log file at commit-level visibility. sync(); - outputStream.close(); - outputStream = null; - closed = true; + } catch (IOException | RuntimeException e) { + failure = e; + } + + try { + closeOutputStream(); + } catch (IOException | RuntimeException closeException) { + if (failure != null) { + failure.addSuppressed(closeException); + } else { + failure = closeException; + } + } + + if (failure != null) { + ExceptionUtil.throwAsIOExceptionOrRuntimeException(failure); + } + } + + private void closeOutputStreamOnAppendFailure(Throwable failure) { + try { + closeOutputStream(); + } catch (IOException | RuntimeException closeException) { + failure.addSuppressed(closeException); + log.warn("Failed to close output stream after append failure for log file {}", logFile, closeException); + } + } + + private void closeOutputStream() throws IOException { + if (outputStream != null) { + try { + outputStream.close(); + } finally { + outputStream = null; + closed = true; + } } } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatWriter.java new file mode 100644 index 0000000000000..15316012a97ab --- /dev/null +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatWriter.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.table.log; + +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestHoodieLogFormatWriter { + + private static final String WRITE_FAIL = "write-fail"; + private static final String CLOSE_FAIL = "close-fail"; + private static final String SYNC_FAIL = "sync-fail"; + + @TempDir + java.nio.file.Path tempDir; + + @Test + void testCloseOutputOnAppendWriteException() throws IOException { + HoodieStorage storage = HoodieTestUtils.getStorage(tempDir.toString()); + HoodieLogFormatWriter writer = newWriter(storage); + try { + CloseTrackingOutputStream outputStream = new CloseTrackingOutputStream(true, false); + writer.withOutputStream(newFSDataOutputStream(outputStream, storage)); + + IOException exception = assertThrows(IOException.class, () -> writer.appendBlock(commandBlock())); + + assertEquals(WRITE_FAIL, exception.getMessage()); + assertTrue(outputStream.isClosed()); + assertThrows(IllegalStateException.class, writer::getCurrentSize); + } finally { + writer.close(); + } + } + + @Test + void testPreserveAppendExceptionWhenCloseFails() throws IOException { + HoodieStorage storage = HoodieTestUtils.getStorage(tempDir.toString()); + HoodieLogFormatWriter writer = newWriter(storage); + try { + CloseTrackingOutputStream outputStream = new CloseTrackingOutputStream(true, true); + writer.withOutputStream(newFSDataOutputStream(outputStream, storage)); + + IOException exception = assertThrows(IOException.class, () -> writer.appendBlock(commandBlock())); + + assertEquals(WRITE_FAIL, exception.getMessage()); + assertTrue(outputStream.isClosed()); + assertEquals(1, exception.getSuppressed().length); + assertEquals(CLOSE_FAIL, exception.getSuppressed()[0].getMessage()); + assertThrows(IllegalStateException.class, writer::getCurrentSize); + } finally { + writer.close(); + } + } + + @Test + void testCloseOutputWhenSyncFailsOnClose() throws IOException { + HoodieStorage storage = HoodieTestUtils.getStorage(tempDir.toString()); + HoodieLogFormatWriter writer = newWriter(storage); + try { + CloseTrackingOutputStream outputStream = new CloseTrackingOutputStream(false, false); + writer.withOutputStream(new SyncFailingFSDataOutputStream(outputStream, storage)); + + IOException exception = assertThrows(IOException.class, writer::close); + + assertEquals(SYNC_FAIL, exception.getMessage()); + assertTrue(outputStream.isClosed()); + assertThrows(IllegalStateException.class, writer::getCurrentSize); + } finally { + writer.close(); + } + } + + private HoodieLogFormatWriter newWriter(HoodieStorage storage) throws IOException { + return HoodieLogFormatWriter.builder() + .withParentPath(new StoragePath(tempDir.toString())) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid") + .withInstantTime("100") + .withLogVersion(1) + .withStorage(storage) + .build(); + } + + private HoodieCommandBlock commandBlock() { + Map header = new HashMap<>(); + header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE, + String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_BLOCK.ordinal())); + return new HoodieCommandBlock(header); + } + + private FSDataOutputStream newFSDataOutputStream(CloseTrackingOutputStream outputStream, HoodieStorage storage) + throws IOException { + return new FSDataOutputStream(outputStream, new FileSystem.Statistics(storage.getScheme())); + } + + private static class CloseTrackingOutputStream extends OutputStream { + + private final boolean failOnWrite; + private final boolean failOnClose; + private boolean closed; + + private CloseTrackingOutputStream(boolean failOnWrite, boolean failOnClose) { + this.failOnWrite = failOnWrite; + this.failOnClose = failOnClose; + } + + @Override + public void write(int b) throws IOException { + if (failOnWrite) { + throw new IOException(WRITE_FAIL); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (failOnWrite) { + throw new IOException(WRITE_FAIL); + } + } + + @Override + public void close() throws IOException { + closed = true; + if (failOnClose) { + throw new IOException(CLOSE_FAIL); + } + } + + private boolean isClosed() { + return closed; + } + } + + private static class SyncFailingFSDataOutputStream extends FSDataOutputStream { + + private SyncFailingFSDataOutputStream(CloseTrackingOutputStream outputStream, HoodieStorage storage) + throws IOException { + super(outputStream, new FileSystem.Statistics(storage.getScheme())); + } + + @Override + public void hsync() throws IOException { + throw new IOException(SYNC_FAIL); + } + } +} From f1259ca8cd0274aced9dc9d702609f377a3923b1 Mon Sep 17 00:00:00 2001 From: Peter Huang Date: Tue, 9 Jun 2026 07:42:26 -0700 Subject: [PATCH 054/255] fix(metrics): NPE handling when hudi metrics is disabled (#18947) (cherry picked from commit 1fd2c3671a8081227727f5d6e175ef49d9a66f2d) --- .../apache/hudi/metrics/HoodieMetrics.java | 11 +- .../hudi/metrics/TestHoodieMetrics.java | 316 ++++++++++++++++++ .../common/metrics/HoodieMetaSyncMetrics.java | 3 + 3 files changed, 326 insertions(+), 4 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java index c777e842bbd39..941085f451bd2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java @@ -266,7 +266,7 @@ public Timer.Context getSourceReadAndIndexTimerCtx() { } public Timer.Context getConflictResolutionCtx() { - if (config.isLockingMetricsEnabled() && conflictResolutionTimer == null) { + if (config.isMetricsOn() && config.isLockingMetricsEnabled() && conflictResolutionTimer == null) { conflictResolutionTimer = createTimer(conflictResolutionTimerName); } return conflictResolutionTimer == null ? null : conflictResolutionTimer.time(); @@ -555,6 +555,9 @@ private void updateMetric(final String action, final String metricName, final lo * Given a commit action, metrics name and value this method reports custom metrics. */ public void reportMetrics(String commitAction, String metricName, long value) { + if (!config.isMetricsOn()) { + return; + } metrics.registerGauge(getMetricsName(commitAction, metricName), value); } @@ -566,7 +569,7 @@ public long getDurationInMs(long ctxDuration) { } public void emitConflictResolutionSuccessful() { - if (config.isLockingMetricsEnabled()) { + if (config.isMetricsOn() && config.isLockingMetricsEnabled()) { log.info("Sending conflict resolution success metric"); conflictResolutionSuccessCounter = getCounter(conflictResolutionSuccessCounter, conflictResolutionSuccessCounterName); conflictResolutionSuccessCounter.inc(); @@ -574,7 +577,7 @@ public void emitConflictResolutionSuccessful() { } public void emitConflictResolutionFailed() { - if (config.isLockingMetricsEnabled()) { + if (config.isMetricsOn() && config.isLockingMetricsEnabled()) { log.info("Sending conflict resolution failure metric"); conflictResolutionFailureCounter = getCounter(conflictResolutionFailureCounter, conflictResolutionFailureCounterName); conflictResolutionFailureCounter.inc(); @@ -582,7 +585,7 @@ public void emitConflictResolutionFailed() { } public void emitConflictResolutionByCategory(HoodieWriteConflictException.ConflictCategory category) { - if (config.isLockingMetricsEnabled()) { + if (config.isMetricsOn() && config.isLockingMetricsEnabled()) { switch (category) { case INGESTION_VS_INGESTION: conflictResolutionIngestionVsIngestionCounter = getCounter( diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java index b785d90252880..1a2702a77a040 100755 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java @@ -40,6 +40,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.stream.Stream; @@ -48,7 +50,10 @@ import static org.apache.hudi.metrics.HoodieMetrics.COUNTER_METRIC_EXTENSION; import static org.apache.hudi.metrics.HoodieMetrics.FAILURE_COUNTER; import static org.apache.hudi.metrics.HoodieMetrics.SOURCE_READ_AND_INDEX_ACTION; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -322,6 +327,317 @@ public MockHoodieActiveTimeline(HoodieInstant... instants) { } } + // ----------------------------------------------------------------------- + // Metrics-off safety tests (NPE guards) + // ----------------------------------------------------------------------- + + private HoodieMetrics buildMetricsOff() { + HoodieWriteConfig offConfig = mock(HoodieWriteConfig.class); + when(offConfig.isMetricsOn()).thenReturn(false); + when(offConfig.getTableName()).thenReturn("test_table"); + return new HoodieMetrics(offConfig, HoodieTestUtils.getDefaultStorage()); + } + + @Test + public void testTimerContextsReturnNullWhenMetricsOff() { + HoodieMetrics metricsOff = buildMetricsOff(); + assertNull(metricsOff.getRollbackCtx()); + assertNull(metricsOff.getCompactionCtx()); + assertNull(metricsOff.getLogCompactionCtx()); + assertNull(metricsOff.getClusteringCtx()); + assertNull(metricsOff.getCleanCtx()); + assertNull(metricsOff.getArchiveCtx()); + assertNull(metricsOff.getCommitCtx()); + assertNull(metricsOff.getFinalizeCtx()); + assertNull(metricsOff.getDeltaCommitCtx()); + assertNull(metricsOff.getIndexCtx()); + assertNull(metricsOff.getSourceReadAndIndexTimerCtx()); + assertNull(metricsOff.getConflictResolutionCtx()); + } + + @Test + public void testUpdateMethodsAreNoOpsWhenMetricsOff() { + HoodieMetrics metricsOff = buildMetricsOff(); + HoodieCommitMetadata metadata = mock(HoodieCommitMetadata.class); + + assertDoesNotThrow(() -> metricsOff.updateCommitMetrics(0L, 0L, metadata, "commit")); + assertDoesNotThrow(() -> metricsOff.updateRollbackMetrics(0L, 0L)); + assertDoesNotThrow(() -> metricsOff.updateCleanMetrics(0L, 0)); + assertDoesNotThrow(() -> metricsOff.updateFinalizeWriteMetrics(0L, 0L)); + assertDoesNotThrow(() -> metricsOff.updateIndexMetrics("action", 0L)); + assertDoesNotThrow(() -> metricsOff.updateSourceReadAndIndexMetrics("action", 0L)); + assertDoesNotThrow(() -> metricsOff.updateArchiveMetrics(0L, 0)); + assertDoesNotThrow(() -> metricsOff.updateArchivalMetrics(new HashMap<>())); + assertDoesNotThrow(() -> metricsOff.updatePostCommitMetrics(true, 0L)); + assertDoesNotThrow(() -> metricsOff.updatePostCommitMetrics(false, 0L)); + assertDoesNotThrow(() -> metricsOff.reportMetrics("action", "metric", 0L)); + assertDoesNotThrow(() -> metricsOff.updateClusteringFileCreationMetrics(0L)); + assertDoesNotThrow(() -> metricsOff.emitRollbackFailure("SomeException")); + assertDoesNotThrow(() -> metricsOff.emitRollbackFailure(null)); + assertDoesNotThrow(() -> metricsOff.emitCompactionRequested()); + assertDoesNotThrow(() -> metricsOff.emitCompactionCompleted()); + assertDoesNotThrow(() -> metricsOff.emitIndexTypeMetrics(0)); + assertDoesNotThrow(() -> metricsOff.emitMetadataEnablementMetrics(false, false, false, false)); + assertDoesNotThrow(() -> metricsOff.emitVersionMetrics()); + } + + @Test + public void testConflictResolutionMetricsAreNoOpsWhenMetricsOffButLockingEnabled() { + // Key NPE scenario: global metrics are off so metrics field is null. + // The isMetricsOn() && isLockingMetricsEnabled() guard short-circuits before + // reaching getCounter() -> metrics.getRegistry(), so no NPE should occur. + HoodieWriteConfig offConfig = mock(HoodieWriteConfig.class); + when(offConfig.isMetricsOn()).thenReturn(false); + when(offConfig.getTableName()).thenReturn("test_table"); + HoodieMetrics metricsOff = new HoodieMetrics(offConfig, HoodieTestUtils.getDefaultStorage()); + + assertNull(metricsOff.getConflictResolutionCtx()); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionSuccessful()); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionFailed()); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionByCategory( + HoodieWriteConflictException.ConflictCategory.INGESTION_VS_INGESTION)); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionByCategory( + HoodieWriteConflictException.ConflictCategory.INGESTION_VS_TABLE_SERVICE)); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionByCategory( + HoodieWriteConflictException.ConflictCategory.TABLE_SERVICE_VS_INGESTION)); + assertDoesNotThrow(() -> metricsOff.emitConflictResolutionByCategory( + HoodieWriteConflictException.ConflictCategory.TABLE_SERVICE_VS_TABLE_SERVICE)); + } + + // ----------------------------------------------------------------------- + // Conflict resolution counters and timer + // ----------------------------------------------------------------------- + + @Test + public void testConflictResolutionSuccessAndFailureCounters() { + when(writeConfig.isLockingMetricsEnabled()).thenReturn(true); + + String successName = hoodieMetrics.getMetricsName(HoodieMetrics.CONFLICT_RESOLUTION_STR, HoodieMetrics.SUCCESS_COUNTER); + String failureName = hoodieMetrics.getMetricsName(HoodieMetrics.CONFLICT_RESOLUTION_STR, HoodieMetrics.FAILURE_COUNTER); + + hoodieMetrics.emitConflictResolutionSuccessful(); + hoodieMetrics.emitConflictResolutionSuccessful(); + assertEquals(2, metrics.getRegistry().getCounters().get(successName).getCount()); + + hoodieMetrics.emitConflictResolutionFailed(); + assertEquals(1, metrics.getRegistry().getCounters().get(failureName).getCount()); + } + + @Test + public void testConflictResolutionTimerCtx() throws InterruptedException { + when(writeConfig.isLockingMetricsEnabled()).thenReturn(true); + + Timer.Context ctx = hoodieMetrics.getConflictResolutionCtx(); + assertNotNull(ctx); + Thread.sleep(5); + assertTrue(hoodieMetrics.getDurationInMs(ctx.stop()) > 0); + } + + // ----------------------------------------------------------------------- + // Compaction counters + // ----------------------------------------------------------------------- + + @Test + public void testCompactionCounters() { + String requestedName = hoodieMetrics.getMetricsName( + HoodieTimeline.COMPACTION_ACTION, + HoodieTimeline.REQUESTED_COMPACTION_SUFFIX + HoodieMetrics.COUNTER_METRIC_EXTENSION); + String completedName = hoodieMetrics.getMetricsName( + HoodieTimeline.COMPACTION_ACTION, + HoodieTimeline.COMPLETED_COMPACTION_SUFFIX + HoodieMetrics.COUNTER_METRIC_EXTENSION); + + hoodieMetrics.emitCompactionRequested(); + hoodieMetrics.emitCompactionRequested(); + assertEquals(2, metrics.getRegistry().getCounters().get(requestedName).getCount()); + + hoodieMetrics.emitCompactionCompleted(); + assertEquals(1, metrics.getRegistry().getCounters().get(completedName).getCount()); + } + + // ----------------------------------------------------------------------- + // Archive metrics + // ----------------------------------------------------------------------- + + @Test + public void testArchiveTimerAndMetrics() throws InterruptedException { + Timer.Context ctx = hoodieMetrics.getArchiveCtx(); + assertNotNull(ctx); + Thread.sleep(5); + int numInstantsArchived = 7; + hoodieMetrics.updateArchiveMetrics(hoodieMetrics.getDurationInMs(ctx.stop()), numInstantsArchived); + + String durationName = hoodieMetrics.getMetricsName(HoodieMetrics.ARCHIVE_ACTION, HoodieMetrics.DURATION_STR); + String countName = hoodieMetrics.getMetricsName(HoodieMetrics.ARCHIVE_ACTION, HoodieMetrics.DELETE_INSTANTS_NUM_STR); + assertTrue((Long) metrics.getRegistry().getGauges().get(durationName).getValue() > 0); + assertEquals(numInstantsArchived, (long) metrics.getRegistry().getGauges().get(countName).getValue()); + } + + @Test + public void testUpdateArchivalMetrics() { + Map archivalMetrics = new HashMap<>(); + archivalMetrics.put("numFilesArchived", 10L); + archivalMetrics.put("archiveDurationMs", 250L); + + hoodieMetrics.updateArchivalMetrics(archivalMetrics); + + assertEquals(10L, (long) metrics.getRegistry().getGauges().get( + hoodieMetrics.getMetricsName("archival", "numFilesArchived")).getValue()); + assertEquals(250L, (long) metrics.getRegistry().getGauges().get( + hoodieMetrics.getMetricsName("archival", "archiveDurationMs")).getValue()); + } + + // ----------------------------------------------------------------------- + // Post-commit metrics + // ----------------------------------------------------------------------- + + @Test + public void testPostCommitMetrics() { + String successName = hoodieMetrics.getMetricsName(HoodieMetrics.POST_COMMIT_STR, HoodieMetrics.SUCCESS_COUNTER); + String failureName = hoodieMetrics.getMetricsName(HoodieMetrics.POST_COMMIT_STR, HoodieMetrics.FAILURE_COUNTER); + String durationName = hoodieMetrics.getMetricsName(HoodieMetrics.POST_COMMIT_STR, HoodieMetrics.DURATION_STR); + + hoodieMetrics.updatePostCommitMetrics(true, 100L); + hoodieMetrics.updatePostCommitMetrics(true, 200L); + hoodieMetrics.updatePostCommitMetrics(false, 50L); + + assertEquals(2, metrics.getRegistry().getCounters().get(successName).getCount()); + assertEquals(1, metrics.getRegistry().getCounters().get(failureName).getCount()); + assertEquals(50L, (long) metrics.getRegistry().getGauges().get(durationName).getValue()); + } + + // ----------------------------------------------------------------------- + // reportMetrics / updateClusteringFileCreationMetrics + // ----------------------------------------------------------------------- + + @Test + public void testReportMetrics() { + hoodieMetrics.reportMetrics("commit", "customMetric", 42L); + String metricName = hoodieMetrics.getMetricsName("commit", "customMetric"); + assertEquals(42L, (long) metrics.getRegistry().getGauges().get(metricName).getValue()); + } + + @Test + public void testUpdateClusteringFileCreationMetrics() { + hoodieMetrics.updateClusteringFileCreationMetrics(150L); + String metricName = hoodieMetrics.getMetricsName(HoodieTimeline.CLUSTERING_ACTION, "fileCreationTime"); + assertEquals(150L, (long) metrics.getRegistry().getGauges().get(metricName).getValue()); + } + + // ----------------------------------------------------------------------- + // Log-compaction and clustering timer contexts + // ----------------------------------------------------------------------- + + @Test + public void testLogCompactionTimerCtx() throws InterruptedException { + Timer.Context ctx = hoodieMetrics.getLogCompactionCtx(); + assertNotNull(ctx); + Thread.sleep(5); + assertTrue(hoodieMetrics.getDurationInMs(ctx.stop()) > 0); + } + + @Test + public void testClusteringTimerCtx() throws InterruptedException { + Timer.Context ctx = hoodieMetrics.getClusteringCtx(); + assertNotNull(ctx); + Thread.sleep(5); + assertTrue(hoodieMetrics.getDurationInMs(ctx.stop()) > 0); + } + + @Test + public void testCleanTimerCtx() throws InterruptedException { + Timer.Context ctx = hoodieMetrics.getCleanCtx(); + assertNotNull(ctx); + Thread.sleep(5); + assertTrue(hoodieMetrics.getDurationInMs(ctx.stop()) > 0); + } + + // ----------------------------------------------------------------------- + // Version metrics + // ----------------------------------------------------------------------- + + @Test + public void testVersionMetrics() { + hoodieMetrics.emitVersionMetrics(); + assertTrue(metrics.getRegistry().getGauges().keySet().stream() + .anyMatch(name -> name.startsWith("version."))); + } + + // ----------------------------------------------------------------------- + // Commit metrics with event time (latency / freshness) + // ----------------------------------------------------------------------- + + @Test + public void testCommitMetricsWithEventTime() { + long commitEpochTimeMs = System.currentTimeMillis(); + long durationMs = 1000L; + long minEventTimeMs = commitEpochTimeMs - 5000L; + long maxEventTimeMs = commitEpochTimeMs - 2000L; + + HoodieCommitMetadata metadata = mock(HoodieCommitMetadata.class); + when(metadata.fetchTotalPartitionsWritten()).thenReturn(1L); + when(metadata.fetchTotalFilesInsert()).thenReturn(0L); + when(metadata.fetchTotalFilesUpdated()).thenReturn(0L); + when(metadata.fetchTotalRecordsWritten()).thenReturn(10L); + when(metadata.fetchTotalUpdateRecordsWritten()).thenReturn(5L); + when(metadata.fetchTotalInsertRecordsWritten()).thenReturn(5L); + when(metadata.fetchTotalBytesWritten()).thenReturn(1024L); + when(metadata.getTotalScanTime()).thenReturn(0L); + when(metadata.getTotalCreateTime()).thenReturn(0L); + when(metadata.getTotalUpsertTime()).thenReturn(0L); + when(metadata.getTotalCompactedRecordsUpdated()).thenReturn(0L); + when(metadata.getTotalLogFilesCompacted()).thenReturn(0L); + when(metadata.getTotalLogFilesSize()).thenReturn(0L); + when(metadata.getTotalRecordsDeleted()).thenReturn(0L); + when(metadata.getMinAndMaxEventTime()).thenReturn(Pair.of(Option.of(minEventTimeMs), Option.of(maxEventTimeMs))); + when(writeConfig.isCompactionLogBlockMetricsOn()).thenReturn(false); + + hoodieMetrics.updateCommitMetrics(commitEpochTimeMs, durationMs, metadata, "commit"); + + long expectedLatency = commitEpochTimeMs + durationMs - minEventTimeMs; + long expectedFreshness = commitEpochTimeMs + durationMs - maxEventTimeMs; + assertEquals(expectedLatency, (long) metrics.getRegistry().getGauges().get( + hoodieMetrics.getMetricsName("commit", HoodieMetrics.COMMIT_LATENCY_IN_MS_STR)).getValue()); + assertEquals(expectedFreshness, (long) metrics.getRegistry().getGauges().get( + hoodieMetrics.getMetricsName("commit", HoodieMetrics.COMMIT_FRESHNESS_IN_MS_STR)).getValue()); + } + + // ----------------------------------------------------------------------- + // Rollback failure with null exception type + // ----------------------------------------------------------------------- + + @Test + public void testEmitRollbackFailureWithNullExceptionType() { + hoodieMetrics.emitRollbackFailure(null); + + String failureName = hoodieMetrics.getMetricsName("rollback", FAILURE_COUNTER); + assertEquals(1, metrics.getRegistry().getCounters().get(failureName).getCount()); + // No per-exception counter should be registered + long exceptionCounters = metrics.getRegistry().getCounters().keySet().stream() + .filter(n -> n.startsWith(hoodieMetrics.getMetricsName("rollback", "")) && !n.equals(failureName)) + .count(); + assertEquals(0, exceptionCounters); + } + + // ----------------------------------------------------------------------- + // getMetricsName prefix handling + // ----------------------------------------------------------------------- + + @Test + public void testGetMetricsNameWithPrefix() { + when(writeConfig.getMetricReporterMetricsNamePrefix()).thenReturn("my_prefix"); + assertEquals("my_prefix.action.metric", hoodieMetrics.getMetricsName("action", "metric")); + } + + @Test + public void testGetMetricsNameWithoutPrefix() { + when(writeConfig.getMetricReporterMetricsNamePrefix()).thenReturn(""); + assertEquals("action.metric", hoodieMetrics.getMetricsName("action", "metric")); + } + + // ----------------------------------------------------------------------- + // Existing rollback-failure and conflict-resolution-by-category tests + // ----------------------------------------------------------------------- + @Test public void testRollbackFailureMetric() { // Test that rollback failure metric is emitted correctly diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/metrics/HoodieMetaSyncMetrics.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/metrics/HoodieMetaSyncMetrics.java index c21adf7bc8124..c0ea52b475758 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/metrics/HoodieMetaSyncMetrics.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/metrics/HoodieMetaSyncMetrics.java @@ -78,6 +78,9 @@ private Timer createTimer(String name) { } public void incrementRecreateAndSyncFailureCounter() { + if (!metricsConfig.isMetricsOn()) { + return; + } recreateAndSyncFailureCounter = getCounter(recreateAndSyncFailureCounter, recreateAndSyncFailureCounterName); recreateAndSyncFailureCounter.inc(); } From e142c4054be965a524e4137d8913e99f5f79406a Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Wed, 10 Jun 2026 11:53:56 +0800 Subject: [PATCH 055/255] feat(flink): Backport Flink 2.1 Dremel nested Parquet reader to hudi-flink1.20 (#18953) (cherry picked from commit 86d1650d0f1915d710b4690cf461639e83626ebc) --- .../format/cow/ParquetSplitReaderUtil.java | 682 +++++++++++------- .../format/cow/utils/BooleanArrayList.java | 69 ++ .../table/format/cow/utils/IntArrayList.java | 93 +++ .../table/format/cow/utils/LongArrayList.java | 83 +++ .../format/cow/utils/NestedPositionUtil.java | 209 ++++++ .../cow/vector/ColumnarGroupArrayData.java | 179 ----- .../cow/vector/ColumnarGroupMapData.java | 63 -- .../cow/vector/ColumnarGroupRowData.java | 138 ---- .../vector/HeapArrayGroupColumnVector.java | 53 -- .../format/cow/vector/HeapArrayVector.java | 31 + .../cow/vector/HeapMapColumnVector.java | 74 +- .../cow/vector/HeapRowColumnVector.java | 15 + .../cow/vector/ParquetDecimalVector.java | 197 ++++- .../vector/position/CollectionPosition.java | 57 ++ .../cow/vector/position/LevelDelegation.java | 43 ++ .../cow/vector/position/RowPosition.java | 45 ++ .../cow/vector/reader/ArrayColumnReader.java | 473 ------------ .../cow/vector/reader/ArrayGroupReader.java | 45 -- .../cow/vector/reader/MapColumnReader.java | 55 -- .../cow/vector/reader/NestedColumnReader.java | 277 +++++++ .../reader/NestedPrimitiveColumnReader.java | 638 ++++++++++++++++ .../reader/ParquetColumnarRowSplitReader.java | 34 +- .../ParquetDataColumnReaderFactory.java | 112 ++- .../cow/vector/reader/RowColumnReader.java | 63 -- .../format/cow/vector/type/ParquetField.java | 72 ++ .../cow/vector/type/ParquetGroupField.java | 59 ++ .../vector/type/ParquetPrimitiveField.java | 55 ++ .../vector/TestHeapColumnVectorAccessors.java | 138 ++++ .../cow/vector/TestParquetDecimalVector.java | 187 +++++ .../TestParquetDataColumnReaderFactory.java | 270 +++++++ .../vector/type/TestParquetGroupField.java | 132 ++++ 31 files changed, 3267 insertions(+), 1374 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index 2bb5be1d9614e..7b2bb0fa55f8e 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,19 +19,18 @@ package org.apache.hudi.table.format.cow; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapDecimalVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; -import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader; import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader; import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader; import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader; +import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader; import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader; -import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader; @@ -64,12 +63,13 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; @@ -77,12 +77,18 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.io.ColumnIO; +import org.apache.parquet.io.GroupColumnIO; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.sql.Date; @@ -90,25 +96,38 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.flink.table.utils.DateTimeUtils.toInternal; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** * Util for generating {@link ParquetColumnarRowSplitReader}. * - *

    NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64 - * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that. + *

    Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive + * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader}, + * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For + * nested types (ARRAY / MAP / MULTISET / ROW) we build a {@link ParquetField} tree once per + * split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to + * {@link NestedColumnReader}. + * + *

    Schema evolution: missing top-level fields are still handled by the caller + * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields + * inside a Row are handled here — {@link #constructField} returns {@code null} for a + * child that isn't physically present, and the corresponding child in the pre-allocated vector + * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can + * passthrough the slot (see {@link NestedColumnReader#readToVector}). */ public class ParquetSplitReaderUtil { - /** - * Util for generating partitioned {@link ParquetColumnarRowSplitReader}. - */ + /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( boolean utcTimestamp, boolean caseSensitive, @@ -182,10 +201,13 @@ private static ColumnVector createVector( return readVector; } - private static ColumnVector createVectorFromConstant( - LogicalType type, - Object value, - int batchSize) { + /** + * Builds a constant-filled column vector for either a partition column (non-null value) or a + * missing-column slot (null value). Used both at the batch-generator level for partition + * injection and at the row-reader level for fields absent from the Parquet file. + */ + public static ColumnVector createVectorFromConstant( + LogicalType type, Object value, int batchSize) { switch (type.getTypeRoot()) { case CHAR: case VARCHAR: @@ -278,6 +300,7 @@ private static ColumnVector createVectorFromConstant( value == null ? null : toInternal((Date) value), batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: HeapTimestampVector tv = new HeapTimestampVector(batchSize); if (value == null) { tv.fillWithNulls(); @@ -286,46 +309,41 @@ private static ColumnVector createVectorFromConstant( } return tv; case ARRAY: - ArrayType arrayType = (ArrayType) type; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize); - if (value == null) { - arrayGroup.fillWithNulls(); - return arrayGroup; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } - } else { - HeapArrayVector arrayVector = new HeapArrayVector(batchSize); - if (value == null) { - arrayVector.fillWithNulls(); - return arrayVector; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } + if (value != null) { + throw new UnsupportedOperationException("Unsupported create array with default value."); } + HeapArrayVector arrayVector = new HeapArrayVector(batchSize); + arrayVector.fillWithNulls(); + return arrayVector; case MAP: - HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); - if (value == null) { - mapVector.fillWithNulls(); - return mapVector; - } else { - throw new UnsupportedOperationException("Unsupported create map with default value."); + case MULTISET: + if (value != null) { + throw new UnsupportedOperationException( + "Unsupported create " + type.getTypeRoot() + " with default value."); } + HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); + mapVector.fillWithNulls(); + return mapVector; case ROW: - HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize); - if (value == null) { - rowVector.fillWithNulls(); - return rowVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create row with default value."); } + RowType rowType = (RowType) type; + WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()]; + for (int i = 0; i < childVectors.length; i++) { + childVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); + } + HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors); + rowVector.fillWithNulls(); + return rowVector; default: throw new UnsupportedOperationException("Unsupported type: " + type); } } - private static List filterDescriptors(int depth, Type type, List columns) throws ParquetRuntimeException { + private static List filterDescriptors( + int depth, Type type, List columns) throws ParquetRuntimeException { List filtered = new ArrayList<>(); for (ColumnDescriptor descriptor : columns) { if (depth >= descriptor.getPath().length) { @@ -339,24 +357,61 @@ private static List filterDescriptors(int depth, Type type, Li return filtered; } + /** + * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the + * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP}, + * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by + * the supplied pre-built {@link ParquetField} tree. + * + * @param field the {@link ParquetField} tree for this column, built by + * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested + * types; ignored for primitives. + */ + public static ColumnReader createColumnReader( + boolean utcTimestamp, + LogicalType fieldType, + Type physicalType, + List descriptors, + PageReadStore pages, + @Nullable ParquetField field) throws IOException { + switch (fieldType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + Preconditions.checkNotNull( + field, "ParquetField must be provided for nested type: %s", fieldType); + return new NestedColumnReader(utcTimestamp, pages, field); + default: + return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages); + } + } + + /** + * Backward-compat entry point kept for callers that don't project nested types and therefore + * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload + * with a null field; nested types now go through that overload directly. + * + * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore, + * ParquetField)} so nested types take the Dremel path. + */ + @Deprecated public static ColumnReader createColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List descriptors, PageReadStore pages) throws IOException { - return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, - pages, 0); + return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null); } - private static ColumnReader createColumnReader( + private static ColumnReader createPrimitiveColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List columns, - PageReadStore pages, - int depth) throws IOException { - List descriptors = filterDescriptors(depth, physicalType, columns); + PageReadStore pages) throws IOException { + List descriptors = filterDescriptors(0, physicalType, columns); ColumnDescriptor descriptor = descriptors.get(0); PageReader pageReader = pages.getPageReader(descriptor); switch (fieldType.getTypeRoot()) { @@ -392,7 +447,9 @@ private static ColumnReader createColumnReader( case INT96: return new TimestampColumnReader(utcTimestamp, descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for TIMESTAMP: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } case DECIMAL: switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { @@ -403,106 +460,23 @@ private static ColumnReader createColumnReader( case BINARY: return new BytesColumnReader(descriptor, pageReader); case FIXED_LEN_BYTE_ARRAY: - return new FixedLenBytesColumnReader( - descriptor, pageReader); + return new FixedLenBytesColumnReader(descriptor, pageReader); default: - throw new AssertionError(); - } - case ARRAY: - ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new ArrayGroupReader(createColumnReader( - utcTimestamp, - arrayType.getElementType(), - elementType, - descriptors, - pages, - elementDepth)); - } else { - return new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - fieldType); + throw new AssertionError( + "Unexpected physical type for DECIMAL: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } - case MAP: - MapType mapType = (MapType) fieldType; - ArrayColumnReader keyReader = - new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - new ArrayType(mapType.getKeyType())); - ColumnReader valueReader; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueReader = new ArrayGroupReader(createColumnReader( - utcTimestamp, - mapType.getValueType(), - physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type - descriptors.subList(1, descriptors.size()), // remove the key descriptor - pages, - depth + 2)); // increase the depth by 2, because there's a key_value entry in the path - } else { - valueReader = new ArrayColumnReader( - descriptors.get(1), - pages.getPageReader(descriptors.get(1)), - utcTimestamp, - descriptors.get(1).getPrimitiveType(), - new ArrayType(mapType.getValueType())); - } - return new MapColumnReader(keyReader, valueReader); - case ROW: - RowType rowType = (RowType) fieldType; - GroupType groupType = physicalType.asGroupType(); - List fieldReaders = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - // schema evolution: read the parquet file with a new extended field name. - int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); - if (fieldIndex < 0) { - fieldReaders.add(new EmptyColumnReader()); - } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - fieldReaders.add( - createColumnReader( - utcTimestamp, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } else { - fieldReaders.add( - createColumnReader( - utcTimestamp, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } - } - } - return new RowColumnReader(fieldReaders); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + /** + * Creates the writable column vector that the reader will write into. The returned vector shape + * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled + * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in + * {@link NestedColumnReader} can pass them through unchanged. + */ public static WritableColumnVector createWritableColumnVector( int batchSize, LogicalType fieldType, @@ -523,33 +497,40 @@ private static WritableColumnVector createWritableColumnVector( switch (fieldType.getTypeRoot()) { case BOOLEAN: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBooleanVector(batchSize); case TINYINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapByteVector(batchSize); case DOUBLE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDoubleVector(batchSize); case FLOAT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.FLOAT, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapFloatVector(batchSize); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapIntVector(batchSize); case BIGINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT64, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapLongVector(batchSize); case SMALLINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapShortVector(batchSize); case CHAR: case VARCHAR: @@ -566,112 +547,64 @@ private static WritableColumnVector createWritableColumnVector( case DECIMAL: checkArgument( (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY - || typeName == PrimitiveType.PrimitiveTypeName.BINARY) + || typeName == PrimitiveType.PrimitiveTypeName.BINARY) && primitiveType.getOriginalType() == OriginalType.DECIMAL, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDecimalVector(batchSize); case ARRAY: ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - elementType, - descriptors, - elementDepth)); - } else { - return new HeapArrayVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - physicalType, - descriptors, - depth)); - } - case MAP: + return new HeapArrayVector( + batchSize, + createWritableColumnVector( + batchSize, arrayType.getElementType(), physicalType, descriptors, depth)); + case MAP: { MapType mapType = (MapType) fieldType; - GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType(); - // the map column has three level paths. - WritableColumnVector keyColumnVector = createWritableColumnVector( + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( batchSize, - new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()), - repeatedType.getType(0), - descriptors, - depth + 2); - WritableColumnVector valueColumnVector; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueColumnVector = new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - mapType.getValueType(), - repeatedType.getType(1).asGroupType(), - descriptors, - depth + 2)); - } else { - valueColumnVector = createWritableColumnVector( - batchSize, - new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()), - repeatedType.getType(1), - descriptors, - depth + 2); - } - return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector); + createWritableColumnVector( + batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2), + createWritableColumnVector( + batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2)); + } + case MULTISET: { + MultisetType multisetType = (MultisetType) fieldType; + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( + batchSize, + createWritableColumnVector( + batchSize, + multisetType.getElementType(), + repeatedType.getType(0), + descriptors, + depth + 2), + createWritableColumnVector( + batchSize, + new IntType(false), + repeatedType.getType(1), + descriptors, + depth + 2)); + } case ROW: RowType rowType = (RowType) fieldType; GroupType groupType = physicalType.asGroupType(); WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()]; for (int i = 0; i < columnVectors.length; i++) { - // schema evolution: read the file with a new extended field name. int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); if (fieldIndex < 0) { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant( - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize); - } else { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); - } + // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled + // vector of the correct shape; NestedColumnReader.readRow will pass it through when the + // matching ParquetField child is null. + columnVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = - createWritableColumnVector( - batchSize, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } else { - columnVectors[i] = - createWritableColumnVector( - batchSize, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } + columnVectors[i] = + createWritableColumnVector( + batchSize, + rowType.getTypeAt(i), + groupType.getType(fieldIndex), + descriptors, + depth + 1); } } return new HeapRowColumnVector(batchSize, columnVectors); @@ -681,56 +614,245 @@ private static WritableColumnVector createWritableColumnVector( } /** - * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. - * - * @return The physical field index or -1 if the field does not exist + * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching + * Parquet's canonical 3-level map encoding. */ - private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { - // get index from fileSchema type, else, return -1 - return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + private static GroupType unwrapMapRepeatedType(Type physicalType) { + return physicalType.asGroupType().getType(0).asGroupType(); } + // ------------------------------------------------------------------------------------------ + // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil) + // + // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children + // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This + // matches the Hudi schema-evolution contract and is the companion to the null-child branch in + // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`. + // ------------------------------------------------------------------------------------------ + /** - * Check whether the given list type is a three-level list type. - *

    - * group (LIST) { - * repeated group list { - * element; - * } - * } - * - * @param type list type - * @return true if the list type is a three-level list type + * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed + * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally; + * primitive top-level fields produce {@code null} entries (callers don't need a tree for those). */ - private static boolean isThreeLevelList(Type type) { - if (type.isPrimitive()) { - return false; + public static List buildFieldsList( + List children, List fieldNames, MessageColumnIO columnIO) { + List list = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + RowType.RowField child = children.get(i); + if (isNestedType(child.getType())) { + list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i)))); + } else { + list.add(null); + } } - GroupType groupType = type.asGroupType(); - OriginalType originalType = groupType.getOriginalType(); - return originalType == OriginalType.LIST - && groupType.getType(0).getName().equals("list"); + return list; + } + + private static boolean isNestedType(LogicalType type) { + return type instanceof RowType + || type instanceof ArrayType + || type instanceof MapType + || type instanceof MultisetType; + } + + @Nullable + private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) { + boolean required = columnIO.getType().getRepetition() == REQUIRED; + int repetitionLevel = columnIO.getRepetitionLevel(); + int definitionLevel = columnIO.getDefinitionLevel(); + LogicalType type = rowField.getType(); + String fieldName = rowField.getName(); + if (type instanceof RowType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + RowType rowType = (RowType) type; + List childFields = rowType.getFields(); + List fieldsList = new ArrayList<>(childFields.size()); + for (RowType.RowField childField : childFields) { + // Hudi schema evolution: a logical child may be absent from the Parquet file. In that + // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the + // pre-filled null vector instead of recursing. + ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName()); + if (childIo == null) { + fieldsList.add(null); + } else { + fieldsList.add(constructField(childField, childIo)); + } + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(fieldsList)); + } + + if (type instanceof MapType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MapType mapType = (MapType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof MultisetType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MultisetType multisetType = (MultisetType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", multisetType.getElementType()), + keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof ArrayType) { + ArrayType arrayType = (ArrayType) type; + ColumnIO elementTypeColumnIO; + if (columnIO instanceof GroupColumnIO) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { + while (!Objects.equals(groupColumnIO.getName(), fieldName)) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + elementTypeColumnIO = groupColumnIO; + } else { + if (arrayType.getElementType() instanceof RowType) { + elementTypeColumnIO = groupColumnIO; + } else { + elementTypeColumnIO = groupColumnIO.getChild(0); + } + } + } else if (columnIO instanceof PrimitiveColumnIO) { + elementTypeColumnIO = columnIO; + } else { + throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); + } + + ParquetField elementField = + constructField( + new RowType.RowField("", arrayType.getElementType()), + getArrayElementColumn(elementTypeColumnIO)); + if (repetitionLevel == elementField.getRepetitionLevel()) { + repetitionLevel = columnIO.getParent().getRepetitionLevel(); + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.singletonList(elementField)); + } + + PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO; + return new ParquetPrimitiveField( + type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId()); } /** - * Construct the error message when primitive type mismatches. - * - * @param primitiveType Primitive type - * @param fieldType Logical field type - * @return The error message + * Parquet column names are case-insensitive in Flink's lookup. Matches upstream + * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent. */ - private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { - return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name()); + public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName); + if (columnIO != null) { + return columnIO; + } + throw new FlinkRuntimeException( + "Can not find column io for parquet reader. Column name: " + columnName); } /** - * Construct the error message when original type mismatches. + * Case-insensitive column lookup that returns {@code null} when no match is found — the + * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to + * emit null {@link ParquetField} children for fields absent from the Parquet file. + */ + @Nullable + private static ColumnIO lookupColumnByNameOrNull( + GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = groupColumnIO.getChild(columnName); + if (columnIO != null) { + return columnIO; + } + for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) { + if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) { + return groupColumnIO.getChild(i); + } + } + return null; + } + + public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { + while (groupColumnIO.getChildrenCount() == 1) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + return groupColumnIO; + } + + public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { + while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { + columnIO = ((GroupColumnIO) columnIO).getChild(0); + } + + // Three-level list: skip the synthetic `element` / `list` wrapper when present. + if (columnIO instanceof GroupColumnIO + && columnIO.getType().getLogicalTypeAnnotation() == null + && ((GroupColumnIO) columnIO).getChildrenCount() == 1 + && !columnIO.getName().equals("array") + && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { + return ((GroupColumnIO) columnIO).getChild(0); + } + return columnIO; + } + + /** + * Returns the field index with given physical row type {@code groupType} and field name + * {@code fieldName}. * - * @param originalType Original type - * @param fieldType Logical field type - * @return The error message + * @return the physical field index or -1 if the field does not exist + */ + private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { + return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + } + + private static String getPrimitiveTypeCheckFailureMessage( + PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Primitive type: %s. Field type: %s.", + primitiveType, fieldType.getTypeRoot().name()); + } + + private static String getOriginalTypeCheckFailureMessage( + OriginalType originalType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Original type: %s. Field type: %s.", + originalType, fieldType.getTypeRoot().name()); + } + + /** + * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience + * for callers that need to mirror Hudi's original behaviour where a missing column produces an + * explicit null-valued reader rather than being omitted from the batch. */ - private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) { - return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name()); + public static ColumnReader emptyColumnReader() { + return new EmptyColumnReader(); } } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java new file mode 100644 index 0000000000000..d51d7ee754b8a --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of booleans. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper. + */ +public class BooleanArrayList { + private int size; + private boolean[] array; + + public BooleanArrayList(int capacity) { + this.size = 0; + this.array = new boolean[capacity]; + } + + public int size() { + return size; + } + + public boolean add(boolean element) { + grow(size + 1); + array[size++] = element; + return true; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public boolean[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final boolean[] t = new boolean[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java new file mode 100644 index 0000000000000..4787dbb5b9ddb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; +import java.util.NoSuchElementException; + +/** + * Minimal implementation of an array-backed list of ints. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class IntArrayList { + + private int size; + private int[] array; + + public IntArrayList(final int capacity) { + this.size = 0; + this.array = new int[capacity]; + } + + public int size() { + return size; + } + + public boolean add(final int number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public int removeLast() { + if (size == 0) { + throw new NoSuchElementException(); + } + --size; + return array[size]; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return size == 0; + } + + private void grow(final int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final int[] t = new int[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } + + public int[] toArray() { + return Arrays.copyOf(array, size); + } + + public static final IntArrayList EMPTY = + new IntArrayList(0) { + + @Override + public boolean add(int number) { + throw new UnsupportedOperationException(); + } + + @Override + public int removeLast() { + throw new UnsupportedOperationException(); + } + }; +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java new file mode 100644 index 0000000000000..a51291f9d8441 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of longs. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class LongArrayList { + + private int size; + private long[] array; + + public LongArrayList(int capacity) { + this.size = 0; + this.array = new long[capacity]; + } + + public int size() { + return size; + } + + public boolean add(long number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public long removeLong(int index) { + if (index >= size) { + throw new IndexOutOfBoundsException( + "Index (" + index + ") is greater than or equal to list size (" + size + ")"); + } + final long old = array[index]; + size--; + if (index != size) { + System.arraycopy(array, index + 1, array, index, size - index); + } + return old; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public long[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final long[] t = new long[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java new file mode 100644 index 0000000000000..3f2f8976b69bf --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; + +import static java.lang.String.format; + +/** + * Utils to calculate nested type position. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.utils.NestedPositionUtil}). + */ +public class NestedPositionUtil { + + /** + * Calculate row offsets according to column's max repetition level, definition level, value's + * repetition level and definition level. Each row has three situation: + *

  • Row is not defined,because it's optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Row is null + *
  • Row is defined and not empty. + * + * @param field field that contains the row column message include max repetition level and + * definition level. + * @param fieldRepetitionLevels int array with each value's repetition level. + * @param fieldDefinitionLevels int array with each value's definition level. + * @return {@link RowPosition} contains collections row count and isNull array. + */ + public static RowPosition calculateRowOffsets( + ParquetField field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) { + int rowDefinitionLevel = field.getDefinitionLevel(); + int rowRepetitionLevel = field.getRepetitionLevel(); + int nullValuesCount = 0; + BooleanArrayList nullRowFlags = new BooleanArrayList(0); + for (int i = 0; i < fieldDefinitionLevels.length; i++) { + // If a row's last field is an array, the repetition levels for the array's items will + // be larger than the parent row's repetition level, so we need to skip those values. + if (fieldRepetitionLevels[i] > rowRepetitionLevel) { + continue; + } + + if (fieldDefinitionLevels[i] >= rowDefinitionLevel) { + // current row is defined and not empty + nullRowFlags.add(false); + } else { + // current row is null + nullRowFlags.add(true); + nullValuesCount++; + } + } + if (nullValuesCount == 0) { + return new RowPosition(null, fieldDefinitionLevels.length); + } + return new RowPosition(nullRowFlags.toArray(), nullRowFlags.size()); + } + + /** + * Calculate the collection's offsets according to column's max repetition level, definition + * level, value's repetition level and definition level. Each collection (Array or Map) has four + * situation: + *
  • Collection is not defined, because optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Collection is null + *
  • Collection is defined but empty + *
  • Collection is defined and not empty. In this case offset value is increased by the number + * of elements in that collection + * + * @param field field that contains array/map column message include max repetition level and + * definition level. + * @param definitionLevels int array with each value's definition level. + * @param repetitionLevels int array with each value's repetition level. + * @return {@link CollectionPosition} contains collections offset array, length array and isNull + * array. + */ + public static CollectionPosition calculateCollectionOffsets( + ParquetField field, int[] definitionLevels, int[] repetitionLevels) { + int collectionDefinitionLevel = field.getDefinitionLevel(); + int collectionRepetitionLevel = field.getRepetitionLevel() + 1; + int offset = 0; + int valueCount = 0; + LongArrayList offsets = new LongArrayList(0); + offsets.add(offset); + BooleanArrayList emptyCollectionFlags = new BooleanArrayList(0); + BooleanArrayList nullCollectionFlags = new BooleanArrayList(0); + int nullValuesCount = 0; + for (int i = 0; + i < definitionLevels.length; + i = getNextCollectionStartIndex(repetitionLevels, collectionRepetitionLevel, i)) { + valueCount++; + if (definitionLevels[i] >= collectionDefinitionLevel - 1) { + boolean isNull = + isOptionalFieldValueNull(definitionLevels[i], collectionDefinitionLevel); + nullCollectionFlags.add(isNull); + nullValuesCount += isNull ? 1 : 0; + // definitionLevels[i] > collectionDefinitionLevel => Collection is defined and not + // empty + // definitionLevels[i] == collectionDefinitionLevel => Collection is defined but + // empty + if (definitionLevels[i] > collectionDefinitionLevel) { + emptyCollectionFlags.add(false); + offset += getCollectionSize(repetitionLevels, collectionRepetitionLevel, i + 1); + } else if (definitionLevels[i] == collectionDefinitionLevel) { + offset++; + emptyCollectionFlags.add(true); + } else { + offset++; + emptyCollectionFlags.add(false); + } + offsets.add(offset); + } else { + // when definitionLevels[i] < collectionDefinitionLevel - 1, it means the collection + // is + // not defined, but we need to regard it as null to avoid getting value wrong. + nullCollectionFlags.add(true); + nullValuesCount++; + offsets.add(++offset); + emptyCollectionFlags.add(false); + } + } + long[] offsetsArray = offsets.toArray(); + long[] length = calculateLengthByOffsets(emptyCollectionFlags.toArray(), offsetsArray); + if (nullValuesCount == 0) { + return new CollectionPosition(null, offsetsArray, length, valueCount); + } + return new CollectionPosition( + nullCollectionFlags.toArray(), offsetsArray, length, valueCount); + } + + public static boolean isOptionalFieldValueNull(int definitionLevel, int maxDefinitionLevel) { + return definitionLevel == maxDefinitionLevel - 1; + } + + public static long[] calculateLengthByOffsets( + boolean[] collectionIsEmpty, long[] arrayOffsets) { + LongArrayList lengthList = new LongArrayList(arrayOffsets.length); + for (int i = 0; i < arrayOffsets.length - 1; i++) { + long offset = arrayOffsets[i]; + long length = arrayOffsets[i + 1] - offset; + if (length < 0) { + throw new IllegalArgumentException( + format( + "Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", + i, arrayOffsets[i], i + 1, arrayOffsets[i + 1])); + } + if (collectionIsEmpty[i]) { + length = 0; + } + lengthList.add(length); + } + return lengthList.toArray(); + } + + private static int getNextCollectionStartIndex( + int[] repetitionLevels, int maxRepetitionLevel, int elementIndex) { + do { + elementIndex++; + } while (hasMoreElements(repetitionLevels, elementIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, elementIndex)); + return elementIndex; + } + + /** This method is only called for non-empty collections. */ + private static int getCollectionSize( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + int size = 1; + while (hasMoreElements(repetitionLevels, nextIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, nextIndex)) { + // Collection elements cannot only be primitive, but also can have nested structure + // Counting only elements which belong to current collection, skipping inner elements of + // nested collections/structs + if (repetitionLevels[nextIndex] <= maxRepetitionLevel) { + size++; + } + nextIndex++; + } + return size; + } + + private static boolean isNotCollectionBeginningMarker( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + return repetitionLevels[nextIndex] >= maxRepetitionLevel; + } + + private static boolean hasMoreElements(int[] repetitionLevels, int nextIndex) { + return nextIndex < repetitionLevels.length; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java deleted file mode 100644 index 4c9275f3b0932..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupArrayData implements ArrayData { - - WritableColumnVector vector; - int rowId; - - public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) { - this.vector = vector; - this.rowId = rowId; - } - - @Override - public int size() { - if (vector == null) { - return 0; - } - - if (vector instanceof HeapRowColumnVector) { - // assume all fields have the same size - if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) { - return 0; - } - return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size(); - } - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean isNullAt(int index) { - if (vector == null) { - return true; - } - - if (vector instanceof HeapRowColumnVector) { - return ((HeapRowColumnVector) vector).vectors == null; - } - - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean getBoolean(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float getFloat(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public StringData getString(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public DecimalData getDecimal(int index, int precision, int scale) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public TimestampData getTimestamp(int index, int precision) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RawValueData getRawValue(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] getBinary(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public ArrayData getArray(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public MapData getMap(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RowData getRow(int index, int numFields) { - return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java deleted file mode 100644 index 69cb6feca13e4..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupMapData implements MapData { - - WritableColumnVector keyVector; - WritableColumnVector valueVector; - int rowId; - - public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) { - this.keyVector = keyVector; - this.valueVector = valueVector; - this.rowId = rowId; - } - - @Override - public int size() { - if (keyVector == null) { - return 0; - } - - if (keyVector instanceof HeapArrayVector) { - return ((HeapArrayVector) keyVector).getArray(rowId).size(); - } - throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector"); - } - - @Override - public ArrayData keyArray() { - return ((HeapArrayVector) keyVector).getArray(rowId); - } - - @Override - public ArrayData valueArray() { - if (valueVector instanceof HeapArrayVector) { - return ((HeapArrayVector) valueVector).getArray(rowId); - } else if (valueVector instanceof HeapArrayGroupColumnVector) { - return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId); - } - throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector"); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java deleted file mode 100644 index 439c1880823f1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.types.RowKind; - -public class ColumnarGroupRowData implements RowData { - - HeapRowColumnVector vector; - int rowId; - int index; - - public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) { - this.vector = vector; - this.rowId = rowId; - this.index = index; - } - - @Override - public int getArity() { - return vector.vectors.length; - } - - @Override - public RowKind getRowKind() { - return RowKind.INSERT; - } - - @Override - public void setRowKind(RowKind rowKind) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public boolean isNullAt(int pos) { - return - vector.vectors[pos].isNullAt(rowId) - || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index); - } - - @Override - public boolean getBoolean(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index); - } - - @Override - public byte getByte(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index); - } - - @Override - public short getShort(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index); - } - - @Override - public int getInt(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index); - } - - @Override - public long getLong(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index); - } - - @Override - public float getFloat(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index); - } - - @Override - public double getDouble(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index); - } - - @Override - public StringData getString(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index); - } - - @Override - public DecimalData getDecimal(int pos, int i1, int i2) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2); - } - - @Override - public TimestampData getTimestamp(int pos, int i1) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1); - } - - @Override - public RawValueData getRawValue(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index); - } - - @Override - public byte[] getBinary(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index); - } - - @Override - public ArrayData getArray(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index); - } - - @Override - public MapData getMap(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index); - } - - @Override - public RowData getRow(int pos, int numFields) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java deleted file mode 100644 index 3d7d8b1f0de0f..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -/** - * This class represents a nullable heap row column vector. - */ -public class HeapArrayGroupColumnVector extends AbstractHeapVector - implements WritableColumnVector, ArrayColumnVector { - - public WritableColumnVector vector; - - public HeapArrayGroupColumnVector(int len) { - super(len); - } - - public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) { - super(len); - this.vector = vector; - } - - @Override - public ArrayData getArray(int rowId) { - return new ColumnarGroupArrayData(vector, rowId); - } - - @Override - public void reset() { - super.reset(); - vector.reset(); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java index a0dced01e5e8d..2f21a323302f1 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java @@ -57,6 +57,37 @@ public int getLen() { return this.isNull.length; } + // --------------------------------------------------------------------------------------------- + // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths} + // and {@code child} fields so legacy callers continue to work; the new {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + // future Flink-2.1-style caller use these accessors. + // --------------------------------------------------------------------------------------------- + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public ColumnVector getChild() { + return child; + } + + public void setChild(ColumnVector child) { + this.child = child; + } + @Override public ArrayData getArray(int i) { long offset = offsets[i]; diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index 0d83f82baedf3..14aad22039e0a 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -20,29 +20,97 @@ import lombok.Getter; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.ColumnarMapData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; /** * This class represents a nullable heap map column vector. + * + *

    Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from + * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward + * compatibility: the {@code keys} / {@code values} fields are typed + * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing + * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original + * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use + * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}. */ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { @Getter - private final WritableColumnVector keys; + private WritableColumnVector keys; @Getter - private final WritableColumnVector values; + private WritableColumnVector values; + + // --------------------------------------------------------------------------------------------- + // Flink 2.1 Dremel-style state. Populated by {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and + // consumed by {@link #getMap(int)}. + // --------------------------------------------------------------------------------------------- + private long[] offsets; + private long[] lengths; + private int size; public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) { super(len); + this.offsets = new long[len]; + this.lengths = new long[len]; + this.keys = keys; + this.values = values; + } + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public void setKeys(WritableColumnVector keys) { this.keys = keys; + } + + public void setValues(WritableColumnVector values) { this.values = values; } + /** + * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract + * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}. + */ + public ColumnVector getKeyColumnVector() { + return keys; + } + + /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */ + public ColumnVector getValueColumnVector() { + return values; + } + @Override public MapData getMap(int rowId) { - return new ColumnarGroupMapData(keys, values, rowId); + long offset = offsets[rowId]; + long length = lengths[rowId]; + return new ColumnarMapData(keys, values, (int) offset, (int) length); } } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java index ae194e4e6ab05..0c640ce92ee40 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java @@ -37,6 +37,21 @@ public HeapRowColumnVector(int len, WritableColumnVector... vectors) { this.vectors = vectors; } + /** + * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code + * vectors} field so legacy callers continue to work; the new {@link + * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + * future Flink-2.1-style caller use this accessor. + */ + public WritableColumnVector[] getFields() { + return vectors; + } + + /** Counterpart of {@link #getFields()}. */ + public void setFields(WritableColumnVector[] fields) { + this.vectors = fields; + } + @Override public ColumnarRowData getRow(int i) { ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors)); diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java index 98b5e61050898..a37b88352cf52 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java @@ -18,21 +18,29 @@ package org.apache.hudi.table.format.cow.vector; +import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.columnar.vector.BytesColumnVector; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.DecimalColumnVector; +import org.apache.flink.table.data.columnar.vector.Dictionary; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.table.data.columnar.vector.LongColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector; + +import static org.apache.flink.util.Preconditions.checkArgument; /** - * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to - * provide {@link DecimalColumnVector} interface. - * - *

    Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector} - * because it is not public. + * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide + * {@link DecimalColumnVector} interface. */ -public class ParquetDecimalVector implements DecimalColumnVector { +public class ParquetDecimalVector + implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector { - public final ColumnVector vector; + private final ColumnVector vector; public ParquetDecimalVector(ColumnVector vector) { this.vector = vector; @@ -40,15 +48,180 @@ public ParquetDecimalVector(ColumnVector vector) { @Override public DecimalData getDecimal(int i, int precision, int scale) { - return DecimalData.fromUnscaledBytes( - ((BytesColumnVector) vector).getBytes(i).getBytes(), - precision, - scale); + if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) { + return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale); + } else if (ParquetSchemaConverter.is64BitDecimal(precision) + && vector instanceof LongColumnVector) { + return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale); + } else { + checkArgument( + vector instanceof BytesColumnVector, + "Reading decimal type occur unsupported vector type: %s", + vector.getClass()); + return DecimalData.fromUnscaledBytes( + ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale); + } + } + + public ColumnVector getVector() { + return vector; } @Override public boolean isNullAt(int i) { return vector.isNullAt(i); } -} + @Override + public void reset() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).reset(); + } + } + + @Override + public void setNullAt(int rowId) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNullAt(rowId); + } + } + + @Override + public void setNulls(int rowId, int count) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNulls(rowId, count); + } + } + + @Override + public void fillWithNulls() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).fillWithNulls(); + } + } + + @Override + public void setDictionary(Dictionary dictionary) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setDictionary(dictionary); + } + } + + @Override + public boolean hasDictionary() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).hasDictionary(); + } + return false; + } + + @Override + public WritableIntVector reserveDictionaryIds(int capacity) { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).reserveDictionaryIds(capacity); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public WritableIntVector getDictionaryIds() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).getDictionaryIds(); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public Bytes getBytes(int i) { + if (vector instanceof WritableBytesVector) { + return ((WritableBytesVector) vector).getBytes(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void appendBytes(int rowId, byte[] value, int offset, int length) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length); + } + } + + @Override + public void fill(byte[] value) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).fill(value); + } + } + + @Override + public int getInt(int i) { + if (vector instanceof WritableIntVector) { + return ((WritableIntVector) vector).getInt(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setInt(int rowId, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInt(rowId, value); + } + } + + @Override + public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void setInts(int rowId, int count, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, value); + } + } + + @Override + public void setInts(int rowId, int count, int[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).fill(value); + } + } + + @Override + public long getLong(int i) { + if (vector instanceof WritableLongVector) { + return ((WritableLongVector) vector).getLong(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setLong(int rowId, long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLong(rowId, value); + } + } + + @Override + public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).fill(value); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java new file mode 100644 index 0000000000000..fcdedfbc9d71d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent collection's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.CollectionPosition}). + */ +public class CollectionPosition { + @Nullable private final boolean[] isNull; + private final long[] offsets; + private final long[] length; + private final int valueCount; + + public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { + this.isNull = isNull; + this.offsets = offsets; + this.length = length; + this.valueCount = valueCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public long[] getOffsets() { + return offsets; + } + + public long[] getLength() { + return length; + } + + public int getValueCount() { + return valueCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java new file mode 100644 index 0000000000000..fe95419ac3218 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +/** + * To delegate repetition level and definition level. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.LevelDelegation}). + */ +public class LevelDelegation { + private final int[] repetitionLevel; + private final int[] definitionLevel; + + public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) { + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + } + + public int[] getRepetitionLevel() { + return repetitionLevel; + } + + public int[] getDefinitionLevel() { + return definitionLevel; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java new file mode 100644 index 0000000000000..5438b67973238 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent struct's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.RowPosition}). + */ +public class RowPosition { + @Nullable private final boolean[] isNull; + private final int positionsCount; + + public RowPosition(boolean[] isNull, int positionsCount) { + this.isNull = isNull; + this.positionsCount = positionsCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public int getPositionsCount() { + return positionsCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java deleted file mode 100644 index 6a8a01b74946a..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayVector; -import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; -import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.table.types.logical.ArrayType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Array {@link ColumnReader}. - */ -public class ArrayColumnReader extends BaseVectorizedColumnReader { - - // The value read in last time - private Object lastValue; - - // flag to indicate if there is no data in parquet data page - private boolean eof = false; - - // flag to indicate if it's the first time to read parquet data page with this instance - boolean isFirstRow = true; - - public ArrayColumnReader( - ColumnDescriptor descriptor, - PageReader pageReader, - boolean isUtcTimestamp, - Type type, - LogicalType logicalType) - throws IOException { - super(descriptor, pageReader, isUtcTimestamp, type, logicalType); - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayVector lcv = (HeapArrayVector) vector; - // before readBatch, initial the size of offsets & lengths as the default value, - // the actual size will be assigned in setChildrenInfo() after reading complete. - lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - // Because the length of ListColumnVector.child can't be known now, - // the valueList will save all data for ListColumnVector temporary. - List valueList = new ArrayList<>(); - - LogicalType category = ((ArrayType) logicalType).getElementType(); - - // read the first row in parquet data page, this will be only happened once for this - // instance - if (isFirstRow) { - if (!fetchNextValue(category)) { - return; - } - isFirstRow = false; - } - - int index = collectDataFromParquetPage(readNumber, lcv, valueList, category); - - // Convert valueList to array for the ListColumnVector.child - fillColumnVector(category, lcv, valueList, index); - } - - /** - * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating - * if there is more values to read (true). - * - * @param category - * @return boolean - * @throws IOException - */ - private boolean fetchNextValue(LogicalType category) throws IOException { - int left = readPageIfNeed(); - if (left > 0) { - // get the values of repetition and definitionLevel - readRepetitionAndDefinitionLevels(); - // read the data if it isn't null - if (definitionLevel == maxDefLevel) { - if (isCurrentPageDictionaryEncoded) { - lastValue = dataColumn.readValueDictionaryId(); - } else { - lastValue = readPrimitiveTypedRow(category); - } - } else { - lastValue = null; - } - return true; - } else { - eof = true; - return false; - } - } - - private int readPageIfNeed() throws IOException { - // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); - } - return leftInPage; - } - - // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper - // TODO Reduce the duplicated code - private Object readPrimitiveTypedRow(LogicalType category) { - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dataColumn.readString(); - case BOOLEAN: - return dataColumn.readBoolean(); - case TIME_WITHOUT_TIME_ZONE: - case DATE: - case INTEGER: - return dataColumn.readInteger(); - case TINYINT: - return dataColumn.readTinyInt(); - case SMALLINT: - return dataColumn.readSmallInt(); - case BIGINT: - return dataColumn.readLong(); - case FLOAT: - return dataColumn.readFloat(); - case DOUBLE: - return dataColumn.readDouble(); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dataColumn.readInteger(); - case INT64: - return dataColumn.readLong(); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return dataColumn.readString(); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dataColumn.readTimestamp(); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { - if (dictionaryValue == null) { - return null; - } - - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dictionary.readString(dictionaryValue); - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case INTEGER: - return dictionary.readInteger(dictionaryValue); - case BOOLEAN: - return dictionary.readBoolean(dictionaryValue) ? 1 : 0; - case DOUBLE: - return dictionary.readDouble(dictionaryValue); - case FLOAT: - return dictionary.readFloat(dictionaryValue); - case TINYINT: - return dictionary.readTinyInt(dictionaryValue); - case SMALLINT: - return dictionary.readSmallInt(dictionaryValue); - case BIGINT: - return dictionary.readLong(dictionaryValue); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dictionary.readInteger(dictionaryValue); - case INT64: - return dictionary.readLong(dictionaryValue); - case FIXED_LEN_BYTE_ARRAY: - case BINARY: - return dictionary.readString(dictionaryValue); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dictionary.readTimestamp(dictionaryValue); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - /** - * Collects data from a parquet page and returns the final row index where it stopped. The - * returned index can be equal to or less than total. - * - * @param total maximum number of rows to collect - * @param lcv column vector to do initial setup in data collection time - * @param valueList collection of values that will be fed into the vector later - * @param category - * @return int - * @throws IOException - */ - private int collectDataFromParquetPage( - int total, HeapArrayVector lcv, List valueList, LogicalType category) - throws IOException { - int index = 0; - /* - * Here is a nested loop for collecting all values from a parquet page. - * A column of array type can be considered as a list of lists, so the two loops are as below: - * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.: - * [0, 2, 3] <- index: 0 - * [NULL, 3, 4] <- index: 1 - * - * 2. The inner loop iterates on values within a row (sets all data from parquet data page - * for an element in ListColumnVector), so fetchNextValue returns values one-by-one: - * 0, 2, 3, NULL, 3, 4 - * - * As described below, the repetition level (repetitionLevel != 0) - * can be used to decide when we'll start to read values for the next list. - */ - while (!eof && index < total) { - // add element to ListColumnVector one by one - lcv.offsets[index] = valueList.size(); - /* - * Let's collect all values for a single list. - * Repetition level = 0 means that a new list started there in the parquet page, - * in that case, let's exit from the loop, and start to collect value for a new list. - */ - do { - /* - * Definition level = 0 when a NULL value was returned instead of a list - * (this is not the same as a NULL value in of a list). - */ - if (definitionLevel == 0) { - lcv.setNullAt(index); - } - valueList.add( - isCurrentPageDictionaryEncoded - ? dictionaryDecodeValue(category, (Integer) lastValue) - : lastValue); - } while (fetchNextValue(category) && (repetitionLevel != 0)); - - lcv.lengths[index] = valueList.size() - lcv.offsets[index]; - index++; - } - return index; - } - - /** - * The lengths & offsets will be initialized as default size (1024), it should be set to the - * actual size according to the element number. - */ - private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) { - lcv.setSize(itemNum); - long[] lcvLength = new long[elementNum]; - long[] lcvOffset = new long[elementNum]; - System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum); - System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum); - lcv.lengths = lcvLength; - lcv.offsets = lcvOffset; - } - - private void fillColumnVector( - LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) { - int total = valueList.size(); - setChildrenInfo(lcv, total, elementNum); - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - lcv.child = new HeapBytesVector(total); - ((HeapBytesVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (src == null) { - ((HeapBytesVector) lcv.child).setNullAt(i); - } else { - ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length); - } - } - break; - case BOOLEAN: - lcv.child = new HeapBooleanVector(total); - ((HeapBooleanVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapBooleanVector) lcv.child).setNullAt(i); - } else { - ((HeapBooleanVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TINYINT: - lcv.child = new HeapByteVector(total); - ((HeapByteVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapByteVector) lcv.child).setNullAt(i); - } else { - ((HeapByteVector) lcv.child).vector[i] = - (byte) ((List) valueList).get(i).intValue(); - } - } - break; - case SMALLINT: - lcv.child = new HeapShortVector(total); - ((HeapShortVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapShortVector) lcv.child).setNullAt(i); - } else { - ((HeapShortVector) lcv.child).vector[i] = - (short) ((List) valueList).get(i).intValue(); - } - } - break; - case INTEGER: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - lcv.child = new HeapIntVector(total); - ((HeapIntVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) lcv.child).setNullAt(i); - } else { - ((HeapIntVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case FLOAT: - lcv.child = new HeapFloatVector(total); - ((HeapFloatVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapFloatVector) lcv.child).setNullAt(i); - } else { - ((HeapFloatVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case BIGINT: - lcv.child = new HeapLongVector(total); - ((HeapLongVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) lcv.child).setNullAt(i); - } else { - ((HeapLongVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case DOUBLE: - lcv.child = new HeapDoubleVector(total); - ((HeapDoubleVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapDoubleVector) lcv.child).setNullAt(i); - } else { - ((HeapDoubleVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - lcv.child = new HeapTimestampVector(total); - ((HeapTimestampVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapTimestampVector) lcv.child).setNullAt(i); - } else { - ((HeapTimestampVector) lcv.child) - .setTimestamp(i, ((List) valueList).get(i)); - } - } - break; - case DECIMAL: - PrimitiveType.PrimitiveTypeName primitiveTypeName = - descriptor.getPrimitiveType().getPrimitiveTypeName(); - switch (primitiveTypeName) { - case INT32: - lcv.child = new ParquetDecimalVector(new HeapIntVector(total)); - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - case INT64: - lcv.child = new ParquetDecimalVector(new HeapLongVector(total)); - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - default: - lcv.child = new ParquetDecimalVector(new HeapBytesVector(total)); - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (valueList.get(i) == null) { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .appendBytes(i, src, 0, src.length); - } - } - break; - } - break; - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java deleted file mode 100644 index df7c5d85bc4ab..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}. - */ -public class ArrayGroupReader implements ColumnReader { - - private final ColumnReader fieldReader; - - public ArrayGroupReader(ColumnReader fieldReader) { - this.fieldReader = fieldReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector; - - fieldReader.readToVector(readNumber, rowColumnVector.vector); - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java deleted file mode 100644 index 6d743530fccc7..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Map {@link ColumnReader}. - */ -public class MapColumnReader implements ColumnReader { - - private final ArrayColumnReader keyReader; - private final ColumnReader valueReader; - - public MapColumnReader( - ArrayColumnReader keyReader, ColumnReader valueReader) { - this.keyReader = keyReader; - this.valueReader = valueReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector; - AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys()); - keyReader.readToVector(readNumber, mapColumnVector.getKeys()); - valueReader.readToVector(readNumber, mapColumnVector.getValues()); - for (int i = 0; i < keyArrayColumnVector.getLen(); i++) { - if (keyArrayColumnVector.isNullAt(i)) { - mapColumnVector.setNullAt(i); - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java new file mode 100644 index 0000000000000..9e49493cbed75 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.NestedPositionUtil; +import org.apache.hudi.table.format.cow.vector.HeapArrayVector; +import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; +import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.Preconditions; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code + * Row}). Resolves nested structures using Dremel striping/assembly; see the + * striping and assembly algorithms from the Dremel paper. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream: + * + *

      + *
    • Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code + * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code + * HeapMapVector}/{@code HeapArrayVector}. + *
    • Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a + * {@link RowType} may contain {@code null} children — meaning the corresponding logical + * field is absent from the Parquet file. Those slots are passed through unchanged and do + * not contribute to the row's repetition/definition-level stream. + *
    + */ +public class NestedColumnReader implements ColumnReader { + + private final Map columnReaders; + private final boolean isUtcTimestamp; + + private final PageReadStore pages; + + private final ParquetField field; + + public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) { + this.isUtcTimestamp = isUtcTimestamp; + this.pages = pages; + this.field = field; + this.columnReaders = new HashMap<>(); + } + + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + readData(field, readNumber, vector, false); + } + + private Tuple2 readData( + ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException { + if (field.getType() instanceof RowType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) { + return readMap((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof ArrayType) { + return readArray((ParquetGroupField) field, readNumber, vector, inside); + } else { + return readPrimitive((ParquetPrimitiveField) field, readNumber, vector); + } + } + + private Tuple2 readRow( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector; + LevelDelegation levelDelegation = null; + List children = field.getChildren(); + WritableColumnVector[] childrenVectors = heapRowVector.getFields(); + WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length]; + for (int i = 0; i < children.size(); i++) { + ParquetField child = children.get(i); + if (child == null) { + // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector + // (ROW branch); keep it as is and skip contributing to the level stream. + finalChildrenVectors[i] = childrenVectors[i]; + continue; + } + Tuple2 tuple = + readData(child, readNumber, childrenVectors[i], true); + levelDelegation = tuple.f0; + finalChildrenVectors[i] = tuple.f1; + } + if (levelDelegation == null) { + throw new FlinkRuntimeException( + String.format("Row field does not have any non-null children: %s.", field)); + } + + RowPosition rowPosition = + NestedPositionUtil.calculateRowOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If row was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors); + } else { + heapRowVector.setFields(finalChildrenVectors); + } + + if (rowPosition.getIsNull() != null) { + setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); + } + + // Hudi-specific: collapse a present row whose every child is null into a null row, so that a + // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the + // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely + // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the + // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + int rowCount = rowPosition.getPositionsCount(); + for (int j = 0; j < rowCount; j++) { + if (heapRowVector.isNullAt(j)) { + continue; + } + boolean allChildrenNull = true; + for (WritableColumnVector child : finalChildrenVectors) { + if (!child.isNullAt(j)) { + allChildrenNull = false; + break; + } + } + if (allChildrenNull) { + heapRowVector.setNullAt(j); + } + } + return Tuple2.of(levelDelegation, heapRowVector); + } + + private Tuple2 readMap( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapMapColumnVector mapVector = (HeapMapColumnVector) vector; + mapVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 2, + "Maps must have two type parameters, found %s", + children.size()); + Tuple2 keyTuple = + readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true); + Tuple2 valueTuple = + readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true); + + LevelDelegation levelDelegation = keyTuple.f0; + + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If map was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1); + } else { + mapVector.setKeys(keyTuple.f1); + mapVector.setValues(valueTuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), mapVector); + } + + mapVector.setLengths(collectionPosition.getLength()); + mapVector.setOffsets(collectionPosition.getOffsets()); + + return Tuple2.of(levelDelegation, mapVector); + } + + private Tuple2 readArray( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapArrayVector arrayVector = (HeapArrayVector) vector; + arrayVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 1, + "Arrays must have a single type parameter, found %s", + children.size()); + Tuple2 tuple = + readData(children.get(0), readNumber, arrayVector.getChild(), true); + + LevelDelegation levelDelegation = tuple.f0; + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If array was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1); + } else { + arrayVector.setChild(tuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), arrayVector); + } + arrayVector.setLengths(collectionPosition.getLength()); + arrayVector.setOffsets(collectionPosition.getOffsets()); + return Tuple2.of(levelDelegation, arrayVector); + } + + private Tuple2 readPrimitive( + ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException { + ColumnDescriptor descriptor = field.getDescriptor(); + NestedPrimitiveColumnReader reader = columnReaders.get(descriptor); + if (reader == null) { + reader = + new NestedPrimitiveColumnReader( + descriptor, + pages.getPageReader(descriptor), + isUtcTimestamp, + descriptor.getPrimitiveType(), + field.getType()); + columnReaders.put(descriptor, reader); + } + WritableColumnVector writableColumnVector = + reader.readAndNewVector(readNumber, (WritableColumnVector) vector); + return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); + } + + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { + for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { + if (nullFlags[index]) { + vector.setNullAt(index); + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java new file mode 100644 index 0000000000000..a18520c3b5cd5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -0,0 +1,638 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.IntArrayList; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; + +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; +import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; +import static org.apache.parquet.column.ValuesType.VALUES; + +/** + * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package + * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} + * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code + * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- + * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it + * a faithful copy of upstream. + */ +public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); + + private final IntArrayList repetitionLevelList = new IntArrayList(0); + private final IntArrayList definitionLevelList = new IntArrayList(0); + + private final PageReader pageReader; + private final ColumnDescriptor descriptor; + private final Type type; + private final LogicalType logicalType; + + /** The dictionary, if this column has dictionary encoding. */ + private final ParquetDataColumnReader dictionary; + + /** Maximum definition level for this column. */ + private final int maxDefLevel; + + private boolean isUtcTimestamp; + + /** Total number of values read. */ + private long valuesRead; + + /** + * value that indicates the end of the current page. That is, if valuesRead == + * endOfPageValueCount, we are at the end of the page. + */ + private long endOfPageValueCount; + + /** If true, the current page is dictionary encoded. */ + private boolean isCurrentPageDictionaryEncoded; + + private int definitionLevel; + private int repetitionLevel; + + /** Repetition/Definition/Value readers. */ + private IntIterator repetitionLevelColumn; + + private IntIterator definitionLevelColumn; + private ParquetDataColumnReader dataColumn; + + /** Total values in the current page. */ + private int pageValueCount; + + // flag to indicate if there is no data in parquet data page + private boolean eof = false; + + private boolean isFirstRow = true; + + private Object lastValue; + + public NestedPrimitiveColumnReader( + ColumnDescriptor descriptor, + PageReader pageReader, + boolean isUtcTimestamp, + Type parquetType, + LogicalType logicalType) + throws IOException { + this.descriptor = descriptor; + this.type = parquetType; + this.pageReader = pageReader; + this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.isUtcTimestamp = isUtcTimestamp; + this.logicalType = logicalType; + + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); + if (dictionaryPage != null) { + try { + this.dictionary = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + parquetType.asPrimitiveType(), + dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } catch (IOException e) { + throw new IOException( + String.format("Could not decode the dictionary for %s", descriptor), e); + } + } else { + this.dictionary = null; + this.isCurrentPageDictionaryEncoded = false; + } + } + + // Not invoked directly; callers use readAndNewVector instead. + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + throw new UnsupportedOperationException("This function should not be called."); + } + + public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector) + throws IOException { + if (isFirstRow) { + if (!readValue()) { + return vector; + } + isFirstRow = false; + } + + // index to set value. + int index = 0; + int valueIndex = 0; + List valueList = new ArrayList<>(); + + // repeated type need two loops to read data. + while (!eof && index < readNumber) { + do { + valueList.add(lastValue); + valueIndex++; + } while (readValue() && (repetitionLevel != 0)); + index++; + } + + return fillColumnVector(valueIndex, valueList); + } + + public LevelDelegation getLevelDelegation() { + int[] repetition = repetitionLevelList.toArray(); + int[] definition = definitionLevelList.toArray(); + repetitionLevelList.clear(); + definitionLevelList.clear(); + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + return new LevelDelegation(repetition, definition); + } + + private boolean readValue() throws IOException { + int left = readPageIfNeed(); + if (left > 0) { + // get the values of repetition and definitionLevel + readAndSaveRepetitionAndDefinitionLevels(); + // read the data if it isn't null + if (definitionLevel == maxDefLevel) { + if (isCurrentPageDictionaryEncoded) { + int dictionaryId = dataColumn.readValueDictionaryId(); + lastValue = dictionaryDecodeValue(logicalType, dictionaryId); + } else { + lastValue = readPrimitiveTypedRow(logicalType); + } + } else { + lastValue = null; + } + return true; + } else { + eof = true; + return false; + } + } + + private void readAndSaveRepetitionAndDefinitionLevels() { + // get the values of repetition and definitionLevel + repetitionLevel = repetitionLevelColumn.nextInt(); + definitionLevel = definitionLevelColumn.nextInt(); + valuesRead++; + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + } + + private int readPageIfNeed() throws IOException { + // Compute the number of values we want to read in this page. + int leftInPage = (int) (endOfPageValueCount - valuesRead); + if (leftInPage == 0) { + // no data left in current page, load data from new page + readPage(); + leftInPage = (int) (endOfPageValueCount - valuesRead); + } + return leftInPage; + } + + private Object readPrimitiveTypedRow(LogicalType category) { + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dataColumn.readBytes(); + case BOOLEAN: + return dataColumn.readBoolean(); + case TIME_WITHOUT_TIME_ZONE: + case DATE: + case INTEGER: + return dataColumn.readInteger(); + case TINYINT: + return dataColumn.readTinyInt(); + case SMALLINT: + return dataColumn.readSmallInt(); + case BIGINT: + return dataColumn.readLong(); + case FLOAT: + return dataColumn.readFloat(); + case DOUBLE: + return dataColumn.readDouble(); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dataColumn.readInteger(); + case INT64: + return dataColumn.readLong(); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return dataColumn.readBytes(); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dataColumn.readTimestamp(); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { + if (dictionaryValue == null) { + return null; + } + + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dictionary.readBytes(dictionaryValue); + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTEGER: + return dictionary.readInteger(dictionaryValue); + case BOOLEAN: + return dictionary.readBoolean(dictionaryValue) ? 1 : 0; + case DOUBLE: + return dictionary.readDouble(dictionaryValue); + case FLOAT: + return dictionary.readFloat(dictionaryValue); + case TINYINT: + return dictionary.readTinyInt(dictionaryValue); + case SMALLINT: + return dictionary.readSmallInt(dictionaryValue); + case BIGINT: + return dictionary.readLong(dictionaryValue); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dictionary.readInteger(dictionaryValue); + case INT64: + return dictionary.readLong(dictionaryValue); + case FIXED_LEN_BYTE_ARRAY: + case BINARY: + return dictionary.readBytes(dictionaryValue); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dictionary.readTimestamp(dictionaryValue); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private WritableColumnVector fillColumnVector(int total, List valueList) { + switch (logicalType.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + HeapBytesVector heapBytesVector = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (src == null) { + heapBytesVector.setNullAt(i); + } else { + heapBytesVector.appendBytes(i, src, 0, src.length); + } + } + return heapBytesVector; + case BOOLEAN: + HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapBooleanVector.setNullAt(i); + } else { + heapBooleanVector.vector[i] = ((List) valueList).get(i); + } + } + return heapBooleanVector; + case TINYINT: + HeapByteVector heapByteVector = new HeapByteVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapByteVector.setNullAt(i); + } else { + heapByteVector.vector[i] = (byte) ((List) valueList).get(i).intValue(); + } + } + return heapByteVector; + case SMALLINT: + HeapShortVector heapShortVector = new HeapShortVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapShortVector.setNullAt(i); + } else { + heapShortVector.vector[i] = (short) ((List) valueList).get(i).intValue(); + } + } + return heapShortVector; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + HeapIntVector heapIntVector = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapIntVector.setNullAt(i); + } else { + heapIntVector.vector[i] = ((List) valueList).get(i); + } + } + return heapIntVector; + case FLOAT: + HeapFloatVector heapFloatVector = new HeapFloatVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapFloatVector.setNullAt(i); + } else { + heapFloatVector.vector[i] = ((List) valueList).get(i); + } + } + return heapFloatVector; + case BIGINT: + HeapLongVector heapLongVector = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapLongVector.setNullAt(i); + } else { + heapLongVector.vector[i] = ((List) valueList).get(i); + } + } + return heapLongVector; + case DOUBLE: + HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapDoubleVector.setNullAt(i); + } else { + heapDoubleVector.vector[i] = ((List) valueList).get(i); + } + } + return heapDoubleVector; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapTimestampVector.setNullAt(i); + } else { + heapTimestampVector.setTimestamp(i, ((List) valueList).get(i)); + } + } + return heapTimestampVector; + case DECIMAL: + PrimitiveType.PrimitiveTypeName primitiveTypeName = + descriptor.getPrimitiveType().getPrimitiveTypeName(); + switch (primitiveTypeName) { + case INT32: + HeapIntVector phiv = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phiv.setNullAt(i); + } else { + phiv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phiv); + case INT64: + HeapLongVector phlv = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phlv.setNullAt(i); + } else { + phlv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phlv); + default: + HeapBytesVector phbv = getHeapBytesVector(total, valueList); + return new ParquetDecimalVector(phbv); + } + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static HeapBytesVector getHeapBytesVector(int total, List valueList) { + HeapBytesVector phbv = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (valueList.get(i) == null) { + phbv.setNullAt(i); + } else { + phbv.appendBytes(i, src, 0, src.length); + } + } + return phbv; + } + + protected void readPage() { + DataPage page = pageReader.readPage(); + + if (page == null) { + return; + } + + page.accept( + new DataPage.Visitor() { + @Override + public Void visit(DataPageV1 dataPageV1) { + readPageV1(dataPageV1); + return null; + } + + @Override + public Void visit(DataPageV2 dataPageV2) { + readPageV2(dataPageV2); + return null; + } + }); + } + + private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) + throws IOException { + this.pageValueCount = valueCount; + this.endOfPageValueCount = valuesRead + pageValueCount; + if (dataEncoding.usesDictionary()) { + this.dataColumn = null; + if (dictionary == null) { + throw new IOException( + String.format( + "Could not read page in col %s because the dictionary was missing for encoding %s.", + descriptor, dataEncoding)); + } + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getDictionaryBasedValuesReader( + descriptor, VALUES, dictionary.getDictionary()), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } else { + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getValuesReader(descriptor, VALUES), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = false; + } + + try { + dataColumn.initFromPage(pageValueCount, in); + } catch (IOException e) { + throw new IOException(String.format("Could not read page in col %s.", descriptor), e); + } + } + + private void readPageV1(DataPageV1 page) { + ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); + ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); + this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); + this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); + try { + BytesInput bytes = page.getBytes(); + LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount); + ByteBufferInputStream in = bytes.toInputStream(); + LOG.debug("Reading repetition levels at {}.", in.position()); + rlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading definition levels at {}.", in.position()); + dlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading data at {}.", in.position()); + initDataReader(page.getValueEncoding(), in, page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private void readPageV2(DataPageV2 page) { + this.pageValueCount = page.getValueCount(); + this.repetitionLevelColumn = + newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); + this.definitionLevelColumn = + newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); + try { + LOG.debug( + "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); + initDataReader( + page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) { + try { + if (maxLevel == 0) { + return new NullIntIterator(); + } + return new RLEIntIterator( + new RunLengthBitPackingHybridDecoder( + BytesUtils.getWidthFromMaxInt(maxLevel), + new ByteArrayInputStream(bytes.toByteArray()))); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read levels in page for col %s.", descriptor), e); + } + } + + /** Utility interface to abstract over different way to read ints with different encodings. */ + interface IntIterator { + int nextInt(); + } + + /** Reading int from {@link ValuesReader}. */ + protected static final class ValuesReaderIntIterator implements IntIterator { + ValuesReader delegate; + + public ValuesReaderIntIterator(ValuesReader delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + return delegate.readInteger(); + } + } + + /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */ + protected static final class RLEIntIterator implements IntIterator { + RunLengthBitPackingHybridDecoder delegate; + + public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + try { + return delegate.readInt(); + } catch (IOException e) { + throw new ParquetDecodingException(e); + } + } + } + + /** Reading zero always. */ + protected static final class NullIntIterator implements IntIterator { + @Override + public int nextInt() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index 3572b117a6313..1826419db5d44 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -18,7 +18,9 @@ package org.apache.hudi.table.format.cow.vector.reader; +import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.table.data.RowData; @@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,6 +42,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -46,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -53,7 +59,6 @@ import java.util.Map; import java.util.stream.IntStream; -import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; @@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable { private final MessageType requestedSchema; + /** + * {@link ParquetField} tree per top-level requested column, used by + * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List, + * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for + * nested types. Entries are {@code null} for primitive top-level fields. Built once per split. + */ + private final List requestedFields; + /** * The total number of rows this RecordReader will eventually read. The sum of the rows of all * the row groups. @@ -158,6 +171,20 @@ public ParquetColumnarRowSplitReader( checkSchema(); + // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across + // row groups). Only columns with nested logical type get a non-null entry — primitive columns + // still use Hudi's specialized ColumnReaders. + MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema); + List requestedRowFields = new ArrayList<>(requestedTypes.length); + List requestedFieldNames = new ArrayList<>(requestedTypes.length); + for (int i = 0; i < requestedTypes.length; i++) { + String name = requestedSchema.getFieldName(i); + requestedRowFields.add(new RowType.RowField(name, requestedTypes[i])); + requestedFieldNames.add(name); + } + this.requestedFields = ParquetSplitReaderUtil.buildFieldsList( + requestedRowFields, requestedFieldNames, messageColumnIO); + this.writableVectors = createWritableVectors(); ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices); this.columnarBatch = generator.generate(columnVectors); @@ -340,12 +367,13 @@ private void readNextRowGroup() throws IOException { List columns = requestedSchema.getColumns(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = createColumnReader( + columnReaders[i] = ParquetSplitReaderUtil.createColumnReader( utcTimestamp, requestedTypes[i], types.get(i), columns, - pages); + pages, + requestedFields.get(i)); } totalCountLoadedSoFar += pages.getRowCount(); } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index fdfe5d6fa3a33..1abc6ed56c0db 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -26,12 +26,16 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY; @@ -252,21 +256,115 @@ public TimestampData readTimestamp() { } } + /** + * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard + * timestamp encoding defined by Parquet's + * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy + * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations. + * (The older INT96 encoding is marked deprecated by the Parquet format spec — see + * + * LogicalTypes.md — but is still supported here via {@link TypesFromInt96PageReader} for + * backwards compatibility with files written by older Hive / Spark / Impala versions.) + * + *

    Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a + * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use + * {@link Int64TimestampColumnReader} for batched-vector efficiency. + */ + public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader { + private final boolean isUtcTimestamp; + private final ChronoUnit chronoUnit; + + public TypesFromInt64PageReader( + ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(realReader); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + public TypesFromInt64PageReader( + Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(dict); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + @Override + public TimestampData readTimestamp() { + return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit); + } + + @Override + public TimestampData readTimestamp(int id) { + return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper( boolean isDictionary, PrimitiveType parquetType, Dictionary dictionary, ValuesReader valuesReader, boolean isUtcTimestamp) { - if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { + PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName(); + if (typeName == PrimitiveType.PrimitiveTypeName.INT96) { return isDictionary ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp) : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp); - } else { - return isDictionary - ? new DefaultParquetDataColumnReader(dictionary) - : new DefaultParquetDataColumnReader(valuesReader); } + if (typeName == PrimitiveType.PrimitiveTypeName.INT64) { + ChronoUnit unit = resolveInt64TimestampUnit(parquetType); + if (unit != null) { + return isDictionary + ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit) + : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit); + } + } + return isDictionary + ? new DefaultParquetDataColumnReader(dictionary) + : new DefaultParquetDataColumnReader(valuesReader); + } + + /** + * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the + * column is a plain INT64 (not a timestamp). + * + *

    Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and + * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} + * encodings. + */ + private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { + LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); + if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + LogicalTypeAnnotation.TimeUnit unit = + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit(); + switch (unit) { + case MILLIS: + return ChronoUnit.MILLIS; + case MICROS: + return ChronoUnit.MICROS; + case NANOS: + return ChronoUnit.NANOS; + default: + return null; + } + } + OriginalType originalType = parquetType.getOriginalType(); + if (originalType == OriginalType.TIMESTAMP_MILLIS) { + return ChronoUnit.MILLIS; + } + if (originalType == OriginalType.TIMESTAMP_MICROS) { + return ChronoUnit.MICROS; + } + return null; + } + + private static TimestampData int64ToTimestamp( + boolean isUtcTimestamp, long value, ChronoUnit unit) { + Instant instant = Instant.EPOCH.plus(value, unit); + if (isUtcTimestamp) { + return TimestampData.fromInstant(instant); + } + return TimestampData.fromTimestamp(Timestamp.from(instant)); } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( @@ -281,10 +379,10 @@ public static ParquetDataColumnReader getDataColumnReaderByType( } private static TimestampData int96ToTimestamp( - boolean utcTimestamp, long nanosOfDay, int julianDay) { + boolean isUtcTimestamp, long nanosOfDay, int julianDay) { long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND); - if (utcTimestamp) { + if (isUtcTimestamp) { int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND); return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond); } else { diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java deleted file mode 100644 index 79b50487f13c1..0000000000000 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** - * Row {@link ColumnReader}. - */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector; - WritableColumnVector[] vectors = rowColumnVector.vectors; - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowColumnVector.setNullAt(j); - } - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java new file mode 100644 index 0000000000000..0f5e00779a2f5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +/** + * Field that represent parquet's field type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetField}). + */ +public abstract class ParquetField { + private final LogicalType type; + private final int repetitionLevel; + private final int definitionLevel; + private final boolean required; + + public ParquetField( + LogicalType type, int repetitionLevel, int definitionLevel, boolean required) { + this.type = type; + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + this.required = required; + } + + public LogicalType getType() { + return type; + } + + public int getRepetitionLevel() { + return repetitionLevel; + } + + public int getDefinitionLevel() { + return definitionLevel; + } + + public boolean isRequired() { + return required; + } + + @Override + public String toString() { + return "Field{" + + "type=" + + type + + ", repetitionLevel=" + + repetitionLevel + + ", definitionLevel=" + + definitionLevel + + ", required=" + + required + + '}'; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java new file mode 100644 index 0000000000000..f91dcca965d64 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's Group Field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension: + * entries in the {@code children} list may be {@code null} to denote a Row child that is absent + * from the parquet file but present in the requested logical schema (schema evolution). This + * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees. + */ +public class ParquetGroupField extends ParquetField { + + private final List children; + + public ParquetGroupField( + LogicalType type, + int repetitionLevel, + int definitionLevel, + boolean required, + List children) { + super(type, repetitionLevel, definitionLevel, required); + // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for + // schema-evolution missing children in ROW types. + this.children = + Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null"))); + } + + /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */ + public List getChildren() { + return children; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java new file mode 100644 index 0000000000000..f6af6f9ff479e --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.column.ColumnDescriptor; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's primitive field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}). + */ +public class ParquetPrimitiveField extends ParquetField { + + private final ColumnDescriptor descriptor; + private final int id; + + public ParquetPrimitiveField( + LogicalType type, boolean required, ColumnDescriptor descriptor, int id) { + super( + type, + descriptor.getMaxRepetitionLevel(), + descriptor.getMaxDefinitionLevel(), + required); + this.descriptor = requireNonNull(descriptor, "descriptor is required"); + this.id = id; + } + + public ColumnDescriptor getDescriptor() { + return descriptor; + } + + public int getId() { + return id; + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java new file mode 100644 index 0000000000000..aff1c32917cf9 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector}, + * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's + * nested-Parquet reader (FLINK-35702). + * + *

    The accessors are wrappers over the existing public fields so legacy callers continue to + * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the + * Dremel-style read path is exercised end-to-end by integration tests in + * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes / + * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes). + */ +class TestHeapColumnVectorAccessors { + + // ----------------------------------------------------------------------------------------------- + // HeapArrayVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapArrayVectorAccessorsReflectPublicFields() { + HeapIntVector child = new HeapIntVector(4); + HeapArrayVector vector = new HeapArrayVector(2, child); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector replacementChild = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setChild(replacementChild); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(replacementChild, vector.getChild()); + assertEquals(2, vector.getSize()); + + // Backing public fields are kept in sync — preserves backward compatibility. + assertSame(offsets, vector.offsets); + assertSame(lengths, vector.lengths); + assertSame(replacementChild, vector.child); + } + + // ----------------------------------------------------------------------------------------------- + // HeapMapColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + + HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values); + + assertEquals(3, vector.getOffsets().length); + assertEquals(3, vector.getLengths().length); + } + + @Test + void heapMapColumnVectorAccessorsReflectInternalState() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector newKeys = new HeapLongVector(4); + HeapLongVector newValues = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setKeys(newKeys); + vector.setValues(newValues); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(newKeys, vector.getKeys()); + assertSame(newValues, vector.getValues()); + // The Flink-2.1-style ColumnVector accessors return the same underlying child. + assertSame(newKeys, vector.getKeyColumnVector()); + assertSame(newValues, vector.getValueColumnVector()); + assertEquals(2, vector.getSize()); + } + + // ----------------------------------------------------------------------------------------------- + // HeapRowColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() { + HeapIntVector intField = new HeapIntVector(2); + HeapLongVector longField = new HeapLongVector(2); + HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField); + + WritableColumnVector[] originalFields = vector.getFields(); + assertEquals(2, originalFields.length); + assertSame(intField, originalFields[0]); + assertSame(longField, originalFields[1]); + // Backing public field is kept in sync — preserves backward compatibility. + assertSame(originalFields, vector.vectors); + + HeapIntVector replacement = new HeapIntVector(2); + WritableColumnVector[] replacementFields = {replacement, longField}; + vector.setFields(replacementFields); + + assertSame(replacementFields, vector.getFields()); + assertSame(replacementFields, vector.vectors); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java new file mode 100644 index 0000000000000..04f5809b9dac4 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.columnar.vector.BytesColumnVector; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ParquetDecimalVector}. + */ +public class TestParquetDecimalVector { + + @Test + void testGetDecimalFromInt32Vector() { + // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true + HeapIntVector intVector = new HeapIntVector(1); + intVector.vector[0] = 12345; + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromInt64Vector() { + // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true + HeapLongVector longVector = new HeapLongVector(1); + longVector.vector[0] = 1234567890123456L; + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + DecimalData decoded = wrapped.getDecimal(0, 18, 4); + + assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtLargePrecision() { + // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path + BigDecimal original = new BigDecimal("12345678901234567890.1234567890"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 30, 10); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtSmallPrecision() { + // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the + // dispatch must fall through to the bytes branch rather than require an IntColumnVector. + BigDecimal original = new BigDecimal("123.45"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalThrowsOnUnsupportedVectorType() { + // A large-precision request must have a bytes-backed child; any other writable child is an + // illegal combination and must be surfaced via Preconditions.checkArgument. + ColumnVector unsupported = new HeapShortVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported); + + assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10)); + } + + @Test + void testIsNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + intVector.vector[0] = 1; + intVector.setNullAt(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + assertFalse(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testWritableIntRoundTrip() { + HeapIntVector intVector = new HeapIntVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setInt(0, 42); + + assertEquals(42, wrapped.getInt(0)); + assertEquals(42, intVector.vector[0]); + } + + @Test + void testWritableLongRoundTrip() { + HeapLongVector longVector = new HeapLongVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + wrapped.setLong(0, 9876543210L); + + assertEquals(9876543210L, wrapped.getLong(0)); + assertEquals(9876543210L, longVector.vector[0]); + } + + @Test + void testWritableBytesRoundTrip() { + HeapBytesVector bytesVector = new HeapBytesVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + byte[] payload = new byte[] {0x01, 0x02, 0x03}; + + wrapped.appendBytes(0, payload, 0, payload.length); + + BytesColumnVector.Bytes out = wrapped.getBytes(0); + assertEquals(payload.length, out.len); + assertEquals(0x01, out.data[out.offset]); + assertEquals(0x02, out.data[out.offset + 1]); + assertEquals(0x03, out.data[out.offset + 2]); + } + + @Test + void testResetDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(1); + intVector.setNullAt(0); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + assertTrue(wrapped.isNullAt(0)); + + wrapped.reset(); + + assertFalse(wrapped.isNullAt(0)); + } + + @Test + void testFillWithNullsDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.fillWithNulls(); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testSetNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setNullAt(0); + wrapped.setNulls(1, 1); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java new file mode 100644 index 0000000000000..ea222dad576a5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.flink.table.data.TimestampData; +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when + * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702). + * + *

    The factory is exercised end-to-end by integration tests through + * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece + * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each + * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} + * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values + * using both the values-reader and dictionary code paths. + */ +class TestParquetDataColumnReaderFactory { + + // ----------------------------------------------------------------------------------------------- + // Type dispatch + // ----------------------------------------------------------------------------------------------- + + @Test + void valuesReaderDispatchInt96TimestampUsesInt96Reader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MILLIS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MICROS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt32DoesNotUseTimestampReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + // ----------------------------------------------------------------------------------------------- + // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches) + // ----------------------------------------------------------------------------------------------- + + @Test + void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_123L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMillis), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + assertEquals(0, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + long epochMicros = 1_700_000_000_123_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMicros), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMicros / 1_000L, ts.getMillisecond()); + // 456 microseconds remain → 456_000 nanoseconds within the millisecond + assertEquals(456_000, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + long epochNanos = 1_700_000_000_123_456_789L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochNanos), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochNanos / 1_000_000L, ts.getMillisecond()); + assertEquals(456_789, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMillisFromDictionaryInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(epochMillis), true); + + TimestampData ts = reader.readTimestamp(0); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + } + + // ----------------------------------------------------------------------------------------------- + // Stubs (only the methods exercised by the dispatch + decoding tests above) + // ----------------------------------------------------------------------------------------------- + + /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */ + private static final class StubValuesReader extends ValuesReader { + private final long fixedLong; + + StubValuesReader() { + this(0L); + } + + StubValuesReader(long fixedLong) { + this.fixedLong = fixedLong; + } + + @Override + public long readLong() { + return fixedLong; + } + + @Override + public void skip() { + // unused + } + } + + /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */ + private static final class StubDictionary extends Dictionary { + private final long fixedLong; + + StubDictionary() { + this(0L); + } + + StubDictionary(long fixedLong) { + super(null); + this.fixedLong = fixedLong; + } + + @Override + public Binary decodeToBinary(int id) { + throw new UnsupportedOperationException(); + } + + @Override + public long decodeToLong(int id) { + return fixedLong; + } + + @Override + public int getMaxId() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java new file mode 100644 index 0000000000000..2b71bae4dc152 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link ParquetGroupField}. + */ +public class TestParquetGroupField { + + @Test + void testChildrenWithAllNonNullEntriesAreRetained() { + ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1); + List children = Arrays.asList(c0, c1); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertSame(c0, group.getChildren().get(0)); + assertSame(c1, group.getChildren().get(1)); + } + + @Test + void testChildrenMayContainNullForSchemaEvolution() { + // A ROW field present in the requested Flink schema but absent from the Parquet file is + // represented by a null slot in `children`. The group must allow this (Hudi-specific + // extension over Flink's ImmutableList-backed equivalent). + ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List children = Arrays.asList(present, null); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertNotNull(group.getChildren().get(0)); + assertNull(group.getChildren().get(1)); + } + + @Test + void testChildrenListIsUnmodifiable() { + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child)); + + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null)); + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0)); + } + + @Test + void testChildrenListIsDefensivelyCopied() { + // Mutations to the caller-supplied list must not be visible through the group. + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List mutable = new ArrayList<>(); + mutable.add(child); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable); + mutable.add(null); + + assertEquals(1, group.getChildren().size()); + } + + @Test + void testNullChildrenListThrows() { + assertThrows( + NullPointerException.class, + () -> new ParquetGroupField(rowType(), 0, 1, true, null)); + } + + @Test + void testEmptyChildrenListIsAllowed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList()); + + assertEquals(0, group.getChildren().size()); + } + + @Test + void testFieldMetadataIsExposed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList()); + + assertEquals(2, group.getRepetitionLevel()); + assertEquals(5, group.getDefinitionLevel()); + assertFalse(group.isRequired()); + } + + private static LogicalType rowType() { + return RowType.of(new IntType()); + } + + private static ColumnDescriptor descriptor() { + PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f"); + return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0); + } +} From b414c9cc0189c52f4766de7e853d413614f51461 Mon Sep 17 00:00:00 2001 From: fhan Date: Wed, 10 Jun 2026 23:27:13 +0800 Subject: [PATCH 056/255] fix(clustering): retain missing partitions in selected/regex incremental scheduling (#18945) * fix(clustering): retain missing partitions in selected/regex incremental scheduling * fix(clustering): refine getMissPartitions helper method * fix(clustering): return mutable missing partitions list --------- Co-authored-by: fhan (cherry picked from commit 89332241ea0b42671dae81b5ef06cb7ab9fad64b) --- .../strategy/ClusteringPlanStrategy.java | 17 ++++++++++ .../PartitionAwareClusteringPlanStrategy.java | 12 ++++--- ...tPartitionAwareClusteringPlanStrategy.java | 34 ++++++++++++++++++- .../cluster/TestIncrementalClustering.java | 4 +-- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java index 353f590c3f971..a2a32825d12ba 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java @@ -41,8 +41,11 @@ import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -137,6 +140,20 @@ protected Stream getFileSlicesEligibleForClustering(String partition) */ protected abstract Map getStrategyParams(); + /** + * Keep partitions from the current scheduling window that are not scheduled in this plan as missing + * partitions so that they can be picked up by later incremental clustering schedules. + */ + protected List getMissingPartitionsFromCurrentWindow(List partitionsToSchedule, + List partitionsInCurrentWindow) { + if (!getWriteConfig().isIncrementalTableServiceEnabled()) { + return new ArrayList<>(); + } + Set missingPartitions = new LinkedHashSet<>(partitionsInCurrentWindow); + missingPartitions.removeAll(new HashSet<>(partitionsToSchedule)); + return new ArrayList<>(missingPartitions); + } + /** * Returns any specific parameters to be stored as part of clustering metadata. */ diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java index 077f1bb77e5fd..3d49c5f406e15 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java @@ -24,7 +24,6 @@ import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.model.FileSlice; -import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; @@ -41,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -169,13 +169,15 @@ public Option generateClusteringPlan(ClusteringPlanActionE if (StringUtils.isNullOrEmpty(partitionSelected)) { // get matched partitions if set - partitionPaths = getRegexPatternMatchedPartitions(config, partitions.get()); + // partitionsInCurrentWindow = incremental partitions + missing partitions in last plan + List partitionsInCurrentWindow = partitions.get(); + partitionPaths = getRegexPatternMatchedPartitions(config, partitionsInCurrentWindow); + missingPartitions = getMissingPartitionsFromCurrentWindow(partitionPaths, partitionsInCurrentWindow); // filter the partition paths if needed to reduce list status } else { partitionPaths = Arrays.asList(partitionSelected.split(",")); - // Users may temporarily set specific partitions for clustering. - // Ensure the coherence of the missing partitions. - missingPartitions = (List)executor.fetchMissingPartitions(TableServiceType.CLUSTER).getRight(); + missingPartitions = getMissingPartitionsFromCurrentWindow(partitionPaths, + config.isIncrementalTableServiceEnabled() ? partitions.get() : Collections.emptyList()); } Pair, List> partitionsPair = filterPartitionPaths(getWriteConfig(), partitionPaths); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java index 471245e3bc513..23a0b6223ca2f 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java @@ -31,6 +31,8 @@ import org.mockito.MockitoAnnotations; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -40,6 +42,8 @@ import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestPartitionAwareClusteringPlanStrategy { @@ -83,6 +87,29 @@ public void testFilterPartitionPaths() { assertTrue(list.contains("20210723")); } + @Test + public void testResolveMissingPartitionsFromCurrentWindow() { + HoodieWriteConfig incrementalConfig = mock(HoodieWriteConfig.class); + when(incrementalConfig.isIncrementalTableServiceEnabled()).thenReturn(true); + DummyPartitionAwareClusteringPlanStrategy incrementalStrategy = + new DummyPartitionAwareClusteringPlanStrategy(table, context, incrementalConfig); + + assertEquals(Arrays.asList("p2", "p4"), + incrementalStrategy.resolveMissingPartitionsFromCurrentWindow( + Arrays.asList("p1", "p3"), Arrays.asList("p1", "p2", "p3", "p4"))); + + HoodieWriteConfig nonIncrementalConfig = mock(HoodieWriteConfig.class); + when(nonIncrementalConfig.isIncrementalTableServiceEnabled()).thenReturn(false); + DummyPartitionAwareClusteringPlanStrategy nonIncrementalStrategy = + new DummyPartitionAwareClusteringPlanStrategy(table, context, nonIncrementalConfig); + + List nonIncrementalMissingPartitions = nonIncrementalStrategy.resolveMissingPartitionsFromCurrentWindow( + Arrays.asList("p1", "p3"), Arrays.asList("p1", "p2", "p3", "p4")); + assertTrue(nonIncrementalMissingPartitions.isEmpty()); + nonIncrementalMissingPartitions.addAll(Collections.singletonList("p5")); + assertEquals(Collections.singletonList("p5"), nonIncrementalMissingPartitions); + } + @Test public void testResolveEngineContextUsesLocalWhenEnabled() { HoodieEngineContext engineContext = new HoodieLocalEngineContext(new HadoopStorageConfiguration(false)); @@ -127,10 +154,15 @@ public DummyPartitionAwareClusteringPlanStrategy(HoodieTable table, HoodieEngine super(table, engineContext, writeConfig); } + List resolveMissingPartitionsFromCurrentWindow(List partitionsToSchedule, + List partitionsInCurrentWindow) { + return getMissingPartitionsFromCurrentWindow(partitionsToSchedule, partitionsInCurrentWindow); + } + @Override protected Map getStrategyParams() { return null; } } -} \ No newline at end of file +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java index 8336a6f7e0052..800655065501d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java @@ -131,8 +131,8 @@ public void testPartitionsForIncrClusteringWithFilter(ClusteringPlanPartitionFil switch (mode) { case NONE: { - // For partitions filtered out by the regex expression, they will not be recorded in the missingPartitions - assertEquals(0, clusteringPlan.getMissingSchedulePartitions().size()); + assertEquals(1, clusteringPlan.getMissingSchedulePartitions().size()); + assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(YESTERDAY)); break; } case SELECTED_PARTITIONS: { From c77841848e9cd6e7c1c064275d50e342a15b7f38 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Thu, 11 Jun 2026 10:09:22 +0800 Subject: [PATCH 057/255] fix(flink): Close write client properly in DefaultCleanHandler (#18940) (cherry picked from commit 21121acf356179848f7dbbeb1a9f1d5cadd2e756) --- .../compact/handler/DefaultCleanHandler.java | 2 +- .../handler/TestDefaultCleanHandler.java | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestDefaultCleanHandler.java diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/DefaultCleanHandler.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/DefaultCleanHandler.java index df95b81a06a39..2df75847ddaba 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/DefaultCleanHandler.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/DefaultCleanHandler.java @@ -111,6 +111,6 @@ public void close() { throw new HoodieException("Failed to close executor of clean handler.", e); } } - this.writeClient.clean(); + this.writeClient.close(); } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestDefaultCleanHandler.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestDefaultCleanHandler.java new file mode 100644 index 0000000000000..7c2c5524d49da --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestDefaultCleanHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.compact.handler; + +import org.apache.hudi.client.HoodieFlinkWriteClient; + +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +class TestDefaultCleanHandler { + + @Test + void testCloseClosesWriteClientWithoutTriggeringClean() { + HoodieFlinkWriteClient writeClient = mock(HoodieFlinkWriteClient.class); + DefaultCleanHandler handler = new DefaultCleanHandler(writeClient); + + handler.close(); + + verify(writeClient).close(); + verifyNoMoreInteractions(writeClient); + } +} From ed2f11b16f09db5ab0ceeb56823902b571c170ba Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Thu, 11 Jun 2026 10:50:33 +0800 Subject: [PATCH 058/255] feat(flink): Backport Flink 2.1 Dremel nested Parquet reader to hudi-flink2.1.x (#18960) (cherry picked from commit f1c6fe4d09322171b46fe7b12796b2c06b708d06) --- .../hudi/table/ITTestHoodieDataSource.java | 30 + .../java/org/apache/hudi/utils/TestSQL.java | 5 + .../format/cow/ParquetSplitReaderUtil.java | 750 ++++++++++-------- .../format/cow/utils/BooleanArrayList.java | 69 ++ .../table/format/cow/utils/IntArrayList.java | 93 +++ .../table/format/cow/utils/LongArrayList.java | 83 ++ .../format/cow/utils/NestedPositionUtil.java | 209 +++++ .../cow/vector/ColumnarGroupArrayData.java | 185 ----- .../cow/vector/ColumnarGroupMapData.java | 63 -- .../cow/vector/ColumnarGroupRowData.java | 144 ---- .../vector/HeapArrayGroupColumnVector.java | 53 -- .../format/cow/vector/HeapArrayVector.java | 35 +- .../cow/vector/HeapMapColumnVector.java | 74 +- .../cow/vector/HeapRowColumnVector.java | 15 + .../cow/vector/ParquetDecimalVector.java | 197 ++++- .../vector/position/CollectionPosition.java | 57 ++ .../cow/vector/position/LevelDelegation.java | 43 + .../cow/vector/position/RowPosition.java | 45 ++ .../cow/vector/reader/ArrayColumnReader.java | 473 ----------- .../cow/vector/reader/ArrayGroupReader.java | 45 -- .../cow/vector/reader/MapColumnReader.java | 55 -- .../cow/vector/reader/NestedColumnReader.java | 284 +++++++ .../reader/NestedPrimitiveColumnReader.java | 638 +++++++++++++++ .../reader/ParquetColumnarRowSplitReader.java | 34 +- .../ParquetDataColumnReaderFactory.java | 121 ++- .../cow/vector/reader/RowColumnReader.java | 63 -- .../format/cow/vector/type/ParquetField.java | 72 ++ .../cow/vector/type/ParquetGroupField.java | 59 ++ .../vector/type/ParquetPrimitiveField.java | 55 ++ .../vector/TestHeapColumnVectorAccessors.java | 138 ++++ .../cow/vector/TestParquetDecimalVector.java | 187 +++++ .../TestParquetDataColumnReaderFactory.java | 270 +++++++ .../vector/type/TestParquetGroupField.java | 132 +++ 33 files changed, 3349 insertions(+), 1427 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 610a23b3b0e76..32b5b9476f143 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -2376,6 +2376,36 @@ void testParquetNullChildColumnsRowTypes(String operation) { assertRowsEquals(result, expected); } + @ParameterizedTest + @ValueSource(strings = {"insert", "upsert", "bulk_insert"}) + void testParquetDeeplyNestedRepeatedTypes(String operation) { + // Covers a ROW containing an ARRAY of ROW that itself contains a MAP, i.e. + // ROW>>>, where the MAP is a repeated field + // nested inside another repeated field (repetition level >= 2). + // See HUDI-18491 for the original bug report on this schema shape. + TableEnvironment tableEnv = batchTableEnv; + + String hoodieTableDDL = sql("t1") + .field("f_int int") + .field("f_row row(f_nested_array array)>)") + .pkField("f_int") + .noPartition() + .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) + .option(FlinkOptions.OPERATION, operation) + .end(); + tableEnv.executeSql(hoodieTableDDL); + + execInsertSql(tableEnv, TestSQL.DEEPLY_NESTED_REPEATED_TYPE_INSERT_T1); + + List result = CollectionUtil.iterableToList( + () -> tableEnv.sqlQuery("select * from t1").execute().collect()); + List expected = Arrays.asList( + row(1, row((Object) array(row(11, map("a", 1, "b", 2)), row(12, map("c", 3))))), + row(2, row((Object) array(row(21, map("d", 4))))), + row(3, row((Object) array(row(31, map("e", 5)), row(32, map("f", 6, "g", 7)))))); + assertRowsEqualsUnordered(expected, result); + } + @ParameterizedTest @ValueSource(strings = {"insert", "upsert", "bulk_insert"}) void testBuiltinFunctionWithCatalog(String operation) { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestSQL.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestSQL.java index c7b38d81d90db..c4428b05f530c 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestSQL.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestSQL.java @@ -74,6 +74,11 @@ public class TestSQL { + "(2, row(2, cast(null as varchar))),\n" + "(3, row(cast(null as int), cast(null as varchar)))"; + public static final String DEEPLY_NESTED_REPEATED_TYPE_INSERT_T1 = "insert into t1 values\n" + + "(1, row(array[row(11, map['a', 1, 'b', 2]), row(12, map['c', 3])])),\n" + + "(2, row(array[row(21, map['d', 4])])),\n" + + "(3, row(array[row(31, map['e', 5]), row(32, map['f', 6, 'g', 7])]))"; + public static final String INSERT_DATE_PARTITION_T1 = "insert into t1 values\n" + "('id1','Danny',23,DATE '1970-01-01'),\n" + "('id2','Stephen',33,DATE '1970-01-01'),\n" diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index 3e16417ba541e..d9479bcd658a9 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -20,19 +20,18 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapDecimalVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; -import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader; import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader; import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader; import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader; +import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader; import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader; -import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader; @@ -65,12 +64,15 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarBinaryType; +import org.apache.flink.table.types.logical.VariantType; +import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; @@ -78,12 +80,18 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.io.ColumnIO; +import org.apache.parquet.io.GroupColumnIO; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.sql.Date; @@ -91,25 +99,38 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.flink.table.utils.DateTimeUtils.toInternal; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** * Util for generating {@link ParquetColumnarRowSplitReader}. * - *

    NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64 - * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that. + *

    Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive + * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader}, + * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For + * nested types (ARRAY / MAP / MULTISET / ROW / VARIANT) we build a {@link ParquetField} tree once + * per split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to + * {@link NestedColumnReader}. + * + *

    Schema evolution: missing top-level fields are still handled by the caller + * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields + * inside a Row are handled here — {@link #constructField} returns {@code null} for a + * child that isn't physically present, and the corresponding child in the pre-allocated vector + * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can + * passthrough the slot (see {@link NestedColumnReader#readToVector}). */ public class ParquetSplitReaderUtil { - /** - * Util for generating partitioned {@link ParquetColumnarRowSplitReader}. - */ + /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( boolean utcTimestamp, boolean caseSensitive, @@ -183,10 +204,13 @@ private static ColumnVector createVector( return readVector; } - private static ColumnVector createVectorFromConstant( - LogicalType type, - Object value, - int batchSize) { + /** + * Builds a constant-filled column vector for either a partition column (non-null value) or a + * missing-column slot (null value). Used both at the batch-generator level for partition + * injection and at the row-reader level for fields absent from the Parquet file. + */ + public static ColumnVector createVectorFromConstant( + LogicalType type, Object value, int batchSize) { switch (type.getTypeRoot()) { case CHAR: case VARCHAR: @@ -279,6 +303,7 @@ private static ColumnVector createVectorFromConstant( value == null ? null : toInternal((Date) value), batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: HeapTimestampVector tv = new HeapTimestampVector(batchSize); if (value == null) { tv.fillWithNulls(); @@ -287,57 +312,49 @@ private static ColumnVector createVectorFromConstant( } return tv; case ARRAY: - ArrayType arrayType = (ArrayType) type; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize); - if (value == null) { - arrayGroup.fillWithNulls(); - return arrayGroup; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } - } else { - HeapArrayVector arrayVector = new HeapArrayVector(batchSize); - if (value == null) { - arrayVector.fillWithNulls(); - return arrayVector; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } + if (value != null) { + throw new UnsupportedOperationException("Unsupported create array with default value."); } + HeapArrayVector arrayVector = new HeapArrayVector(batchSize); + arrayVector.fillWithNulls(); + return arrayVector; case MAP: - HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); - if (value == null) { - mapVector.fillWithNulls(); - return mapVector; - } else { - throw new UnsupportedOperationException("Unsupported create map with default value."); + case MULTISET: + if (value != null) { + throw new UnsupportedOperationException( + "Unsupported create " + type.getTypeRoot() + " with default value."); } + HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); + mapVector.fillWithNulls(); + return mapVector; case ROW: - HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize); - if (value == null) { - rowVector.fillWithNulls(); - return rowVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create row with default value."); } + RowType rowType = (RowType) type; + WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()]; + for (int i = 0; i < childVectors.length; i++) { + childVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); + } + HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors); + rowVector.fillWithNulls(); + return rowVector; case VARIANT: - HeapRowColumnVector variantVector = new HeapRowColumnVector( - batchSize, - new HeapBytesVector(batchSize), - new HeapBytesVector(batchSize)); - if (value == null) { - variantVector.fillWithNulls(); - return variantVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create variant with default value."); } + HeapRowColumnVector variantVector = new HeapRowColumnVector( + batchSize, new HeapBytesVector(batchSize), new HeapBytesVector(batchSize)); + variantVector.fillWithNulls(); + return variantVector; default: throw new UnsupportedOperationException("Unsupported type: " + type); } } - private static List filterDescriptors(int depth, Type type, List columns) throws ParquetRuntimeException { + private static List filterDescriptors( + int depth, Type type, List columns) throws ParquetRuntimeException { List filtered = new ArrayList<>(); for (ColumnDescriptor descriptor : columns) { if (depth >= descriptor.getPath().length) { @@ -351,24 +368,65 @@ private static List filterDescriptors(int depth, Type type, Li return filtered; } + /** + * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the + * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP}, + * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by + * the supplied pre-built {@link ParquetField} tree. + * + * @param field the {@link ParquetField} tree for this column, built by + * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested + * types; ignored for primitives. + */ + public static ColumnReader createColumnReader( + boolean utcTimestamp, + LogicalType fieldType, + Type physicalType, + List descriptors, + PageReadStore pages, + @Nullable ParquetField field) throws IOException { + switch (fieldType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + case VARIANT: + // VARIANT is physically a non-shredded group of two binary leaves (value, metadata), so it + // is assembled like a two-field row by the Dremel-style NestedColumnReader (see + // NestedColumnReader#readVariant). The ParquetField tree is built by #constructField. + Preconditions.checkNotNull( + field, "ParquetField must be provided for nested type: %s", fieldType); + return new NestedColumnReader(utcTimestamp, pages, field); + default: + return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages); + } + } + + /** + * Backward-compat entry point kept for callers that don't project nested types and therefore + * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload + * with a null field; nested types now go through that overload directly. + * + * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore, + * ParquetField)} so nested types take the Dremel path. + */ + @Deprecated public static ColumnReader createColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List descriptors, PageReadStore pages) throws IOException { - return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, - pages, 0); + return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null); } - private static ColumnReader createColumnReader( + private static ColumnReader createPrimitiveColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List columns, - PageReadStore pages, - int depth) throws IOException { - List descriptors = filterDescriptors(depth, physicalType, columns); + PageReadStore pages) throws IOException { + List descriptors = filterDescriptors(0, physicalType, columns); ColumnDescriptor descriptor = descriptors.get(0); PageReader pageReader = pages.getPageReader(descriptor); switch (fieldType.getTypeRoot()) { @@ -404,7 +462,9 @@ private static ColumnReader createColumnReader( case INT96: return new TimestampColumnReader(utcTimestamp, descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for TIMESTAMP: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } case DECIMAL: switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { @@ -415,120 +475,23 @@ private static ColumnReader createColumnReader( case BINARY: return new BytesColumnReader(descriptor, pageReader); case FIXED_LEN_BYTE_ARRAY: - return new FixedLenBytesColumnReader( - descriptor, pageReader); + return new FixedLenBytesColumnReader(descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for DECIMAL: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } - case ARRAY: - ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new ArrayGroupReader(createColumnReader( - utcTimestamp, - arrayType.getElementType(), - elementType, - descriptors, - pages, - elementDepth)); - } else { - return new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - fieldType); - } - case MAP: - MapType mapType = (MapType) fieldType; - ArrayColumnReader keyReader = - new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - new ArrayType(mapType.getKeyType())); - ColumnReader valueReader; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueReader = new ArrayGroupReader(createColumnReader( - utcTimestamp, - mapType.getValueType(), - physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type - descriptors.subList(1, descriptors.size()), // remove the key descriptor - pages, - depth + 2)); // increase the depth by 2, because there's a key_value entry in the path - } else { - valueReader = new ArrayColumnReader( - descriptors.get(1), - pages.getPageReader(descriptors.get(1)), - utcTimestamp, - descriptors.get(1).getPrimitiveType(), - new ArrayType(mapType.getValueType())); - } - return new MapColumnReader(keyReader, valueReader); - case ROW: - RowType rowType = (RowType) fieldType; - GroupType groupType = physicalType.asGroupType(); - List fieldReaders = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - // schema evolution: read the parquet file with a new extended field name. - int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); - if (fieldIndex < 0) { - fieldReaders.add(new EmptyColumnReader()); - } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - fieldReaders.add( - createColumnReader( - utcTimestamp, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } else { - fieldReaders.add( - createColumnReader( - utcTimestamp, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } - } - } - return new RowColumnReader(fieldReaders); - case VARIANT: - ColumnDescriptor valueDescriptor = getVariantColumnDescriptor( - physicalType, - descriptors, - depth, - HoodieSchema.Variant.VARIANT_VALUE_FIELD); - ColumnDescriptor metadataDescriptor = getVariantColumnDescriptor( - physicalType, - descriptors, - depth, - HoodieSchema.Variant.VARIANT_METADATA_FIELD); - return new RowColumnReader(Arrays.asList( - new BytesColumnReader(valueDescriptor, pages.getPageReader(valueDescriptor)), - new BytesColumnReader(metadataDescriptor, pages.getPageReader(metadataDescriptor)))); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + /** + * Creates the writable column vector that the reader will write into. The returned vector shape + * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled + * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in + * {@link NestedColumnReader} can pass them through unchanged. + */ public static WritableColumnVector createWritableColumnVector( int batchSize, LogicalType fieldType, @@ -549,33 +512,40 @@ private static WritableColumnVector createWritableColumnVector( switch (fieldType.getTypeRoot()) { case BOOLEAN: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBooleanVector(batchSize); case TINYINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapByteVector(batchSize); case DOUBLE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDoubleVector(batchSize); case FLOAT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.FLOAT, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapFloatVector(batchSize); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapIntVector(batchSize); case BIGINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT64, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapLongVector(batchSize); case SMALLINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapShortVector(batchSize); case CHAR: case VARCHAR: @@ -592,112 +562,64 @@ private static WritableColumnVector createWritableColumnVector( case DECIMAL: checkArgument( (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY - || typeName == PrimitiveType.PrimitiveTypeName.BINARY) + || typeName == PrimitiveType.PrimitiveTypeName.BINARY) && primitiveType.getOriginalType() == OriginalType.DECIMAL, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDecimalVector(batchSize); case ARRAY: ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - elementType, - descriptors, - elementDepth)); - } else { - return new HeapArrayVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - physicalType, - descriptors, - depth)); - } - case MAP: + return new HeapArrayVector( + batchSize, + createWritableColumnVector( + batchSize, arrayType.getElementType(), physicalType, descriptors, depth)); + case MAP: { MapType mapType = (MapType) fieldType; - GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType(); - // the map column has three level paths. - WritableColumnVector keyColumnVector = createWritableColumnVector( + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( batchSize, - new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()), - repeatedType.getType(0), - descriptors, - depth + 2); - WritableColumnVector valueColumnVector; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueColumnVector = new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - mapType.getValueType(), - repeatedType.getType(1).asGroupType(), - descriptors, - depth + 2)); - } else { - valueColumnVector = createWritableColumnVector( - batchSize, - new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()), - repeatedType.getType(1), - descriptors, - depth + 2); - } - return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector); + createWritableColumnVector( + batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2), + createWritableColumnVector( + batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2)); + } + case MULTISET: { + MultisetType multisetType = (MultisetType) fieldType; + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( + batchSize, + createWritableColumnVector( + batchSize, + multisetType.getElementType(), + repeatedType.getType(0), + descriptors, + depth + 2), + createWritableColumnVector( + batchSize, + new IntType(false), + repeatedType.getType(1), + descriptors, + depth + 2)); + } case ROW: RowType rowType = (RowType) fieldType; GroupType groupType = physicalType.asGroupType(); WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()]; for (int i = 0; i < columnVectors.length; i++) { - // schema evolution: read the file with a new extended field name. int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); if (fieldIndex < 0) { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant( - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize); - } else { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); - } + // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled + // vector of the correct shape; NestedColumnReader.readRow will pass it through when the + // matching ParquetField child is null. + columnVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = - createWritableColumnVector( - batchSize, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } else { - columnVectors[i] = - createWritableColumnVector( - batchSize, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } + columnVectors[i] = + createWritableColumnVector( + batchSize, + rowType.getTypeAt(i), + groupType.getType(fieldIndex), + descriptors, + depth + 1); } } return new HeapRowColumnVector(batchSize, columnVectors); @@ -712,19 +634,6 @@ private static WritableColumnVector createWritableColumnVector( } } - private static ColumnDescriptor getVariantColumnDescriptor( - Type physicalType, - List descriptors, - int depth, - String fieldName) { - return descriptors.stream() - .filter(descriptor -> descriptor.getPath().length > depth + 1 - && fieldName.equals(descriptor.getPath()[depth + 1])) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "Invalid Variant Parquet schema: missing binary field '" + fieldName + "'.")); - } - private static void validateVariantType(Type physicalType) { if (!physicalType.isPrimitive()) { GroupType groupType = physicalType.asGroupType(); @@ -764,56 +673,269 @@ private static void validateVariantField(GroupType groupType, String fieldName) } /** - * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. - * - * @return The physical field index or -1 if the field does not exist + * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching + * Parquet's canonical 3-level map encoding. */ - private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { - // get index from fileSchema type, else, return -1 - return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + private static GroupType unwrapMapRepeatedType(Type physicalType) { + return physicalType.asGroupType().getType(0).asGroupType(); } + // ------------------------------------------------------------------------------------------ + // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil) + // + // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children + // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This + // matches the Hudi schema-evolution contract and is the companion to the null-child branch in + // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`. + // ------------------------------------------------------------------------------------------ + /** - * Check whether the given list type is a three-level list type. - *

    - * group (LIST) { - * repeated group list { - * element; - * } - * } - * - * @param type list type - * @return true if the list type is a three-level list type + * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed + * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally; + * primitive top-level fields produce {@code null} entries (callers don't need a tree for those). + */ + public static List buildFieldsList( + List children, List fieldNames, MessageColumnIO columnIO) { + List list = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + RowType.RowField child = children.get(i); + if (isNestedType(child.getType())) { + list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i)))); + } else { + list.add(null); + } + } + return list; + } + + private static boolean isNestedType(LogicalType type) { + return type instanceof RowType + || type instanceof ArrayType + || type instanceof MapType + || type instanceof MultisetType + || type instanceof VariantType; + } + + @Nullable + private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) { + boolean required = columnIO.getType().getRepetition() == REQUIRED; + int repetitionLevel = columnIO.getRepetitionLevel(); + int definitionLevel = columnIO.getDefinitionLevel(); + LogicalType type = rowField.getType(); + String fieldName = rowField.getName(); + if (type instanceof RowType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + RowType rowType = (RowType) type; + List childFields = rowType.getFields(); + List fieldsList = new ArrayList<>(childFields.size()); + for (RowType.RowField childField : childFields) { + // Hudi schema evolution: a logical child may be absent from the Parquet file. In that + // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the + // pre-filled null vector instead of recursing. + ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName()); + if (childIo == null) { + fieldsList.add(null); + } else { + fieldsList.add(constructField(childField, childIo)); + } + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(fieldsList)); + } + + if (type instanceof VariantType) { + // A variant is physically a non-shredded group of two binary leaves (value, metadata). Build + // it as a two-field group of binary primitives so NestedColumnReader#readVariant can assemble + // it like a row. Shredded variants (with a typed_value field) are rejected here. + validateVariantType(columnIO.getType()); + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + LogicalType binaryType = new VarBinaryType(VarBinaryType.MAX_LENGTH); + ParquetField valueField = + constructField( + new RowType.RowField(HoodieSchema.Variant.VARIANT_VALUE_FIELD, binaryType), + lookupColumnByName(groupColumnIO, HoodieSchema.Variant.VARIANT_VALUE_FIELD)); + ParquetField metadataField = + constructField( + new RowType.RowField(HoodieSchema.Variant.VARIANT_METADATA_FIELD, binaryType), + lookupColumnByName(groupColumnIO, HoodieSchema.Variant.VARIANT_METADATA_FIELD)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(valueField, metadataField))); + } + + if (type instanceof MapType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MapType mapType = (MapType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof MultisetType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MultisetType multisetType = (MultisetType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", multisetType.getElementType()), + keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof ArrayType) { + ArrayType arrayType = (ArrayType) type; + ColumnIO elementTypeColumnIO; + if (columnIO instanceof GroupColumnIO) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { + while (!Objects.equals(groupColumnIO.getName(), fieldName)) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + elementTypeColumnIO = groupColumnIO; + } else { + if (arrayType.getElementType() instanceof RowType) { + elementTypeColumnIO = groupColumnIO; + } else { + elementTypeColumnIO = groupColumnIO.getChild(0); + } + } + } else if (columnIO instanceof PrimitiveColumnIO) { + elementTypeColumnIO = columnIO; + } else { + throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); + } + + ParquetField elementField = + constructField( + new RowType.RowField("", arrayType.getElementType()), + getArrayElementColumn(elementTypeColumnIO)); + if (repetitionLevel == elementField.getRepetitionLevel()) { + repetitionLevel = columnIO.getParent().getRepetitionLevel(); + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.singletonList(elementField)); + } + + PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO; + return new ParquetPrimitiveField( + type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId()); + } + + /** + * Parquet column names are case-insensitive in Flink's lookup. Matches upstream + * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent. */ - private static boolean isThreeLevelList(Type type) { - if (type.isPrimitive()) { - return false; + public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName); + if (columnIO != null) { + return columnIO; } - GroupType groupType = type.asGroupType(); - OriginalType originalType = groupType.getOriginalType(); - return originalType == OriginalType.LIST - && groupType.getType(0).getName().equals("list"); + throw new FlinkRuntimeException( + "Can not find column io for parquet reader. Column name: " + columnName); } /** - * Construct the error message when primitive type mismatches. - * - * @param primitiveType Primitive type - * @param fieldType Logical field type - * @return The error message + * Case-insensitive column lookup that returns {@code null} when no match is found — the + * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to + * emit null {@link ParquetField} children for fields absent from the Parquet file. */ - private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { - return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name()); + @Nullable + private static ColumnIO lookupColumnByNameOrNull( + GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = groupColumnIO.getChild(columnName); + if (columnIO != null) { + return columnIO; + } + for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) { + if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) { + return groupColumnIO.getChild(i); + } + } + return null; + } + + public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { + while (groupColumnIO.getChildrenCount() == 1) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + return groupColumnIO; + } + + public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { + while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { + columnIO = ((GroupColumnIO) columnIO).getChild(0); + } + + // Three-level list: skip the synthetic `element` / `list` wrapper when present. + if (columnIO instanceof GroupColumnIO + && columnIO.getType().getLogicalTypeAnnotation() == null + && ((GroupColumnIO) columnIO).getChildrenCount() == 1 + && !columnIO.getName().equals("array") + && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { + return ((GroupColumnIO) columnIO).getChild(0); + } + return columnIO; } /** - * Construct the error message when original type mismatches. + * Returns the field index with given physical row type {@code groupType} and field name + * {@code fieldName}. * - * @param originalType Original type - * @param fieldType Logical field type - * @return The error message + * @return the physical field index or -1 if the field does not exist + */ + private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { + return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + } + + private static String getPrimitiveTypeCheckFailureMessage( + PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Primitive type: %s. Field type: %s.", + primitiveType, fieldType.getTypeRoot().name()); + } + + private static String getOriginalTypeCheckFailureMessage( + OriginalType originalType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Original type: %s. Field type: %s.", + originalType, fieldType.getTypeRoot().name()); + } + + /** + * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience + * for callers that need to mirror Hudi's original behaviour where a missing column produces an + * explicit null-valued reader rather than being omitted from the batch. */ - private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) { - return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name()); + public static ColumnReader emptyColumnReader() { + return new EmptyColumnReader(); } } diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java new file mode 100644 index 0000000000000..d51d7ee754b8a --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of booleans. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper. + */ +public class BooleanArrayList { + private int size; + private boolean[] array; + + public BooleanArrayList(int capacity) { + this.size = 0; + this.array = new boolean[capacity]; + } + + public int size() { + return size; + } + + public boolean add(boolean element) { + grow(size + 1); + array[size++] = element; + return true; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public boolean[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final boolean[] t = new boolean[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java new file mode 100644 index 0000000000000..4787dbb5b9ddb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; +import java.util.NoSuchElementException; + +/** + * Minimal implementation of an array-backed list of ints. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class IntArrayList { + + private int size; + private int[] array; + + public IntArrayList(final int capacity) { + this.size = 0; + this.array = new int[capacity]; + } + + public int size() { + return size; + } + + public boolean add(final int number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public int removeLast() { + if (size == 0) { + throw new NoSuchElementException(); + } + --size; + return array[size]; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return size == 0; + } + + private void grow(final int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final int[] t = new int[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } + + public int[] toArray() { + return Arrays.copyOf(array, size); + } + + public static final IntArrayList EMPTY = + new IntArrayList(0) { + + @Override + public boolean add(int number) { + throw new UnsupportedOperationException(); + } + + @Override + public int removeLast() { + throw new UnsupportedOperationException(); + } + }; +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java new file mode 100644 index 0000000000000..a51291f9d8441 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of longs. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class LongArrayList { + + private int size; + private long[] array; + + public LongArrayList(int capacity) { + this.size = 0; + this.array = new long[capacity]; + } + + public int size() { + return size; + } + + public boolean add(long number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public long removeLong(int index) { + if (index >= size) { + throw new IndexOutOfBoundsException( + "Index (" + index + ") is greater than or equal to list size (" + size + ")"); + } + final long old = array[index]; + size--; + if (index != size) { + System.arraycopy(array, index + 1, array, index, size - index); + } + return old; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public long[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final long[] t = new long[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java new file mode 100644 index 0000000000000..3f2f8976b69bf --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; + +import static java.lang.String.format; + +/** + * Utils to calculate nested type position. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.utils.NestedPositionUtil}). + */ +public class NestedPositionUtil { + + /** + * Calculate row offsets according to column's max repetition level, definition level, value's + * repetition level and definition level. Each row has three situation: + *

  • Row is not defined,because it's optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Row is null + *
  • Row is defined and not empty. + * + * @param field field that contains the row column message include max repetition level and + * definition level. + * @param fieldRepetitionLevels int array with each value's repetition level. + * @param fieldDefinitionLevels int array with each value's definition level. + * @return {@link RowPosition} contains collections row count and isNull array. + */ + public static RowPosition calculateRowOffsets( + ParquetField field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) { + int rowDefinitionLevel = field.getDefinitionLevel(); + int rowRepetitionLevel = field.getRepetitionLevel(); + int nullValuesCount = 0; + BooleanArrayList nullRowFlags = new BooleanArrayList(0); + for (int i = 0; i < fieldDefinitionLevels.length; i++) { + // If a row's last field is an array, the repetition levels for the array's items will + // be larger than the parent row's repetition level, so we need to skip those values. + if (fieldRepetitionLevels[i] > rowRepetitionLevel) { + continue; + } + + if (fieldDefinitionLevels[i] >= rowDefinitionLevel) { + // current row is defined and not empty + nullRowFlags.add(false); + } else { + // current row is null + nullRowFlags.add(true); + nullValuesCount++; + } + } + if (nullValuesCount == 0) { + return new RowPosition(null, fieldDefinitionLevels.length); + } + return new RowPosition(nullRowFlags.toArray(), nullRowFlags.size()); + } + + /** + * Calculate the collection's offsets according to column's max repetition level, definition + * level, value's repetition level and definition level. Each collection (Array or Map) has four + * situation: + *
  • Collection is not defined, because optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Collection is null + *
  • Collection is defined but empty + *
  • Collection is defined and not empty. In this case offset value is increased by the number + * of elements in that collection + * + * @param field field that contains array/map column message include max repetition level and + * definition level. + * @param definitionLevels int array with each value's definition level. + * @param repetitionLevels int array with each value's repetition level. + * @return {@link CollectionPosition} contains collections offset array, length array and isNull + * array. + */ + public static CollectionPosition calculateCollectionOffsets( + ParquetField field, int[] definitionLevels, int[] repetitionLevels) { + int collectionDefinitionLevel = field.getDefinitionLevel(); + int collectionRepetitionLevel = field.getRepetitionLevel() + 1; + int offset = 0; + int valueCount = 0; + LongArrayList offsets = new LongArrayList(0); + offsets.add(offset); + BooleanArrayList emptyCollectionFlags = new BooleanArrayList(0); + BooleanArrayList nullCollectionFlags = new BooleanArrayList(0); + int nullValuesCount = 0; + for (int i = 0; + i < definitionLevels.length; + i = getNextCollectionStartIndex(repetitionLevels, collectionRepetitionLevel, i)) { + valueCount++; + if (definitionLevels[i] >= collectionDefinitionLevel - 1) { + boolean isNull = + isOptionalFieldValueNull(definitionLevels[i], collectionDefinitionLevel); + nullCollectionFlags.add(isNull); + nullValuesCount += isNull ? 1 : 0; + // definitionLevels[i] > collectionDefinitionLevel => Collection is defined and not + // empty + // definitionLevels[i] == collectionDefinitionLevel => Collection is defined but + // empty + if (definitionLevels[i] > collectionDefinitionLevel) { + emptyCollectionFlags.add(false); + offset += getCollectionSize(repetitionLevels, collectionRepetitionLevel, i + 1); + } else if (definitionLevels[i] == collectionDefinitionLevel) { + offset++; + emptyCollectionFlags.add(true); + } else { + offset++; + emptyCollectionFlags.add(false); + } + offsets.add(offset); + } else { + // when definitionLevels[i] < collectionDefinitionLevel - 1, it means the collection + // is + // not defined, but we need to regard it as null to avoid getting value wrong. + nullCollectionFlags.add(true); + nullValuesCount++; + offsets.add(++offset); + emptyCollectionFlags.add(false); + } + } + long[] offsetsArray = offsets.toArray(); + long[] length = calculateLengthByOffsets(emptyCollectionFlags.toArray(), offsetsArray); + if (nullValuesCount == 0) { + return new CollectionPosition(null, offsetsArray, length, valueCount); + } + return new CollectionPosition( + nullCollectionFlags.toArray(), offsetsArray, length, valueCount); + } + + public static boolean isOptionalFieldValueNull(int definitionLevel, int maxDefinitionLevel) { + return definitionLevel == maxDefinitionLevel - 1; + } + + public static long[] calculateLengthByOffsets( + boolean[] collectionIsEmpty, long[] arrayOffsets) { + LongArrayList lengthList = new LongArrayList(arrayOffsets.length); + for (int i = 0; i < arrayOffsets.length - 1; i++) { + long offset = arrayOffsets[i]; + long length = arrayOffsets[i + 1] - offset; + if (length < 0) { + throw new IllegalArgumentException( + format( + "Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", + i, arrayOffsets[i], i + 1, arrayOffsets[i + 1])); + } + if (collectionIsEmpty[i]) { + length = 0; + } + lengthList.add(length); + } + return lengthList.toArray(); + } + + private static int getNextCollectionStartIndex( + int[] repetitionLevels, int maxRepetitionLevel, int elementIndex) { + do { + elementIndex++; + } while (hasMoreElements(repetitionLevels, elementIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, elementIndex)); + return elementIndex; + } + + /** This method is only called for non-empty collections. */ + private static int getCollectionSize( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + int size = 1; + while (hasMoreElements(repetitionLevels, nextIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, nextIndex)) { + // Collection elements cannot only be primitive, but also can have nested structure + // Counting only elements which belong to current collection, skipping inner elements of + // nested collections/structs + if (repetitionLevels[nextIndex] <= maxRepetitionLevel) { + size++; + } + nextIndex++; + } + return size; + } + + private static boolean isNotCollectionBeginningMarker( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + return repetitionLevels[nextIndex] >= maxRepetitionLevel; + } + + private static boolean hasMoreElements(int[] repetitionLevels, int nextIndex) { + return nextIndex < repetitionLevels.length; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java deleted file mode 100644 index bed7c71a1848b..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.types.variant.Variant; - -public class ColumnarGroupArrayData implements ArrayData { - - WritableColumnVector vector; - int rowId; - - public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) { - this.vector = vector; - this.rowId = rowId; - } - - @Override - public int size() { - if (vector == null) { - return 0; - } - - if (vector instanceof HeapRowColumnVector) { - // assume all fields have the same size - if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) { - return 0; - } - return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size(); - } - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean isNullAt(int index) { - if (vector == null) { - return true; - } - - if (vector instanceof HeapRowColumnVector) { - return ((HeapRowColumnVector) vector).vectors == null; - } - - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean getBoolean(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float getFloat(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public StringData getString(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public DecimalData getDecimal(int index, int precision, int scale) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public TimestampData getTimestamp(int index, int precision) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RawValueData getRawValue(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public Variant getVariant(int i) { - throw new UnsupportedOperationException("Variant is not supported yet."); - } - - @Override - public byte[] getBinary(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public ArrayData getArray(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public MapData getMap(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RowData getRow(int index, int numFields) { - return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java deleted file mode 100644 index 69cb6feca13e4..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupMapData implements MapData { - - WritableColumnVector keyVector; - WritableColumnVector valueVector; - int rowId; - - public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) { - this.keyVector = keyVector; - this.valueVector = valueVector; - this.rowId = rowId; - } - - @Override - public int size() { - if (keyVector == null) { - return 0; - } - - if (keyVector instanceof HeapArrayVector) { - return ((HeapArrayVector) keyVector).getArray(rowId).size(); - } - throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector"); - } - - @Override - public ArrayData keyArray() { - return ((HeapArrayVector) keyVector).getArray(rowId); - } - - @Override - public ArrayData valueArray() { - if (valueVector instanceof HeapArrayVector) { - return ((HeapArrayVector) valueVector).getArray(rowId); - } else if (valueVector instanceof HeapArrayGroupColumnVector) { - return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId); - } - throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector"); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java deleted file mode 100644 index bfd0978f31565..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.types.RowKind; -import org.apache.flink.types.variant.Variant; - -public class ColumnarGroupRowData implements RowData { - - HeapRowColumnVector vector; - int rowId; - int index; - - public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) { - this.vector = vector; - this.rowId = rowId; - this.index = index; - } - - @Override - public int getArity() { - return vector.vectors.length; - } - - @Override - public RowKind getRowKind() { - return RowKind.INSERT; - } - - @Override - public void setRowKind(RowKind rowKind) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public boolean isNullAt(int pos) { - return - vector.vectors[pos].isNullAt(rowId) - || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index); - } - - @Override - public boolean getBoolean(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index); - } - - @Override - public byte getByte(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index); - } - - @Override - public short getShort(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index); - } - - @Override - public int getInt(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index); - } - - @Override - public long getLong(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index); - } - - @Override - public float getFloat(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index); - } - - @Override - public double getDouble(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index); - } - - @Override - public StringData getString(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index); - } - - @Override - public DecimalData getDecimal(int pos, int i1, int i2) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2); - } - - @Override - public TimestampData getTimestamp(int pos, int i1) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1); - } - - @Override - public RawValueData getRawValue(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index); - } - - @Override - public byte[] getBinary(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index); - } - - @Override - public ArrayData getArray(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index); - } - - @Override - public MapData getMap(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index); - } - - @Override - public RowData getRow(int pos, int numFields) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields); - } - - @Override - public Variant getVariant(int i) { - throw new UnsupportedOperationException("Variant is not supported yet."); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java deleted file mode 100644 index 3d7d8b1f0de0f..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -/** - * This class represents a nullable heap row column vector. - */ -public class HeapArrayGroupColumnVector extends AbstractHeapVector - implements WritableColumnVector, ArrayColumnVector { - - public WritableColumnVector vector; - - public HeapArrayGroupColumnVector(int len) { - super(len); - } - - public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) { - super(len); - this.vector = vector; - } - - @Override - public ArrayData getArray(int rowId) { - return new ColumnarGroupArrayData(vector, rowId); - } - - @Override - public void reset() { - super.reset(); - vector.reset(); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java index 7db66d23d6fc8..c597bafad1ed2 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java @@ -49,6 +49,10 @@ public HeapArrayVector(int len, ColumnVector vector) { this.child = vector; } + public int getLen() { + return this.isNull.length; + } + public int getSize() { return size; } @@ -57,8 +61,35 @@ public void setSize(int size) { this.size = size; } - public int getLen() { - return this.isNull.length; + // --------------------------------------------------------------------------------------------- + // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths} + // and {@code child} fields so legacy callers continue to work; the new {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + // future Flink-2.1-style caller use these accessors. + // --------------------------------------------------------------------------------------------- + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public ColumnVector getChild() { + return child; + } + + public void setChild(ColumnVector child) { + this.child = child; } @Override diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index f828ae9dffa78..293604e02c4aa 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -19,35 +19,103 @@ package org.apache.hudi.table.format.cow.vector; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.ColumnarMapData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; /** * This class represents a nullable heap map column vector. + * + *

    Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from + * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward + * compatibility: the {@code keys} / {@code values} fields are typed + * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing + * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original + * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use + * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}. */ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { - private final WritableColumnVector keys; - private final WritableColumnVector values; + private WritableColumnVector keys; + private WritableColumnVector values; + + // --------------------------------------------------------------------------------------------- + // Flink 2.1 Dremel-style state. Populated by {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and + // consumed by {@link #getMap(int)}. + // --------------------------------------------------------------------------------------------- + private long[] offsets; + private long[] lengths; + private int size; public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) { super(len); + this.offsets = new long[len]; + this.lengths = new long[len]; this.keys = keys; this.values = values; } + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + public WritableColumnVector getKeys() { return keys; } + public void setKeys(WritableColumnVector keys) { + this.keys = keys; + } + public WritableColumnVector getValues() { return values; } + public void setValues(WritableColumnVector values) { + this.values = values; + } + + /** + * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract + * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}. + */ + public ColumnVector getKeyColumnVector() { + return keys; + } + + /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */ + public ColumnVector getValueColumnVector() { + return values; + } + @Override public MapData getMap(int rowId) { - return new ColumnarGroupMapData(keys, values, rowId); + long offset = offsets[rowId]; + long length = lengths[rowId]; + return new ColumnarMapData(keys, values, (int) offset, (int) length); } } diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java index ae194e4e6ab05..0c640ce92ee40 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java @@ -37,6 +37,21 @@ public HeapRowColumnVector(int len, WritableColumnVector... vectors) { this.vectors = vectors; } + /** + * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code + * vectors} field so legacy callers continue to work; the new {@link + * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + * future Flink-2.1-style caller use this accessor. + */ + public WritableColumnVector[] getFields() { + return vectors; + } + + /** Counterpart of {@link #getFields()}. */ + public void setFields(WritableColumnVector[] fields) { + this.vectors = fields; + } + @Override public ColumnarRowData getRow(int i) { ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors)); diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java index 98b5e61050898..a37b88352cf52 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java @@ -18,21 +18,29 @@ package org.apache.hudi.table.format.cow.vector; +import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.columnar.vector.BytesColumnVector; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.DecimalColumnVector; +import org.apache.flink.table.data.columnar.vector.Dictionary; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.table.data.columnar.vector.LongColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector; + +import static org.apache.flink.util.Preconditions.checkArgument; /** - * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to - * provide {@link DecimalColumnVector} interface. - * - *

    Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector} - * because it is not public. + * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide + * {@link DecimalColumnVector} interface. */ -public class ParquetDecimalVector implements DecimalColumnVector { +public class ParquetDecimalVector + implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector { - public final ColumnVector vector; + private final ColumnVector vector; public ParquetDecimalVector(ColumnVector vector) { this.vector = vector; @@ -40,15 +48,180 @@ public ParquetDecimalVector(ColumnVector vector) { @Override public DecimalData getDecimal(int i, int precision, int scale) { - return DecimalData.fromUnscaledBytes( - ((BytesColumnVector) vector).getBytes(i).getBytes(), - precision, - scale); + if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) { + return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale); + } else if (ParquetSchemaConverter.is64BitDecimal(precision) + && vector instanceof LongColumnVector) { + return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale); + } else { + checkArgument( + vector instanceof BytesColumnVector, + "Reading decimal type occur unsupported vector type: %s", + vector.getClass()); + return DecimalData.fromUnscaledBytes( + ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale); + } + } + + public ColumnVector getVector() { + return vector; } @Override public boolean isNullAt(int i) { return vector.isNullAt(i); } -} + @Override + public void reset() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).reset(); + } + } + + @Override + public void setNullAt(int rowId) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNullAt(rowId); + } + } + + @Override + public void setNulls(int rowId, int count) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNulls(rowId, count); + } + } + + @Override + public void fillWithNulls() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).fillWithNulls(); + } + } + + @Override + public void setDictionary(Dictionary dictionary) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setDictionary(dictionary); + } + } + + @Override + public boolean hasDictionary() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).hasDictionary(); + } + return false; + } + + @Override + public WritableIntVector reserveDictionaryIds(int capacity) { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).reserveDictionaryIds(capacity); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public WritableIntVector getDictionaryIds() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).getDictionaryIds(); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public Bytes getBytes(int i) { + if (vector instanceof WritableBytesVector) { + return ((WritableBytesVector) vector).getBytes(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void appendBytes(int rowId, byte[] value, int offset, int length) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length); + } + } + + @Override + public void fill(byte[] value) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).fill(value); + } + } + + @Override + public int getInt(int i) { + if (vector instanceof WritableIntVector) { + return ((WritableIntVector) vector).getInt(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setInt(int rowId, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInt(rowId, value); + } + } + + @Override + public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void setInts(int rowId, int count, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, value); + } + } + + @Override + public void setInts(int rowId, int count, int[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).fill(value); + } + } + + @Override + public long getLong(int i) { + if (vector instanceof WritableLongVector) { + return ((WritableLongVector) vector).getLong(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setLong(int rowId, long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLong(rowId, value); + } + } + + @Override + public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).fill(value); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java new file mode 100644 index 0000000000000..fcdedfbc9d71d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent collection's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.CollectionPosition}). + */ +public class CollectionPosition { + @Nullable private final boolean[] isNull; + private final long[] offsets; + private final long[] length; + private final int valueCount; + + public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { + this.isNull = isNull; + this.offsets = offsets; + this.length = length; + this.valueCount = valueCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public long[] getOffsets() { + return offsets; + } + + public long[] getLength() { + return length; + } + + public int getValueCount() { + return valueCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java new file mode 100644 index 0000000000000..fe95419ac3218 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +/** + * To delegate repetition level and definition level. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.LevelDelegation}). + */ +public class LevelDelegation { + private final int[] repetitionLevel; + private final int[] definitionLevel; + + public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) { + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + } + + public int[] getRepetitionLevel() { + return repetitionLevel; + } + + public int[] getDefinitionLevel() { + return definitionLevel; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java new file mode 100644 index 0000000000000..5438b67973238 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent struct's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.RowPosition}). + */ +public class RowPosition { + @Nullable private final boolean[] isNull; + private final int positionsCount; + + public RowPosition(boolean[] isNull, int positionsCount) { + this.isNull = isNull; + this.positionsCount = positionsCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public int getPositionsCount() { + return positionsCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java deleted file mode 100644 index 6a8a01b74946a..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayVector; -import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; -import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.table.types.logical.ArrayType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Array {@link ColumnReader}. - */ -public class ArrayColumnReader extends BaseVectorizedColumnReader { - - // The value read in last time - private Object lastValue; - - // flag to indicate if there is no data in parquet data page - private boolean eof = false; - - // flag to indicate if it's the first time to read parquet data page with this instance - boolean isFirstRow = true; - - public ArrayColumnReader( - ColumnDescriptor descriptor, - PageReader pageReader, - boolean isUtcTimestamp, - Type type, - LogicalType logicalType) - throws IOException { - super(descriptor, pageReader, isUtcTimestamp, type, logicalType); - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayVector lcv = (HeapArrayVector) vector; - // before readBatch, initial the size of offsets & lengths as the default value, - // the actual size will be assigned in setChildrenInfo() after reading complete. - lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - // Because the length of ListColumnVector.child can't be known now, - // the valueList will save all data for ListColumnVector temporary. - List valueList = new ArrayList<>(); - - LogicalType category = ((ArrayType) logicalType).getElementType(); - - // read the first row in parquet data page, this will be only happened once for this - // instance - if (isFirstRow) { - if (!fetchNextValue(category)) { - return; - } - isFirstRow = false; - } - - int index = collectDataFromParquetPage(readNumber, lcv, valueList, category); - - // Convert valueList to array for the ListColumnVector.child - fillColumnVector(category, lcv, valueList, index); - } - - /** - * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating - * if there is more values to read (true). - * - * @param category - * @return boolean - * @throws IOException - */ - private boolean fetchNextValue(LogicalType category) throws IOException { - int left = readPageIfNeed(); - if (left > 0) { - // get the values of repetition and definitionLevel - readRepetitionAndDefinitionLevels(); - // read the data if it isn't null - if (definitionLevel == maxDefLevel) { - if (isCurrentPageDictionaryEncoded) { - lastValue = dataColumn.readValueDictionaryId(); - } else { - lastValue = readPrimitiveTypedRow(category); - } - } else { - lastValue = null; - } - return true; - } else { - eof = true; - return false; - } - } - - private int readPageIfNeed() throws IOException { - // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); - } - return leftInPage; - } - - // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper - // TODO Reduce the duplicated code - private Object readPrimitiveTypedRow(LogicalType category) { - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dataColumn.readString(); - case BOOLEAN: - return dataColumn.readBoolean(); - case TIME_WITHOUT_TIME_ZONE: - case DATE: - case INTEGER: - return dataColumn.readInteger(); - case TINYINT: - return dataColumn.readTinyInt(); - case SMALLINT: - return dataColumn.readSmallInt(); - case BIGINT: - return dataColumn.readLong(); - case FLOAT: - return dataColumn.readFloat(); - case DOUBLE: - return dataColumn.readDouble(); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dataColumn.readInteger(); - case INT64: - return dataColumn.readLong(); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return dataColumn.readString(); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dataColumn.readTimestamp(); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { - if (dictionaryValue == null) { - return null; - } - - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dictionary.readString(dictionaryValue); - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case INTEGER: - return dictionary.readInteger(dictionaryValue); - case BOOLEAN: - return dictionary.readBoolean(dictionaryValue) ? 1 : 0; - case DOUBLE: - return dictionary.readDouble(dictionaryValue); - case FLOAT: - return dictionary.readFloat(dictionaryValue); - case TINYINT: - return dictionary.readTinyInt(dictionaryValue); - case SMALLINT: - return dictionary.readSmallInt(dictionaryValue); - case BIGINT: - return dictionary.readLong(dictionaryValue); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dictionary.readInteger(dictionaryValue); - case INT64: - return dictionary.readLong(dictionaryValue); - case FIXED_LEN_BYTE_ARRAY: - case BINARY: - return dictionary.readString(dictionaryValue); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dictionary.readTimestamp(dictionaryValue); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - /** - * Collects data from a parquet page and returns the final row index where it stopped. The - * returned index can be equal to or less than total. - * - * @param total maximum number of rows to collect - * @param lcv column vector to do initial setup in data collection time - * @param valueList collection of values that will be fed into the vector later - * @param category - * @return int - * @throws IOException - */ - private int collectDataFromParquetPage( - int total, HeapArrayVector lcv, List valueList, LogicalType category) - throws IOException { - int index = 0; - /* - * Here is a nested loop for collecting all values from a parquet page. - * A column of array type can be considered as a list of lists, so the two loops are as below: - * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.: - * [0, 2, 3] <- index: 0 - * [NULL, 3, 4] <- index: 1 - * - * 2. The inner loop iterates on values within a row (sets all data from parquet data page - * for an element in ListColumnVector), so fetchNextValue returns values one-by-one: - * 0, 2, 3, NULL, 3, 4 - * - * As described below, the repetition level (repetitionLevel != 0) - * can be used to decide when we'll start to read values for the next list. - */ - while (!eof && index < total) { - // add element to ListColumnVector one by one - lcv.offsets[index] = valueList.size(); - /* - * Let's collect all values for a single list. - * Repetition level = 0 means that a new list started there in the parquet page, - * in that case, let's exit from the loop, and start to collect value for a new list. - */ - do { - /* - * Definition level = 0 when a NULL value was returned instead of a list - * (this is not the same as a NULL value in of a list). - */ - if (definitionLevel == 0) { - lcv.setNullAt(index); - } - valueList.add( - isCurrentPageDictionaryEncoded - ? dictionaryDecodeValue(category, (Integer) lastValue) - : lastValue); - } while (fetchNextValue(category) && (repetitionLevel != 0)); - - lcv.lengths[index] = valueList.size() - lcv.offsets[index]; - index++; - } - return index; - } - - /** - * The lengths & offsets will be initialized as default size (1024), it should be set to the - * actual size according to the element number. - */ - private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) { - lcv.setSize(itemNum); - long[] lcvLength = new long[elementNum]; - long[] lcvOffset = new long[elementNum]; - System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum); - System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum); - lcv.lengths = lcvLength; - lcv.offsets = lcvOffset; - } - - private void fillColumnVector( - LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) { - int total = valueList.size(); - setChildrenInfo(lcv, total, elementNum); - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - lcv.child = new HeapBytesVector(total); - ((HeapBytesVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (src == null) { - ((HeapBytesVector) lcv.child).setNullAt(i); - } else { - ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length); - } - } - break; - case BOOLEAN: - lcv.child = new HeapBooleanVector(total); - ((HeapBooleanVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapBooleanVector) lcv.child).setNullAt(i); - } else { - ((HeapBooleanVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TINYINT: - lcv.child = new HeapByteVector(total); - ((HeapByteVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapByteVector) lcv.child).setNullAt(i); - } else { - ((HeapByteVector) lcv.child).vector[i] = - (byte) ((List) valueList).get(i).intValue(); - } - } - break; - case SMALLINT: - lcv.child = new HeapShortVector(total); - ((HeapShortVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapShortVector) lcv.child).setNullAt(i); - } else { - ((HeapShortVector) lcv.child).vector[i] = - (short) ((List) valueList).get(i).intValue(); - } - } - break; - case INTEGER: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - lcv.child = new HeapIntVector(total); - ((HeapIntVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) lcv.child).setNullAt(i); - } else { - ((HeapIntVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case FLOAT: - lcv.child = new HeapFloatVector(total); - ((HeapFloatVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapFloatVector) lcv.child).setNullAt(i); - } else { - ((HeapFloatVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case BIGINT: - lcv.child = new HeapLongVector(total); - ((HeapLongVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) lcv.child).setNullAt(i); - } else { - ((HeapLongVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case DOUBLE: - lcv.child = new HeapDoubleVector(total); - ((HeapDoubleVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapDoubleVector) lcv.child).setNullAt(i); - } else { - ((HeapDoubleVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - lcv.child = new HeapTimestampVector(total); - ((HeapTimestampVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapTimestampVector) lcv.child).setNullAt(i); - } else { - ((HeapTimestampVector) lcv.child) - .setTimestamp(i, ((List) valueList).get(i)); - } - } - break; - case DECIMAL: - PrimitiveType.PrimitiveTypeName primitiveTypeName = - descriptor.getPrimitiveType().getPrimitiveTypeName(); - switch (primitiveTypeName) { - case INT32: - lcv.child = new ParquetDecimalVector(new HeapIntVector(total)); - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - case INT64: - lcv.child = new ParquetDecimalVector(new HeapLongVector(total)); - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - default: - lcv.child = new ParquetDecimalVector(new HeapBytesVector(total)); - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (valueList.get(i) == null) { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .appendBytes(i, src, 0, src.length); - } - } - break; - } - break; - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java deleted file mode 100644 index df7c5d85bc4ab..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}. - */ -public class ArrayGroupReader implements ColumnReader { - - private final ColumnReader fieldReader; - - public ArrayGroupReader(ColumnReader fieldReader) { - this.fieldReader = fieldReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector; - - fieldReader.readToVector(readNumber, rowColumnVector.vector); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java deleted file mode 100644 index 6d743530fccc7..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Map {@link ColumnReader}. - */ -public class MapColumnReader implements ColumnReader { - - private final ArrayColumnReader keyReader; - private final ColumnReader valueReader; - - public MapColumnReader( - ArrayColumnReader keyReader, ColumnReader valueReader) { - this.keyReader = keyReader; - this.valueReader = valueReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector; - AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys()); - keyReader.readToVector(readNumber, mapColumnVector.getKeys()); - valueReader.readToVector(readNumber, mapColumnVector.getValues()); - for (int i = 0; i < keyArrayColumnVector.getLen(); i++) { - if (keyArrayColumnVector.isNullAt(i)) { - mapColumnVector.setNullAt(i); - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java new file mode 100644 index 0000000000000..00abceeee4fab --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.NestedPositionUtil; +import org.apache.hudi.table.format.cow.vector.HeapArrayVector; +import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; +import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VariantType; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.Preconditions; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code + * Row}). Resolves nested structures using Dremel striping/assembly; see the + * striping and assembly algorithms from the Dremel paper. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream: + * + *

      + *
    • Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code + * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code + * HeapMapVector}/{@code HeapArrayVector}. + *
    • Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a + * {@link RowType} may contain {@code null} children — meaning the corresponding logical + * field is absent from the Parquet file. Those slots are passed through unchanged and do + * not contribute to the row's repetition/definition-level stream. + *
    + */ +public class NestedColumnReader implements ColumnReader { + + private final Map columnReaders; + private final boolean isUtcTimestamp; + + private final PageReadStore pages; + + private final ParquetField field; + + public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) { + this.isUtcTimestamp = isUtcTimestamp; + this.pages = pages; + this.field = field; + this.columnReaders = new HashMap<>(); + } + + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + readData(field, readNumber, vector, false); + } + + private Tuple2 readData( + ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException { + if (field.getType() instanceof RowType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof VariantType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) { + return readMap((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof ArrayType) { + return readArray((ParquetGroupField) field, readNumber, vector, inside); + } else { + return readPrimitive((ParquetPrimitiveField) field, readNumber, vector); + } + } + + private Tuple2 readRow( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector; + LevelDelegation levelDelegation = null; + List children = field.getChildren(); + WritableColumnVector[] childrenVectors = heapRowVector.getFields(); + WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length]; + for (int i = 0; i < children.size(); i++) { + ParquetField child = children.get(i); + if (child == null) { + // Schema-evolution: the logical field is not present in the Parquet file. The slot + // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector + // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by + // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null + // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays + // NULL instead of reverting to the type's zero value. Skip contributing to the level stream. + childrenVectors[i].fillWithNulls(); + finalChildrenVectors[i] = childrenVectors[i]; + continue; + } + Tuple2 tuple = + readData(child, readNumber, childrenVectors[i], true); + levelDelegation = tuple.f0; + finalChildrenVectors[i] = tuple.f1; + } + if (levelDelegation == null) { + throw new FlinkRuntimeException( + String.format("Row field does not have any non-null children: %s.", field)); + } + + RowPosition rowPosition = + NestedPositionUtil.calculateRowOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If row was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors); + } else { + heapRowVector.setFields(finalChildrenVectors); + } + + if (rowPosition.getIsNull() != null) { + setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); + } + + // Hudi-specific: collapse a present row whose every child is null into a null row, so that a + // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the + // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely + // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the + // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + int rowCount = rowPosition.getPositionsCount(); + for (int j = 0; j < rowCount; j++) { + if (heapRowVector.isNullAt(j)) { + continue; + } + boolean allChildrenNull = true; + for (WritableColumnVector child : finalChildrenVectors) { + if (!child.isNullAt(j)) { + allChildrenNull = false; + break; + } + } + if (allChildrenNull) { + heapRowVector.setNullAt(j); + } + } + return Tuple2.of(levelDelegation, heapRowVector); + } + + private Tuple2 readMap( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapMapColumnVector mapVector = (HeapMapColumnVector) vector; + mapVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 2, + "Maps must have two type parameters, found %s", + children.size()); + Tuple2 keyTuple = + readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true); + Tuple2 valueTuple = + readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true); + + LevelDelegation levelDelegation = keyTuple.f0; + + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If map was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1); + } else { + mapVector.setKeys(keyTuple.f1); + mapVector.setValues(valueTuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), mapVector); + } + + mapVector.setLengths(collectionPosition.getLength()); + mapVector.setOffsets(collectionPosition.getOffsets()); + + return Tuple2.of(levelDelegation, mapVector); + } + + private Tuple2 readArray( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapArrayVector arrayVector = (HeapArrayVector) vector; + arrayVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 1, + "Arrays must have a single type parameter, found %s", + children.size()); + Tuple2 tuple = + readData(children.get(0), readNumber, arrayVector.getChild(), true); + + LevelDelegation levelDelegation = tuple.f0; + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If array was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1); + } else { + arrayVector.setChild(tuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), arrayVector); + } + arrayVector.setLengths(collectionPosition.getLength()); + arrayVector.setOffsets(collectionPosition.getOffsets()); + return Tuple2.of(levelDelegation, arrayVector); + } + + private Tuple2 readPrimitive( + ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException { + ColumnDescriptor descriptor = field.getDescriptor(); + NestedPrimitiveColumnReader reader = columnReaders.get(descriptor); + if (reader == null) { + reader = + new NestedPrimitiveColumnReader( + descriptor, + pages.getPageReader(descriptor), + isUtcTimestamp, + descriptor.getPrimitiveType(), + field.getType()); + columnReaders.put(descriptor, reader); + } + WritableColumnVector writableColumnVector = + reader.readAndNewVector(readNumber, (WritableColumnVector) vector); + return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); + } + + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { + for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { + if (nullFlags[index]) { + vector.setNullAt(index); + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java new file mode 100644 index 0000000000000..a18520c3b5cd5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -0,0 +1,638 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.IntArrayList; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; + +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; +import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; +import static org.apache.parquet.column.ValuesType.VALUES; + +/** + * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package + * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} + * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code + * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- + * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it + * a faithful copy of upstream. + */ +public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); + + private final IntArrayList repetitionLevelList = new IntArrayList(0); + private final IntArrayList definitionLevelList = new IntArrayList(0); + + private final PageReader pageReader; + private final ColumnDescriptor descriptor; + private final Type type; + private final LogicalType logicalType; + + /** The dictionary, if this column has dictionary encoding. */ + private final ParquetDataColumnReader dictionary; + + /** Maximum definition level for this column. */ + private final int maxDefLevel; + + private boolean isUtcTimestamp; + + /** Total number of values read. */ + private long valuesRead; + + /** + * value that indicates the end of the current page. That is, if valuesRead == + * endOfPageValueCount, we are at the end of the page. + */ + private long endOfPageValueCount; + + /** If true, the current page is dictionary encoded. */ + private boolean isCurrentPageDictionaryEncoded; + + private int definitionLevel; + private int repetitionLevel; + + /** Repetition/Definition/Value readers. */ + private IntIterator repetitionLevelColumn; + + private IntIterator definitionLevelColumn; + private ParquetDataColumnReader dataColumn; + + /** Total values in the current page. */ + private int pageValueCount; + + // flag to indicate if there is no data in parquet data page + private boolean eof = false; + + private boolean isFirstRow = true; + + private Object lastValue; + + public NestedPrimitiveColumnReader( + ColumnDescriptor descriptor, + PageReader pageReader, + boolean isUtcTimestamp, + Type parquetType, + LogicalType logicalType) + throws IOException { + this.descriptor = descriptor; + this.type = parquetType; + this.pageReader = pageReader; + this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.isUtcTimestamp = isUtcTimestamp; + this.logicalType = logicalType; + + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); + if (dictionaryPage != null) { + try { + this.dictionary = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + parquetType.asPrimitiveType(), + dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } catch (IOException e) { + throw new IOException( + String.format("Could not decode the dictionary for %s", descriptor), e); + } + } else { + this.dictionary = null; + this.isCurrentPageDictionaryEncoded = false; + } + } + + // Not invoked directly; callers use readAndNewVector instead. + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + throw new UnsupportedOperationException("This function should not be called."); + } + + public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector) + throws IOException { + if (isFirstRow) { + if (!readValue()) { + return vector; + } + isFirstRow = false; + } + + // index to set value. + int index = 0; + int valueIndex = 0; + List valueList = new ArrayList<>(); + + // repeated type need two loops to read data. + while (!eof && index < readNumber) { + do { + valueList.add(lastValue); + valueIndex++; + } while (readValue() && (repetitionLevel != 0)); + index++; + } + + return fillColumnVector(valueIndex, valueList); + } + + public LevelDelegation getLevelDelegation() { + int[] repetition = repetitionLevelList.toArray(); + int[] definition = definitionLevelList.toArray(); + repetitionLevelList.clear(); + definitionLevelList.clear(); + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + return new LevelDelegation(repetition, definition); + } + + private boolean readValue() throws IOException { + int left = readPageIfNeed(); + if (left > 0) { + // get the values of repetition and definitionLevel + readAndSaveRepetitionAndDefinitionLevels(); + // read the data if it isn't null + if (definitionLevel == maxDefLevel) { + if (isCurrentPageDictionaryEncoded) { + int dictionaryId = dataColumn.readValueDictionaryId(); + lastValue = dictionaryDecodeValue(logicalType, dictionaryId); + } else { + lastValue = readPrimitiveTypedRow(logicalType); + } + } else { + lastValue = null; + } + return true; + } else { + eof = true; + return false; + } + } + + private void readAndSaveRepetitionAndDefinitionLevels() { + // get the values of repetition and definitionLevel + repetitionLevel = repetitionLevelColumn.nextInt(); + definitionLevel = definitionLevelColumn.nextInt(); + valuesRead++; + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + } + + private int readPageIfNeed() throws IOException { + // Compute the number of values we want to read in this page. + int leftInPage = (int) (endOfPageValueCount - valuesRead); + if (leftInPage == 0) { + // no data left in current page, load data from new page + readPage(); + leftInPage = (int) (endOfPageValueCount - valuesRead); + } + return leftInPage; + } + + private Object readPrimitiveTypedRow(LogicalType category) { + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dataColumn.readBytes(); + case BOOLEAN: + return dataColumn.readBoolean(); + case TIME_WITHOUT_TIME_ZONE: + case DATE: + case INTEGER: + return dataColumn.readInteger(); + case TINYINT: + return dataColumn.readTinyInt(); + case SMALLINT: + return dataColumn.readSmallInt(); + case BIGINT: + return dataColumn.readLong(); + case FLOAT: + return dataColumn.readFloat(); + case DOUBLE: + return dataColumn.readDouble(); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dataColumn.readInteger(); + case INT64: + return dataColumn.readLong(); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return dataColumn.readBytes(); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dataColumn.readTimestamp(); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { + if (dictionaryValue == null) { + return null; + } + + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dictionary.readBytes(dictionaryValue); + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTEGER: + return dictionary.readInteger(dictionaryValue); + case BOOLEAN: + return dictionary.readBoolean(dictionaryValue) ? 1 : 0; + case DOUBLE: + return dictionary.readDouble(dictionaryValue); + case FLOAT: + return dictionary.readFloat(dictionaryValue); + case TINYINT: + return dictionary.readTinyInt(dictionaryValue); + case SMALLINT: + return dictionary.readSmallInt(dictionaryValue); + case BIGINT: + return dictionary.readLong(dictionaryValue); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dictionary.readInteger(dictionaryValue); + case INT64: + return dictionary.readLong(dictionaryValue); + case FIXED_LEN_BYTE_ARRAY: + case BINARY: + return dictionary.readBytes(dictionaryValue); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dictionary.readTimestamp(dictionaryValue); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private WritableColumnVector fillColumnVector(int total, List valueList) { + switch (logicalType.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + HeapBytesVector heapBytesVector = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (src == null) { + heapBytesVector.setNullAt(i); + } else { + heapBytesVector.appendBytes(i, src, 0, src.length); + } + } + return heapBytesVector; + case BOOLEAN: + HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapBooleanVector.setNullAt(i); + } else { + heapBooleanVector.vector[i] = ((List) valueList).get(i); + } + } + return heapBooleanVector; + case TINYINT: + HeapByteVector heapByteVector = new HeapByteVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapByteVector.setNullAt(i); + } else { + heapByteVector.vector[i] = (byte) ((List) valueList).get(i).intValue(); + } + } + return heapByteVector; + case SMALLINT: + HeapShortVector heapShortVector = new HeapShortVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapShortVector.setNullAt(i); + } else { + heapShortVector.vector[i] = (short) ((List) valueList).get(i).intValue(); + } + } + return heapShortVector; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + HeapIntVector heapIntVector = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapIntVector.setNullAt(i); + } else { + heapIntVector.vector[i] = ((List) valueList).get(i); + } + } + return heapIntVector; + case FLOAT: + HeapFloatVector heapFloatVector = new HeapFloatVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapFloatVector.setNullAt(i); + } else { + heapFloatVector.vector[i] = ((List) valueList).get(i); + } + } + return heapFloatVector; + case BIGINT: + HeapLongVector heapLongVector = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapLongVector.setNullAt(i); + } else { + heapLongVector.vector[i] = ((List) valueList).get(i); + } + } + return heapLongVector; + case DOUBLE: + HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapDoubleVector.setNullAt(i); + } else { + heapDoubleVector.vector[i] = ((List) valueList).get(i); + } + } + return heapDoubleVector; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapTimestampVector.setNullAt(i); + } else { + heapTimestampVector.setTimestamp(i, ((List) valueList).get(i)); + } + } + return heapTimestampVector; + case DECIMAL: + PrimitiveType.PrimitiveTypeName primitiveTypeName = + descriptor.getPrimitiveType().getPrimitiveTypeName(); + switch (primitiveTypeName) { + case INT32: + HeapIntVector phiv = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phiv.setNullAt(i); + } else { + phiv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phiv); + case INT64: + HeapLongVector phlv = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phlv.setNullAt(i); + } else { + phlv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phlv); + default: + HeapBytesVector phbv = getHeapBytesVector(total, valueList); + return new ParquetDecimalVector(phbv); + } + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static HeapBytesVector getHeapBytesVector(int total, List valueList) { + HeapBytesVector phbv = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (valueList.get(i) == null) { + phbv.setNullAt(i); + } else { + phbv.appendBytes(i, src, 0, src.length); + } + } + return phbv; + } + + protected void readPage() { + DataPage page = pageReader.readPage(); + + if (page == null) { + return; + } + + page.accept( + new DataPage.Visitor() { + @Override + public Void visit(DataPageV1 dataPageV1) { + readPageV1(dataPageV1); + return null; + } + + @Override + public Void visit(DataPageV2 dataPageV2) { + readPageV2(dataPageV2); + return null; + } + }); + } + + private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) + throws IOException { + this.pageValueCount = valueCount; + this.endOfPageValueCount = valuesRead + pageValueCount; + if (dataEncoding.usesDictionary()) { + this.dataColumn = null; + if (dictionary == null) { + throw new IOException( + String.format( + "Could not read page in col %s because the dictionary was missing for encoding %s.", + descriptor, dataEncoding)); + } + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getDictionaryBasedValuesReader( + descriptor, VALUES, dictionary.getDictionary()), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } else { + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getValuesReader(descriptor, VALUES), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = false; + } + + try { + dataColumn.initFromPage(pageValueCount, in); + } catch (IOException e) { + throw new IOException(String.format("Could not read page in col %s.", descriptor), e); + } + } + + private void readPageV1(DataPageV1 page) { + ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); + ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); + this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); + this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); + try { + BytesInput bytes = page.getBytes(); + LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount); + ByteBufferInputStream in = bytes.toInputStream(); + LOG.debug("Reading repetition levels at {}.", in.position()); + rlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading definition levels at {}.", in.position()); + dlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading data at {}.", in.position()); + initDataReader(page.getValueEncoding(), in, page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private void readPageV2(DataPageV2 page) { + this.pageValueCount = page.getValueCount(); + this.repetitionLevelColumn = + newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); + this.definitionLevelColumn = + newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); + try { + LOG.debug( + "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); + initDataReader( + page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) { + try { + if (maxLevel == 0) { + return new NullIntIterator(); + } + return new RLEIntIterator( + new RunLengthBitPackingHybridDecoder( + BytesUtils.getWidthFromMaxInt(maxLevel), + new ByteArrayInputStream(bytes.toByteArray()))); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read levels in page for col %s.", descriptor), e); + } + } + + /** Utility interface to abstract over different way to read ints with different encodings. */ + interface IntIterator { + int nextInt(); + } + + /** Reading int from {@link ValuesReader}. */ + protected static final class ValuesReaderIntIterator implements IntIterator { + ValuesReader delegate; + + public ValuesReaderIntIterator(ValuesReader delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + return delegate.readInteger(); + } + } + + /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */ + protected static final class RLEIntIterator implements IntIterator { + RunLengthBitPackingHybridDecoder delegate; + + public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + try { + return delegate.readInt(); + } catch (IOException e) { + throw new ParquetDecodingException(e); + } + } + } + + /** Reading zero always. */ + protected static final class NullIntIterator implements IntIterator { + @Override + public int nextInt() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index 3572b117a6313..1826419db5d44 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -18,7 +18,9 @@ package org.apache.hudi.table.format.cow.vector.reader; +import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.table.data.RowData; @@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,6 +42,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -46,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -53,7 +59,6 @@ import java.util.Map; import java.util.stream.IntStream; -import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; @@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable { private final MessageType requestedSchema; + /** + * {@link ParquetField} tree per top-level requested column, used by + * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List, + * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for + * nested types. Entries are {@code null} for primitive top-level fields. Built once per split. + */ + private final List requestedFields; + /** * The total number of rows this RecordReader will eventually read. The sum of the rows of all * the row groups. @@ -158,6 +171,20 @@ public ParquetColumnarRowSplitReader( checkSchema(); + // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across + // row groups). Only columns with nested logical type get a non-null entry — primitive columns + // still use Hudi's specialized ColumnReaders. + MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema); + List requestedRowFields = new ArrayList<>(requestedTypes.length); + List requestedFieldNames = new ArrayList<>(requestedTypes.length); + for (int i = 0; i < requestedTypes.length; i++) { + String name = requestedSchema.getFieldName(i); + requestedRowFields.add(new RowType.RowField(name, requestedTypes[i])); + requestedFieldNames.add(name); + } + this.requestedFields = ParquetSplitReaderUtil.buildFieldsList( + requestedRowFields, requestedFieldNames, messageColumnIO); + this.writableVectors = createWritableVectors(); ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices); this.columnarBatch = generator.generate(columnVectors); @@ -340,12 +367,13 @@ private void readNextRowGroup() throws IOException { List columns = requestedSchema.getColumns(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = createColumnReader( + columnReaders[i] = ParquetSplitReaderUtil.createColumnReader( utcTimestamp, requestedTypes[i], types.get(i), columns, - pages); + pages, + requestedFields.get(i)); } totalCountLoadedSoFar += pages.getRowCount(); } diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index 861d5cb00bbe7..460bd42f9e299 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -23,12 +23,16 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY; @@ -64,6 +68,10 @@ public DefaultParquetDataColumnReader(Dictionary dict) { this.dict = dict; } + public boolean isValid() { + return isValid; + } + @Override public void initFromPage(int i, ByteBufferInputStream in) throws IOException { valuesReader.initFromPage(i, in); @@ -170,11 +178,6 @@ public int readInteger(int id) { return dict.decodeToInt(id); } - @Override - public boolean isValid() { - return isValid; - } - @Override public long readLong(int id) { return dict.decodeToLong(id); @@ -255,21 +258,115 @@ public TimestampData readTimestamp() { } } + /** + * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard + * timestamp encoding defined by Parquet's + * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy + * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations. + * (The older INT96 encoding is marked deprecated by the Parquet format spec — see + * + * LogicalTypes.md — but is still supported here via {@link TypesFromInt96PageReader} for + * backwards compatibility with files written by older Hive / Spark / Impala versions.) + * + *

    Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a + * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use + * {@link Int64TimestampColumnReader} for batched-vector efficiency. + */ + public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader { + private final boolean isUtcTimestamp; + private final ChronoUnit chronoUnit; + + public TypesFromInt64PageReader( + ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(realReader); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + public TypesFromInt64PageReader( + Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(dict); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + @Override + public TimestampData readTimestamp() { + return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit); + } + + @Override + public TimestampData readTimestamp(int id) { + return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper( boolean isDictionary, PrimitiveType parquetType, Dictionary dictionary, ValuesReader valuesReader, boolean isUtcTimestamp) { - if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { + PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName(); + if (typeName == PrimitiveType.PrimitiveTypeName.INT96) { return isDictionary ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp) : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp); - } else { - return isDictionary - ? new DefaultParquetDataColumnReader(dictionary) - : new DefaultParquetDataColumnReader(valuesReader); } + if (typeName == PrimitiveType.PrimitiveTypeName.INT64) { + ChronoUnit unit = resolveInt64TimestampUnit(parquetType); + if (unit != null) { + return isDictionary + ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit) + : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit); + } + } + return isDictionary + ? new DefaultParquetDataColumnReader(dictionary) + : new DefaultParquetDataColumnReader(valuesReader); + } + + /** + * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the + * column is a plain INT64 (not a timestamp). + * + *

    Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and + * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} + * encodings. + */ + private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { + LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); + if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + LogicalTypeAnnotation.TimeUnit unit = + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit(); + switch (unit) { + case MILLIS: + return ChronoUnit.MILLIS; + case MICROS: + return ChronoUnit.MICROS; + case NANOS: + return ChronoUnit.NANOS; + default: + return null; + } + } + OriginalType originalType = parquetType.getOriginalType(); + if (originalType == OriginalType.TIMESTAMP_MILLIS) { + return ChronoUnit.MILLIS; + } + if (originalType == OriginalType.TIMESTAMP_MICROS) { + return ChronoUnit.MICROS; + } + return null; + } + + private static TimestampData int64ToTimestamp( + boolean isUtcTimestamp, long value, ChronoUnit unit) { + Instant instant = Instant.EPOCH.plus(value, unit); + if (isUtcTimestamp) { + return TimestampData.fromInstant(instant); + } + return TimestampData.fromTimestamp(Timestamp.from(instant)); } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( @@ -284,10 +381,10 @@ public static ParquetDataColumnReader getDataColumnReaderByType( } private static TimestampData int96ToTimestamp( - boolean utcTimestamp, long nanosOfDay, int julianDay) { + boolean isUtcTimestamp, long nanosOfDay, int julianDay) { long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND); - if (utcTimestamp) { + if (isUtcTimestamp) { int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND); return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond); } else { diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java deleted file mode 100644 index 79b50487f13c1..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** - * Row {@link ColumnReader}. - */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector; - WritableColumnVector[] vectors = rowColumnVector.vectors; - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowColumnVector.setNullAt(j); - } - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java new file mode 100644 index 0000000000000..0f5e00779a2f5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +/** + * Field that represent parquet's field type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetField}). + */ +public abstract class ParquetField { + private final LogicalType type; + private final int repetitionLevel; + private final int definitionLevel; + private final boolean required; + + public ParquetField( + LogicalType type, int repetitionLevel, int definitionLevel, boolean required) { + this.type = type; + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + this.required = required; + } + + public LogicalType getType() { + return type; + } + + public int getRepetitionLevel() { + return repetitionLevel; + } + + public int getDefinitionLevel() { + return definitionLevel; + } + + public boolean isRequired() { + return required; + } + + @Override + public String toString() { + return "Field{" + + "type=" + + type + + ", repetitionLevel=" + + repetitionLevel + + ", definitionLevel=" + + definitionLevel + + ", required=" + + required + + '}'; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java new file mode 100644 index 0000000000000..f91dcca965d64 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's Group Field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension: + * entries in the {@code children} list may be {@code null} to denote a Row child that is absent + * from the parquet file but present in the requested logical schema (schema evolution). This + * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees. + */ +public class ParquetGroupField extends ParquetField { + + private final List children; + + public ParquetGroupField( + LogicalType type, + int repetitionLevel, + int definitionLevel, + boolean required, + List children) { + super(type, repetitionLevel, definitionLevel, required); + // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for + // schema-evolution missing children in ROW types. + this.children = + Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null"))); + } + + /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */ + public List getChildren() { + return children; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java new file mode 100644 index 0000000000000..f6af6f9ff479e --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.column.ColumnDescriptor; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's primitive field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}). + */ +public class ParquetPrimitiveField extends ParquetField { + + private final ColumnDescriptor descriptor; + private final int id; + + public ParquetPrimitiveField( + LogicalType type, boolean required, ColumnDescriptor descriptor, int id) { + super( + type, + descriptor.getMaxRepetitionLevel(), + descriptor.getMaxDefinitionLevel(), + required); + this.descriptor = requireNonNull(descriptor, "descriptor is required"); + this.id = id; + } + + public ColumnDescriptor getDescriptor() { + return descriptor; + } + + public int getId() { + return id; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java new file mode 100644 index 0000000000000..aff1c32917cf9 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector}, + * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's + * nested-Parquet reader (FLINK-35702). + * + *

    The accessors are wrappers over the existing public fields so legacy callers continue to + * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the + * Dremel-style read path is exercised end-to-end by integration tests in + * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes / + * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes). + */ +class TestHeapColumnVectorAccessors { + + // ----------------------------------------------------------------------------------------------- + // HeapArrayVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapArrayVectorAccessorsReflectPublicFields() { + HeapIntVector child = new HeapIntVector(4); + HeapArrayVector vector = new HeapArrayVector(2, child); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector replacementChild = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setChild(replacementChild); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(replacementChild, vector.getChild()); + assertEquals(2, vector.getSize()); + + // Backing public fields are kept in sync — preserves backward compatibility. + assertSame(offsets, vector.offsets); + assertSame(lengths, vector.lengths); + assertSame(replacementChild, vector.child); + } + + // ----------------------------------------------------------------------------------------------- + // HeapMapColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + + HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values); + + assertEquals(3, vector.getOffsets().length); + assertEquals(3, vector.getLengths().length); + } + + @Test + void heapMapColumnVectorAccessorsReflectInternalState() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector newKeys = new HeapLongVector(4); + HeapLongVector newValues = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setKeys(newKeys); + vector.setValues(newValues); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(newKeys, vector.getKeys()); + assertSame(newValues, vector.getValues()); + // The Flink-2.1-style ColumnVector accessors return the same underlying child. + assertSame(newKeys, vector.getKeyColumnVector()); + assertSame(newValues, vector.getValueColumnVector()); + assertEquals(2, vector.getSize()); + } + + // ----------------------------------------------------------------------------------------------- + // HeapRowColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() { + HeapIntVector intField = new HeapIntVector(2); + HeapLongVector longField = new HeapLongVector(2); + HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField); + + WritableColumnVector[] originalFields = vector.getFields(); + assertEquals(2, originalFields.length); + assertSame(intField, originalFields[0]); + assertSame(longField, originalFields[1]); + // Backing public field is kept in sync — preserves backward compatibility. + assertSame(originalFields, vector.vectors); + + HeapIntVector replacement = new HeapIntVector(2); + WritableColumnVector[] replacementFields = {replacement, longField}; + vector.setFields(replacementFields); + + assertSame(replacementFields, vector.getFields()); + assertSame(replacementFields, vector.vectors); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java new file mode 100644 index 0000000000000..04f5809b9dac4 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.columnar.vector.BytesColumnVector; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ParquetDecimalVector}. + */ +public class TestParquetDecimalVector { + + @Test + void testGetDecimalFromInt32Vector() { + // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true + HeapIntVector intVector = new HeapIntVector(1); + intVector.vector[0] = 12345; + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromInt64Vector() { + // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true + HeapLongVector longVector = new HeapLongVector(1); + longVector.vector[0] = 1234567890123456L; + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + DecimalData decoded = wrapped.getDecimal(0, 18, 4); + + assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtLargePrecision() { + // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path + BigDecimal original = new BigDecimal("12345678901234567890.1234567890"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 30, 10); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtSmallPrecision() { + // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the + // dispatch must fall through to the bytes branch rather than require an IntColumnVector. + BigDecimal original = new BigDecimal("123.45"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalThrowsOnUnsupportedVectorType() { + // A large-precision request must have a bytes-backed child; any other writable child is an + // illegal combination and must be surfaced via Preconditions.checkArgument. + ColumnVector unsupported = new HeapShortVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported); + + assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10)); + } + + @Test + void testIsNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + intVector.vector[0] = 1; + intVector.setNullAt(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + assertFalse(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testWritableIntRoundTrip() { + HeapIntVector intVector = new HeapIntVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setInt(0, 42); + + assertEquals(42, wrapped.getInt(0)); + assertEquals(42, intVector.vector[0]); + } + + @Test + void testWritableLongRoundTrip() { + HeapLongVector longVector = new HeapLongVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + wrapped.setLong(0, 9876543210L); + + assertEquals(9876543210L, wrapped.getLong(0)); + assertEquals(9876543210L, longVector.vector[0]); + } + + @Test + void testWritableBytesRoundTrip() { + HeapBytesVector bytesVector = new HeapBytesVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + byte[] payload = new byte[] {0x01, 0x02, 0x03}; + + wrapped.appendBytes(0, payload, 0, payload.length); + + BytesColumnVector.Bytes out = wrapped.getBytes(0); + assertEquals(payload.length, out.len); + assertEquals(0x01, out.data[out.offset]); + assertEquals(0x02, out.data[out.offset + 1]); + assertEquals(0x03, out.data[out.offset + 2]); + } + + @Test + void testResetDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(1); + intVector.setNullAt(0); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + assertTrue(wrapped.isNullAt(0)); + + wrapped.reset(); + + assertFalse(wrapped.isNullAt(0)); + } + + @Test + void testFillWithNullsDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.fillWithNulls(); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testSetNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setNullAt(0); + wrapped.setNulls(1, 1); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java new file mode 100644 index 0000000000000..ea222dad576a5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.flink.table.data.TimestampData; +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when + * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702). + * + *

    The factory is exercised end-to-end by integration tests through + * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece + * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each + * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} + * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values + * using both the values-reader and dictionary code paths. + */ +class TestParquetDataColumnReaderFactory { + + // ----------------------------------------------------------------------------------------------- + // Type dispatch + // ----------------------------------------------------------------------------------------------- + + @Test + void valuesReaderDispatchInt96TimestampUsesInt96Reader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MILLIS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MICROS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt32DoesNotUseTimestampReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + // ----------------------------------------------------------------------------------------------- + // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches) + // ----------------------------------------------------------------------------------------------- + + @Test + void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_123L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMillis), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + assertEquals(0, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + long epochMicros = 1_700_000_000_123_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMicros), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMicros / 1_000L, ts.getMillisecond()); + // 456 microseconds remain → 456_000 nanoseconds within the millisecond + assertEquals(456_000, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + long epochNanos = 1_700_000_000_123_456_789L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochNanos), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochNanos / 1_000_000L, ts.getMillisecond()); + assertEquals(456_789, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMillisFromDictionaryInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(epochMillis), true); + + TimestampData ts = reader.readTimestamp(0); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + } + + // ----------------------------------------------------------------------------------------------- + // Stubs (only the methods exercised by the dispatch + decoding tests above) + // ----------------------------------------------------------------------------------------------- + + /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */ + private static final class StubValuesReader extends ValuesReader { + private final long fixedLong; + + StubValuesReader() { + this(0L); + } + + StubValuesReader(long fixedLong) { + this.fixedLong = fixedLong; + } + + @Override + public long readLong() { + return fixedLong; + } + + @Override + public void skip() { + // unused + } + } + + /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */ + private static final class StubDictionary extends Dictionary { + private final long fixedLong; + + StubDictionary() { + this(0L); + } + + StubDictionary(long fixedLong) { + super(null); + this.fixedLong = fixedLong; + } + + @Override + public Binary decodeToBinary(int id) { + throw new UnsupportedOperationException(); + } + + @Override + public long decodeToLong(int id) { + return fixedLong; + } + + @Override + public int getMaxId() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java new file mode 100644 index 0000000000000..2b71bae4dc152 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link ParquetGroupField}. + */ +public class TestParquetGroupField { + + @Test + void testChildrenWithAllNonNullEntriesAreRetained() { + ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1); + List children = Arrays.asList(c0, c1); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertSame(c0, group.getChildren().get(0)); + assertSame(c1, group.getChildren().get(1)); + } + + @Test + void testChildrenMayContainNullForSchemaEvolution() { + // A ROW field present in the requested Flink schema but absent from the Parquet file is + // represented by a null slot in `children`. The group must allow this (Hudi-specific + // extension over Flink's ImmutableList-backed equivalent). + ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List children = Arrays.asList(present, null); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertNotNull(group.getChildren().get(0)); + assertNull(group.getChildren().get(1)); + } + + @Test + void testChildrenListIsUnmodifiable() { + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child)); + + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null)); + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0)); + } + + @Test + void testChildrenListIsDefensivelyCopied() { + // Mutations to the caller-supplied list must not be visible through the group. + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List mutable = new ArrayList<>(); + mutable.add(child); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable); + mutable.add(null); + + assertEquals(1, group.getChildren().size()); + } + + @Test + void testNullChildrenListThrows() { + assertThrows( + NullPointerException.class, + () -> new ParquetGroupField(rowType(), 0, 1, true, null)); + } + + @Test + void testEmptyChildrenListIsAllowed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList()); + + assertEquals(0, group.getChildren().size()); + } + + @Test + void testFieldMetadataIsExposed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList()); + + assertEquals(2, group.getRepetitionLevel()); + assertEquals(5, group.getDefinitionLevel()); + assertFalse(group.isRequired()); + } + + private static LogicalType rowType() { + return RowType.of(new IntType()); + } + + private static ColumnDescriptor descriptor() { + PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f"); + return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0); + } +} From abae8aafc223f5249e52854e1b6864f6e72a2b65 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Thu, 11 Jun 2026 10:59:36 +0800 Subject: [PATCH 059/255] =?UTF-8?q?feat(flink):=20Backport=20Flink=202.1?= =?UTF-8?q?=20Dremel=20nested=20Parquet=20reader=20to=20hudi-=E2=80=A6=20(?= =?UTF-8?q?#18959)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(flink): Backport Flink 2.1 Dremel nested Parquet reader to hudi-flink2.0.x * fix schema evolution for nested reader (cherry picked from commit 7f3676be39790fbc56a899e75edfa57b48c3e743) --- .../cow/vector/reader/NestedColumnReader.java | 8 +- .../cow/vector/reader/NestedColumnReader.java | 8 +- .../cow/vector/reader/NestedColumnReader.java | 8 +- .../format/cow/ParquetSplitReaderUtil.java | 680 +++++++++++------- .../format/cow/utils/BooleanArrayList.java | 69 ++ .../table/format/cow/utils/IntArrayList.java | 93 +++ .../table/format/cow/utils/LongArrayList.java | 83 +++ .../format/cow/utils/NestedPositionUtil.java | 209 ++++++ .../cow/vector/ColumnarGroupArrayData.java | 179 ----- .../cow/vector/ColumnarGroupMapData.java | 63 -- .../cow/vector/ColumnarGroupRowData.java | 138 ---- .../vector/HeapArrayGroupColumnVector.java | 53 -- .../format/cow/vector/HeapArrayVector.java | 31 + .../cow/vector/HeapMapColumnVector.java | 74 +- .../cow/vector/HeapRowColumnVector.java | 15 + .../cow/vector/ParquetDecimalVector.java | 197 ++++- .../vector/position/CollectionPosition.java | 57 ++ .../cow/vector/position/LevelDelegation.java | 43 ++ .../cow/vector/position/RowPosition.java | 45 ++ .../cow/vector/reader/ArrayColumnReader.java | 473 ------------ .../cow/vector/reader/ArrayGroupReader.java | 45 -- .../cow/vector/reader/MapColumnReader.java | 55 -- .../cow/vector/reader/NestedColumnReader.java | 281 ++++++++ .../reader/NestedPrimitiveColumnReader.java | 638 ++++++++++++++++ .../reader/ParquetColumnarRowSplitReader.java | 34 +- .../ParquetDataColumnReaderFactory.java | 112 ++- .../cow/vector/reader/RowColumnReader.java | 63 -- .../format/cow/vector/type/ParquetField.java | 72 ++ .../cow/vector/type/ParquetGroupField.java | 59 ++ .../vector/type/ParquetPrimitiveField.java | 55 ++ .../vector/TestHeapColumnVectorAccessors.java | 138 ++++ .../cow/vector/TestParquetDecimalVector.java | 187 +++++ .../TestParquetDataColumnReaderFactory.java | 270 +++++++ .../vector/type/TestParquetGroupField.java | 132 ++++ 34 files changed, 3288 insertions(+), 1379 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java delete mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index b4f04b20c4772..2e0bd744cab1c 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -113,9 +113,13 @@ private Tuple2 readRow( for (int i = 0; i < children.size(); i++) { ParquetField child = children.get(i); if (child == null) { - // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // Schema-evolution: the logical field is not present in the Parquet file. The slot // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector - // (ROW branch); keep it as is and skip contributing to the level stream. + // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by + // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null + // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays + // NULL instead of reverting to the type's zero value. Skip contributing to the level stream. + childrenVectors[i].fillWithNulls(); finalChildrenVectors[i] = childrenVectors[i]; continue; } diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index b4f04b20c4772..2e0bd744cab1c 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -113,9 +113,13 @@ private Tuple2 readRow( for (int i = 0; i < children.size(); i++) { ParquetField child = children.get(i); if (child == null) { - // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // Schema-evolution: the logical field is not present in the Parquet file. The slot // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector - // (ROW branch); keep it as is and skip contributing to the level stream. + // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by + // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null + // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays + // NULL instead of reverting to the type's zero value. Skip contributing to the level stream. + childrenVectors[i].fillWithNulls(); finalChildrenVectors[i] = childrenVectors[i]; continue; } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 9e49493cbed75..7ec70490dc45b 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -112,9 +112,13 @@ private Tuple2 readRow( for (int i = 0; i < children.size(); i++) { ParquetField child = children.get(i); if (child == null) { - // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot + // Schema-evolution: the logical field is not present in the Parquet file. The slot // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector - // (ROW branch); keep it as is and skip contributing to the level stream. + // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by + // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null + // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays + // NULL instead of reverting to the type's zero value. Skip contributing to the level stream. + childrenVectors[i].fillWithNulls(); finalChildrenVectors[i] = childrenVectors[i]; continue; } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index bb5d0c55b81de..7b2bb0fa55f8e 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -19,19 +19,18 @@ package org.apache.hudi.table.format.cow; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapDecimalVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; -import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader; import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader; import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader; import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader; -import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader; +import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader; import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader; -import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader; @@ -64,12 +63,13 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; @@ -77,12 +77,18 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.io.ColumnIO; +import org.apache.parquet.io.GroupColumnIO; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.sql.Date; @@ -90,25 +96,38 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.flink.table.utils.DateTimeUtils.toInternal; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** * Util for generating {@link ParquetColumnarRowSplitReader}. * - *

    NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64 - * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that. + *

    Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive + * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader}, + * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For + * nested types (ARRAY / MAP / MULTISET / ROW) we build a {@link ParquetField} tree once per + * split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to + * {@link NestedColumnReader}. + * + *

    Schema evolution: missing top-level fields are still handled by the caller + * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields + * inside a Row are handled here — {@link #constructField} returns {@code null} for a + * child that isn't physically present, and the corresponding child in the pre-allocated vector + * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can + * passthrough the slot (see {@link NestedColumnReader#readToVector}). */ public class ParquetSplitReaderUtil { - /** - * Util for generating partitioned {@link ParquetColumnarRowSplitReader}. - */ + /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */ public static ParquetColumnarRowSplitReader genPartColumnarRowReader( boolean utcTimestamp, boolean caseSensitive, @@ -182,10 +201,13 @@ private static ColumnVector createVector( return readVector; } - private static ColumnVector createVectorFromConstant( - LogicalType type, - Object value, - int batchSize) { + /** + * Builds a constant-filled column vector for either a partition column (non-null value) or a + * missing-column slot (null value). Used both at the batch-generator level for partition + * injection and at the row-reader level for fields absent from the Parquet file. + */ + public static ColumnVector createVectorFromConstant( + LogicalType type, Object value, int batchSize) { switch (type.getTypeRoot()) { case CHAR: case VARCHAR: @@ -278,6 +300,7 @@ private static ColumnVector createVectorFromConstant( value == null ? null : toInternal((Date) value), batchSize); case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: HeapTimestampVector tv = new HeapTimestampVector(batchSize); if (value == null) { tv.fillWithNulls(); @@ -286,46 +309,41 @@ private static ColumnVector createVectorFromConstant( } return tv; case ARRAY: - ArrayType arrayType = (ArrayType) type; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize); - if (value == null) { - arrayGroup.fillWithNulls(); - return arrayGroup; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } - } else { - HeapArrayVector arrayVector = new HeapArrayVector(batchSize); - if (value == null) { - arrayVector.fillWithNulls(); - return arrayVector; - } else { - throw new UnsupportedOperationException("Unsupported create array with default value."); - } + if (value != null) { + throw new UnsupportedOperationException("Unsupported create array with default value."); } + HeapArrayVector arrayVector = new HeapArrayVector(batchSize); + arrayVector.fillWithNulls(); + return arrayVector; case MAP: - HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); - if (value == null) { - mapVector.fillWithNulls(); - return mapVector; - } else { - throw new UnsupportedOperationException("Unsupported create map with default value."); + case MULTISET: + if (value != null) { + throw new UnsupportedOperationException( + "Unsupported create " + type.getTypeRoot() + " with default value."); } + HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null); + mapVector.fillWithNulls(); + return mapVector; case ROW: - HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize); - if (value == null) { - rowVector.fillWithNulls(); - return rowVector; - } else { + if (value != null) { throw new UnsupportedOperationException("Unsupported create row with default value."); } + RowType rowType = (RowType) type; + WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()]; + for (int i = 0; i < childVectors.length; i++) { + childVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); + } + HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors); + rowVector.fillWithNulls(); + return rowVector; default: throw new UnsupportedOperationException("Unsupported type: " + type); } } - private static List filterDescriptors(int depth, Type type, List columns) throws ParquetRuntimeException { + private static List filterDescriptors( + int depth, Type type, List columns) throws ParquetRuntimeException { List filtered = new ArrayList<>(); for (ColumnDescriptor descriptor : columns) { if (depth >= descriptor.getPath().length) { @@ -339,24 +357,61 @@ private static List filterDescriptors(int depth, Type type, Li return filtered; } + /** + * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the + * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP}, + * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by + * the supplied pre-built {@link ParquetField} tree. + * + * @param field the {@link ParquetField} tree for this column, built by + * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested + * types; ignored for primitives. + */ + public static ColumnReader createColumnReader( + boolean utcTimestamp, + LogicalType fieldType, + Type physicalType, + List descriptors, + PageReadStore pages, + @Nullable ParquetField field) throws IOException { + switch (fieldType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + Preconditions.checkNotNull( + field, "ParquetField must be provided for nested type: %s", fieldType); + return new NestedColumnReader(utcTimestamp, pages, field); + default: + return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages); + } + } + + /** + * Backward-compat entry point kept for callers that don't project nested types and therefore + * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload + * with a null field; nested types now go through that overload directly. + * + * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore, + * ParquetField)} so nested types take the Dremel path. + */ + @Deprecated public static ColumnReader createColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List descriptors, PageReadStore pages) throws IOException { - return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, - pages, 0); + return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null); } - private static ColumnReader createColumnReader( + private static ColumnReader createPrimitiveColumnReader( boolean utcTimestamp, LogicalType fieldType, Type physicalType, List columns, - PageReadStore pages, - int depth) throws IOException { - List descriptors = filterDescriptors(depth, physicalType, columns); + PageReadStore pages) throws IOException { + List descriptors = filterDescriptors(0, physicalType, columns); ColumnDescriptor descriptor = descriptors.get(0); PageReader pageReader = pages.getPageReader(descriptor); switch (fieldType.getTypeRoot()) { @@ -392,7 +447,9 @@ private static ColumnReader createColumnReader( case INT96: return new TimestampColumnReader(utcTimestamp, descriptor, pageReader); default: - throw new AssertionError(); + throw new AssertionError( + "Unexpected physical type for TIMESTAMP: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } case DECIMAL: switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { @@ -403,106 +460,23 @@ private static ColumnReader createColumnReader( case BINARY: return new BytesColumnReader(descriptor, pageReader); case FIXED_LEN_BYTE_ARRAY: - return new FixedLenBytesColumnReader( - descriptor, pageReader); + return new FixedLenBytesColumnReader(descriptor, pageReader); default: - throw new AssertionError(); - } - case ARRAY: - ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new ArrayGroupReader(createColumnReader( - utcTimestamp, - arrayType.getElementType(), - elementType, - descriptors, - pages, - elementDepth)); - } else { - return new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - fieldType); + throw new AssertionError( + "Unexpected physical type for DECIMAL: " + + descriptor.getPrimitiveType().getPrimitiveTypeName()); } - case MAP: - MapType mapType = (MapType) fieldType; - ArrayColumnReader keyReader = - new ArrayColumnReader( - descriptor, - pageReader, - utcTimestamp, - descriptor.getPrimitiveType(), - new ArrayType(mapType.getKeyType())); - ColumnReader valueReader; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueReader = new ArrayGroupReader(createColumnReader( - utcTimestamp, - mapType.getValueType(), - physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type - descriptors.subList(1, descriptors.size()), // remove the key descriptor - pages, - depth + 2)); // increase the depth by 2, because there's a key_value entry in the path - } else { - valueReader = new ArrayColumnReader( - descriptors.get(1), - pages.getPageReader(descriptors.get(1)), - utcTimestamp, - descriptors.get(1).getPrimitiveType(), - new ArrayType(mapType.getValueType())); - } - return new MapColumnReader(keyReader, valueReader); - case ROW: - RowType rowType = (RowType) fieldType; - GroupType groupType = physicalType.asGroupType(); - List fieldReaders = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - // schema evolution: read the parquet file with a new extended field name. - int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); - if (fieldIndex < 0) { - fieldReaders.add(new EmptyColumnReader()); - } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - fieldReaders.add( - createColumnReader( - utcTimestamp, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } else { - fieldReaders.add( - createColumnReader( - utcTimestamp, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - pages, - depth + 1)); - } - } - } - return new RowColumnReader(fieldReaders); default: throw new UnsupportedOperationException(fieldType + " is not supported now."); } } + /** + * Creates the writable column vector that the reader will write into. The returned vector shape + * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled + * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in + * {@link NestedColumnReader} can pass them through unchanged. + */ public static WritableColumnVector createWritableColumnVector( int batchSize, LogicalType fieldType, @@ -523,33 +497,40 @@ private static WritableColumnVector createWritableColumnVector( switch (fieldType.getTypeRoot()) { case BOOLEAN: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapBooleanVector(batchSize); case TINYINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapByteVector(batchSize); case DOUBLE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDoubleVector(batchSize); case FLOAT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.FLOAT, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapFloatVector(batchSize); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapIntVector(batchSize); case BIGINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT64, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapLongVector(batchSize); case SMALLINT: checkArgument( - typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); + typeName == PrimitiveType.PrimitiveTypeName.INT32, + getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapShortVector(batchSize); case CHAR: case VARCHAR: @@ -566,112 +547,64 @@ private static WritableColumnVector createWritableColumnVector( case DECIMAL: checkArgument( (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY - || typeName == PrimitiveType.PrimitiveTypeName.BINARY) + || typeName == PrimitiveType.PrimitiveTypeName.BINARY) && primitiveType.getOriginalType() == OriginalType.DECIMAL, getPrimitiveTypeCheckFailureMessage(typeName, fieldType)); return new HeapDecimalVector(batchSize); case ARRAY: ArrayType arrayType = (ArrayType) fieldType; - if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - boolean isThreeLevelList = isThreeLevelList(physicalType); - // 3-level List structure, drill down 2 level to get type for `element` - Type elementType = isThreeLevelList - ? physicalType.asGroupType().getType(0).asGroupType().getType(0) - : physicalType.asGroupType().getType(0); - int elementDepth = isThreeLevelList ? depth + 2 : depth + 1; - return new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - elementType, - descriptors, - elementDepth)); - } else { - return new HeapArrayVector( - batchSize, - createWritableColumnVector( - batchSize, - arrayType.getElementType(), - physicalType, - descriptors, - depth)); - } - case MAP: + return new HeapArrayVector( + batchSize, + createWritableColumnVector( + batchSize, arrayType.getElementType(), physicalType, descriptors, depth)); + case MAP: { MapType mapType = (MapType) fieldType; - GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType(); - // the map column has three level paths. - WritableColumnVector keyColumnVector = createWritableColumnVector( + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( batchSize, - new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()), - repeatedType.getType(0), - descriptors, - depth + 2); - WritableColumnVector valueColumnVector; - if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) { - valueColumnVector = new HeapArrayGroupColumnVector( - batchSize, - createWritableColumnVector( - batchSize, - mapType.getValueType(), - repeatedType.getType(1).asGroupType(), - descriptors, - depth + 2)); - } else { - valueColumnVector = createWritableColumnVector( - batchSize, - new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()), - repeatedType.getType(1), - descriptors, - depth + 2); - } - return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector); + createWritableColumnVector( + batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2), + createWritableColumnVector( + batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2)); + } + case MULTISET: { + MultisetType multisetType = (MultisetType) fieldType; + GroupType repeatedType = unwrapMapRepeatedType(physicalType); + return new HeapMapColumnVector( + batchSize, + createWritableColumnVector( + batchSize, + multisetType.getElementType(), + repeatedType.getType(0), + descriptors, + depth + 2), + createWritableColumnVector( + batchSize, + new IntType(false), + repeatedType.getType(1), + descriptors, + depth + 2)); + } case ROW: RowType rowType = (RowType) fieldType; GroupType groupType = physicalType.asGroupType(); WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()]; for (int i = 0; i < columnVectors.length; i++) { - // schema evolution: read the file with a new extended field name. int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType); if (fieldIndex < 0) { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant( - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize); - } else { - columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); - } + // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled + // vector of the correct shape; NestedColumnReader.readRow will pass it through when the + // matching ParquetField child is null. + columnVectors[i] = + (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize); } else { - // Check for nested row in array with atomic field type. - - // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields. - // In Parquet, an array of rows is stored as separate arrays for each field. - - // Limitations: It won't work for multiple nested arrays and maps. - // The main problem is that the Flink classes and interface don't follow that pattern. - if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) { - columnVectors[i] = - createWritableColumnVector( - batchSize, - new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } else { - columnVectors[i] = - createWritableColumnVector( - batchSize, - rowType.getTypeAt(i), - groupType.getType(fieldIndex), - descriptors, - depth + 1); - } + columnVectors[i] = + createWritableColumnVector( + batchSize, + rowType.getTypeAt(i), + groupType.getType(fieldIndex), + descriptors, + depth + 1); } } return new HeapRowColumnVector(batchSize, columnVectors); @@ -681,56 +614,245 @@ private static WritableColumnVector createWritableColumnVector( } /** - * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}. - * - * @return The physical field index or -1 if the field does not exist + * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching + * Parquet's canonical 3-level map encoding. */ - private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { - // get index from fileSchema type, else, return -1 - return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + private static GroupType unwrapMapRepeatedType(Type physicalType) { + return physicalType.asGroupType().getType(0).asGroupType(); } + // ------------------------------------------------------------------------------------------ + // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil) + // + // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children + // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This + // matches the Hudi schema-evolution contract and is the companion to the null-child branch in + // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`. + // ------------------------------------------------------------------------------------------ + /** - * Check whether the given list type is a three-level list type. - *

    - * group (LIST) { - * repeated group list { - * element; - * } - * } - * - * @param type list type - * @return true if the list type is a three-level list type + * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed + * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally; + * primitive top-level fields produce {@code null} entries (callers don't need a tree for those). */ - private static boolean isThreeLevelList(Type type) { - if (type.isPrimitive()) { - return false; + public static List buildFieldsList( + List children, List fieldNames, MessageColumnIO columnIO) { + List list = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + RowType.RowField child = children.get(i); + if (isNestedType(child.getType())) { + list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i)))); + } else { + list.add(null); + } } - GroupType groupType = type.asGroupType(); - OriginalType originalType = groupType.getOriginalType(); - return originalType == OriginalType.LIST - && groupType.getType(0).getName().equals("list"); + return list; + } + + private static boolean isNestedType(LogicalType type) { + return type instanceof RowType + || type instanceof ArrayType + || type instanceof MapType + || type instanceof MultisetType; + } + + @Nullable + private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) { + boolean required = columnIO.getType().getRepetition() == REQUIRED; + int repetitionLevel = columnIO.getRepetitionLevel(); + int definitionLevel = columnIO.getDefinitionLevel(); + LogicalType type = rowField.getType(); + String fieldName = rowField.getName(); + if (type instanceof RowType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + RowType rowType = (RowType) type; + List childFields = rowType.getFields(); + List fieldsList = new ArrayList<>(childFields.size()); + for (RowType.RowField childField : childFields) { + // Hudi schema evolution: a logical child may be absent from the Parquet file. In that + // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the + // pre-filled null vector instead of recursing. + ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName()); + if (childIo == null) { + fieldsList.add(null); + } else { + fieldsList.add(constructField(childField, childIo)); + } + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(fieldsList)); + } + + if (type instanceof MapType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MapType mapType = (MapType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof MultisetType) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO); + MultisetType multisetType = (MultisetType) type; + ParquetField keyField = + constructField( + new RowType.RowField("", multisetType.getElementType()), + keyValueColumnIO.getChild(0)); + ParquetField valueField = + constructField( + new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1)); + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.unmodifiableList(Arrays.asList(keyField, valueField))); + } + + if (type instanceof ArrayType) { + ArrayType arrayType = (ArrayType) type; + ColumnIO elementTypeColumnIO; + if (columnIO instanceof GroupColumnIO) { + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { + while (!Objects.equals(groupColumnIO.getName(), fieldName)) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + elementTypeColumnIO = groupColumnIO; + } else { + if (arrayType.getElementType() instanceof RowType) { + elementTypeColumnIO = groupColumnIO; + } else { + elementTypeColumnIO = groupColumnIO.getChild(0); + } + } + } else if (columnIO instanceof PrimitiveColumnIO) { + elementTypeColumnIO = columnIO; + } else { + throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); + } + + ParquetField elementField = + constructField( + new RowType.RowField("", arrayType.getElementType()), + getArrayElementColumn(elementTypeColumnIO)); + if (repetitionLevel == elementField.getRepetitionLevel()) { + repetitionLevel = columnIO.getParent().getRepetitionLevel(); + } + return new ParquetGroupField( + type, + repetitionLevel, + definitionLevel, + required, + Collections.singletonList(elementField)); + } + + PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO; + return new ParquetPrimitiveField( + type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId()); } /** - * Construct the error message when primitive type mismatches. - * - * @param primitiveType Primitive type - * @param fieldType Logical field type - * @return The error message + * Parquet column names are case-insensitive in Flink's lookup. Matches upstream + * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent. */ - private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { - return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name()); + public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName); + if (columnIO != null) { + return columnIO; + } + throw new FlinkRuntimeException( + "Can not find column io for parquet reader. Column name: " + columnName); } /** - * Construct the error message when original type mismatches. + * Case-insensitive column lookup that returns {@code null} when no match is found — the + * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to + * emit null {@link ParquetField} children for fields absent from the Parquet file. + */ + @Nullable + private static ColumnIO lookupColumnByNameOrNull( + GroupColumnIO groupColumnIO, String columnName) { + ColumnIO columnIO = groupColumnIO.getChild(columnName); + if (columnIO != null) { + return columnIO; + } + for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) { + if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) { + return groupColumnIO.getChild(i); + } + } + return null; + } + + public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { + while (groupColumnIO.getChildrenCount() == 1) { + groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); + } + return groupColumnIO; + } + + public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { + while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { + columnIO = ((GroupColumnIO) columnIO).getChild(0); + } + + // Three-level list: skip the synthetic `element` / `list` wrapper when present. + if (columnIO instanceof GroupColumnIO + && columnIO.getType().getLogicalTypeAnnotation() == null + && ((GroupColumnIO) columnIO).getChildrenCount() == 1 + && !columnIO.getName().equals("array") + && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { + return ((GroupColumnIO) columnIO).getChild(0); + } + return columnIO; + } + + /** + * Returns the field index with given physical row type {@code groupType} and field name + * {@code fieldName}. * - * @param originalType Original type - * @param fieldType Logical field type - * @return The error message + * @return the physical field index or -1 if the field does not exist + */ + private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) { + return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1; + } + + private static String getPrimitiveTypeCheckFailureMessage( + PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Primitive type: %s. Field type: %s.", + primitiveType, fieldType.getTypeRoot().name()); + } + + private static String getOriginalTypeCheckFailureMessage( + OriginalType originalType, LogicalType fieldType) { + return String.format( + "Unexpected type exception. Original type: %s. Field type: %s.", + originalType, fieldType.getTypeRoot().name()); + } + + /** + * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience + * for callers that need to mirror Hudi's original behaviour where a missing column produces an + * explicit null-valued reader rather than being omitted from the batch. */ - private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) { - return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name()); + public static ColumnReader emptyColumnReader() { + return new EmptyColumnReader(); } } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java new file mode 100644 index 0000000000000..d51d7ee754b8a --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of booleans. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper. + */ +public class BooleanArrayList { + private int size; + private boolean[] array; + + public BooleanArrayList(int capacity) { + this.size = 0; + this.array = new boolean[capacity]; + } + + public int size() { + return size; + } + + public boolean add(boolean element) { + grow(size + 1); + array[size++] = element; + return true; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public boolean[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final boolean[] t = new boolean[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java new file mode 100644 index 0000000000000..4787dbb5b9ddb --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; +import java.util.NoSuchElementException; + +/** + * Minimal implementation of an array-backed list of ints. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class IntArrayList { + + private int size; + private int[] array; + + public IntArrayList(final int capacity) { + this.size = 0; + this.array = new int[capacity]; + } + + public int size() { + return size; + } + + public boolean add(final int number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public int removeLast() { + if (size == 0) { + throw new NoSuchElementException(); + } + --size; + return array[size]; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return size == 0; + } + + private void grow(final int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final int[] t = new int[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } + + public int[] toArray() { + return Arrays.copyOf(array, size); + } + + public static final IntArrayList EMPTY = + new IntArrayList(0) { + + @Override + public boolean add(int number) { + throw new UnsupportedOperationException(); + } + + @Override + public int removeLast() { + throw new UnsupportedOperationException(); + } + }; +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java new file mode 100644 index 0000000000000..a51291f9d8441 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import java.util.Arrays; + +/** + * Minimal implementation of an array-backed list of longs. + * + *

    Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to + * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader. + */ +public class LongArrayList { + + private int size; + private long[] array; + + public LongArrayList(int capacity) { + this.size = 0; + this.array = new long[capacity]; + } + + public int size() { + return size; + } + + public boolean add(long number) { + grow(size + 1); + array[size++] = number; + return true; + } + + public long removeLong(int index) { + if (index >= size) { + throw new IndexOutOfBoundsException( + "Index (" + index + ") is greater than or equal to list size (" + size + ")"); + } + final long old = array[index]; + size--; + if (index != size) { + System.arraycopy(array, index + 1, array, index, size - index); + } + return old; + } + + public void clear() { + size = 0; + } + + public boolean isEmpty() { + return (size == 0); + } + + public long[] toArray() { + return Arrays.copyOf(array, size); + } + + private void grow(int length) { + if (length > array.length) { + final int newLength = + (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); + final long[] t = new long[newLength]; + System.arraycopy(array, 0, t, 0, size); + array = t; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java new file mode 100644 index 0000000000000..3f2f8976b69bf --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.utils; + +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; + +import static java.lang.String.format; + +/** + * Utils to calculate nested type position. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.utils.NestedPositionUtil}). + */ +public class NestedPositionUtil { + + /** + * Calculate row offsets according to column's max repetition level, definition level, value's + * repetition level and definition level. Each row has three situation: + *

  • Row is not defined,because it's optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Row is null + *
  • Row is defined and not empty. + * + * @param field field that contains the row column message include max repetition level and + * definition level. + * @param fieldRepetitionLevels int array with each value's repetition level. + * @param fieldDefinitionLevels int array with each value's definition level. + * @return {@link RowPosition} contains collections row count and isNull array. + */ + public static RowPosition calculateRowOffsets( + ParquetField field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) { + int rowDefinitionLevel = field.getDefinitionLevel(); + int rowRepetitionLevel = field.getRepetitionLevel(); + int nullValuesCount = 0; + BooleanArrayList nullRowFlags = new BooleanArrayList(0); + for (int i = 0; i < fieldDefinitionLevels.length; i++) { + // If a row's last field is an array, the repetition levels for the array's items will + // be larger than the parent row's repetition level, so we need to skip those values. + if (fieldRepetitionLevels[i] > rowRepetitionLevel) { + continue; + } + + if (fieldDefinitionLevels[i] >= rowDefinitionLevel) { + // current row is defined and not empty + nullRowFlags.add(false); + } else { + // current row is null + nullRowFlags.add(true); + nullValuesCount++; + } + } + if (nullValuesCount == 0) { + return new RowPosition(null, fieldDefinitionLevels.length); + } + return new RowPosition(nullRowFlags.toArray(), nullRowFlags.size()); + } + + /** + * Calculate the collection's offsets according to column's max repetition level, definition + * level, value's repetition level and definition level. Each collection (Array or Map) has four + * situation: + *
  • Collection is not defined, because optional parent fields is null, this is decided by its + * parent's repetition level + *
  • Collection is null + *
  • Collection is defined but empty + *
  • Collection is defined and not empty. In this case offset value is increased by the number + * of elements in that collection + * + * @param field field that contains array/map column message include max repetition level and + * definition level. + * @param definitionLevels int array with each value's definition level. + * @param repetitionLevels int array with each value's repetition level. + * @return {@link CollectionPosition} contains collections offset array, length array and isNull + * array. + */ + public static CollectionPosition calculateCollectionOffsets( + ParquetField field, int[] definitionLevels, int[] repetitionLevels) { + int collectionDefinitionLevel = field.getDefinitionLevel(); + int collectionRepetitionLevel = field.getRepetitionLevel() + 1; + int offset = 0; + int valueCount = 0; + LongArrayList offsets = new LongArrayList(0); + offsets.add(offset); + BooleanArrayList emptyCollectionFlags = new BooleanArrayList(0); + BooleanArrayList nullCollectionFlags = new BooleanArrayList(0); + int nullValuesCount = 0; + for (int i = 0; + i < definitionLevels.length; + i = getNextCollectionStartIndex(repetitionLevels, collectionRepetitionLevel, i)) { + valueCount++; + if (definitionLevels[i] >= collectionDefinitionLevel - 1) { + boolean isNull = + isOptionalFieldValueNull(definitionLevels[i], collectionDefinitionLevel); + nullCollectionFlags.add(isNull); + nullValuesCount += isNull ? 1 : 0; + // definitionLevels[i] > collectionDefinitionLevel => Collection is defined and not + // empty + // definitionLevels[i] == collectionDefinitionLevel => Collection is defined but + // empty + if (definitionLevels[i] > collectionDefinitionLevel) { + emptyCollectionFlags.add(false); + offset += getCollectionSize(repetitionLevels, collectionRepetitionLevel, i + 1); + } else if (definitionLevels[i] == collectionDefinitionLevel) { + offset++; + emptyCollectionFlags.add(true); + } else { + offset++; + emptyCollectionFlags.add(false); + } + offsets.add(offset); + } else { + // when definitionLevels[i] < collectionDefinitionLevel - 1, it means the collection + // is + // not defined, but we need to regard it as null to avoid getting value wrong. + nullCollectionFlags.add(true); + nullValuesCount++; + offsets.add(++offset); + emptyCollectionFlags.add(false); + } + } + long[] offsetsArray = offsets.toArray(); + long[] length = calculateLengthByOffsets(emptyCollectionFlags.toArray(), offsetsArray); + if (nullValuesCount == 0) { + return new CollectionPosition(null, offsetsArray, length, valueCount); + } + return new CollectionPosition( + nullCollectionFlags.toArray(), offsetsArray, length, valueCount); + } + + public static boolean isOptionalFieldValueNull(int definitionLevel, int maxDefinitionLevel) { + return definitionLevel == maxDefinitionLevel - 1; + } + + public static long[] calculateLengthByOffsets( + boolean[] collectionIsEmpty, long[] arrayOffsets) { + LongArrayList lengthList = new LongArrayList(arrayOffsets.length); + for (int i = 0; i < arrayOffsets.length - 1; i++) { + long offset = arrayOffsets[i]; + long length = arrayOffsets[i + 1] - offset; + if (length < 0) { + throw new IllegalArgumentException( + format( + "Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", + i, arrayOffsets[i], i + 1, arrayOffsets[i + 1])); + } + if (collectionIsEmpty[i]) { + length = 0; + } + lengthList.add(length); + } + return lengthList.toArray(); + } + + private static int getNextCollectionStartIndex( + int[] repetitionLevels, int maxRepetitionLevel, int elementIndex) { + do { + elementIndex++; + } while (hasMoreElements(repetitionLevels, elementIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, elementIndex)); + return elementIndex; + } + + /** This method is only called for non-empty collections. */ + private static int getCollectionSize( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + int size = 1; + while (hasMoreElements(repetitionLevels, nextIndex) + && isNotCollectionBeginningMarker( + repetitionLevels, maxRepetitionLevel, nextIndex)) { + // Collection elements cannot only be primitive, but also can have nested structure + // Counting only elements which belong to current collection, skipping inner elements of + // nested collections/structs + if (repetitionLevels[nextIndex] <= maxRepetitionLevel) { + size++; + } + nextIndex++; + } + return size; + } + + private static boolean isNotCollectionBeginningMarker( + int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) { + return repetitionLevels[nextIndex] >= maxRepetitionLevel; + } + + private static boolean hasMoreElements(int[] repetitionLevels, int nextIndex) { + return nextIndex < repetitionLevels.length; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java deleted file mode 100644 index 4c9275f3b0932..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupArrayData implements ArrayData { - - WritableColumnVector vector; - int rowId; - - public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) { - this.vector = vector; - this.rowId = rowId; - } - - @Override - public int size() { - if (vector == null) { - return 0; - } - - if (vector instanceof HeapRowColumnVector) { - // assume all fields have the same size - if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) { - return 0; - } - return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size(); - } - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean isNullAt(int index) { - if (vector == null) { - return true; - } - - if (vector instanceof HeapRowColumnVector) { - return ((HeapRowColumnVector) vector).vectors == null; - } - - throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector"); - } - - @Override - public boolean getBoolean(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float getFloat(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public StringData getString(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public DecimalData getDecimal(int index, int precision, int scale) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public TimestampData getTimestamp(int index, int precision) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RawValueData getRawValue(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] getBinary(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public ArrayData getArray(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public MapData getMap(int index) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public RowData getRow(int index, int numFields) { - return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException("Not support the operation!"); - } - -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java deleted file mode 100644 index 69cb6feca13e4..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -public class ColumnarGroupMapData implements MapData { - - WritableColumnVector keyVector; - WritableColumnVector valueVector; - int rowId; - - public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) { - this.keyVector = keyVector; - this.valueVector = valueVector; - this.rowId = rowId; - } - - @Override - public int size() { - if (keyVector == null) { - return 0; - } - - if (keyVector instanceof HeapArrayVector) { - return ((HeapArrayVector) keyVector).getArray(rowId).size(); - } - throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector"); - } - - @Override - public ArrayData keyArray() { - return ((HeapArrayVector) keyVector).getArray(rowId); - } - - @Override - public ArrayData valueArray() { - if (valueVector instanceof HeapArrayVector) { - return ((HeapArrayVector) valueVector).getArray(rowId); - } else if (valueVector instanceof HeapArrayGroupColumnVector) { - return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId); - } - throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector"); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java deleted file mode 100644 index 439c1880823f1..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.MapData; -import org.apache.flink.table.data.RawValueData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.types.RowKind; - -public class ColumnarGroupRowData implements RowData { - - HeapRowColumnVector vector; - int rowId; - int index; - - public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) { - this.vector = vector; - this.rowId = rowId; - this.index = index; - } - - @Override - public int getArity() { - return vector.vectors.length; - } - - @Override - public RowKind getRowKind() { - return RowKind.INSERT; - } - - @Override - public void setRowKind(RowKind rowKind) { - throw new UnsupportedOperationException("Not support the operation!"); - } - - @Override - public boolean isNullAt(int pos) { - return - vector.vectors[pos].isNullAt(rowId) - || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index); - } - - @Override - public boolean getBoolean(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index); - } - - @Override - public byte getByte(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index); - } - - @Override - public short getShort(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index); - } - - @Override - public int getInt(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index); - } - - @Override - public long getLong(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index); - } - - @Override - public float getFloat(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index); - } - - @Override - public double getDouble(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index); - } - - @Override - public StringData getString(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index); - } - - @Override - public DecimalData getDecimal(int pos, int i1, int i2) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2); - } - - @Override - public TimestampData getTimestamp(int pos, int i1) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1); - } - - @Override - public RawValueData getRawValue(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index); - } - - @Override - public byte[] getBinary(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index); - } - - @Override - public ArrayData getArray(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index); - } - - @Override - public MapData getMap(int pos) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index); - } - - @Override - public RowData getRow(int pos, int numFields) { - return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java deleted file mode 100644 index 3d7d8b1f0de0f..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector; - -import org.apache.flink.table.data.ArrayData; -import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -/** - * This class represents a nullable heap row column vector. - */ -public class HeapArrayGroupColumnVector extends AbstractHeapVector - implements WritableColumnVector, ArrayColumnVector { - - public WritableColumnVector vector; - - public HeapArrayGroupColumnVector(int len) { - super(len); - } - - public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) { - super(len); - this.vector = vector; - } - - @Override - public ArrayData getArray(int rowId) { - return new ColumnarGroupArrayData(vector, rowId); - } - - @Override - public void reset() { - super.reset(); - vector.reset(); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java index a0dced01e5e8d..2f21a323302f1 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java @@ -57,6 +57,37 @@ public int getLen() { return this.isNull.length; } + // --------------------------------------------------------------------------------------------- + // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths} + // and {@code child} fields so legacy callers continue to work; the new {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + // future Flink-2.1-style caller use these accessors. + // --------------------------------------------------------------------------------------------- + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public ColumnVector getChild() { + return child; + } + + public void setChild(ColumnVector child) { + this.child = child; + } + @Override public ArrayData getArray(int i) { long offset = offsets[i]; diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java index 0d83f82baedf3..14aad22039e0a 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java @@ -20,29 +20,97 @@ import lombok.Getter; import org.apache.flink.table.data.MapData; +import org.apache.flink.table.data.columnar.ColumnarMapData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.MapColumnVector; import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; /** * This class represents a nullable heap map column vector. + * + *

    Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from + * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward + * compatibility: the {@code keys} / {@code values} fields are typed + * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing + * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original + * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use + * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}. */ public class HeapMapColumnVector extends AbstractHeapVector implements WritableColumnVector, MapColumnVector { @Getter - private final WritableColumnVector keys; + private WritableColumnVector keys; @Getter - private final WritableColumnVector values; + private WritableColumnVector values; + + // --------------------------------------------------------------------------------------------- + // Flink 2.1 Dremel-style state. Populated by {@link + // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and + // consumed by {@link #getMap(int)}. + // --------------------------------------------------------------------------------------------- + private long[] offsets; + private long[] lengths; + private int size; public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) { super(len); + this.offsets = new long[len]; + this.lengths = new long[len]; + this.keys = keys; + this.values = values; + } + + public long[] getOffsets() { + return offsets; + } + + public void setOffsets(long[] offsets) { + this.offsets = offsets; + } + + public long[] getLengths() { + return lengths; + } + + public void setLengths(long[] lengths) { + this.lengths = lengths; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public void setKeys(WritableColumnVector keys) { this.keys = keys; + } + + public void setValues(WritableColumnVector values) { this.values = values; } + /** + * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract + * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}. + */ + public ColumnVector getKeyColumnVector() { + return keys; + } + + /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */ + public ColumnVector getValueColumnVector() { + return values; + } + @Override public MapData getMap(int rowId) { - return new ColumnarGroupMapData(keys, values, rowId); + long offset = offsets[rowId]; + long length = lengths[rowId]; + return new ColumnarMapData(keys, values, (int) offset, (int) length); } } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java index ae194e4e6ab05..0c640ce92ee40 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java @@ -37,6 +37,21 @@ public HeapRowColumnVector(int len, WritableColumnVector... vectors) { this.vectors = vectors; } + /** + * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code + * vectors} field so legacy callers continue to work; the new {@link + * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any + * future Flink-2.1-style caller use this accessor. + */ + public WritableColumnVector[] getFields() { + return vectors; + } + + /** Counterpart of {@link #getFields()}. */ + public void setFields(WritableColumnVector[] fields) { + this.vectors = fields; + } + @Override public ColumnarRowData getRow(int i) { ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors)); diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java index 98b5e61050898..a37b88352cf52 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java @@ -18,21 +18,29 @@ package org.apache.hudi.table.format.cow.vector; +import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.columnar.vector.BytesColumnVector; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.DecimalColumnVector; +import org.apache.flink.table.data.columnar.vector.Dictionary; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.table.data.columnar.vector.LongColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector; + +import static org.apache.flink.util.Preconditions.checkArgument; /** - * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to - * provide {@link DecimalColumnVector} interface. - * - *

    Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector} - * because it is not public. + * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide + * {@link DecimalColumnVector} interface. */ -public class ParquetDecimalVector implements DecimalColumnVector { +public class ParquetDecimalVector + implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector { - public final ColumnVector vector; + private final ColumnVector vector; public ParquetDecimalVector(ColumnVector vector) { this.vector = vector; @@ -40,15 +48,180 @@ public ParquetDecimalVector(ColumnVector vector) { @Override public DecimalData getDecimal(int i, int precision, int scale) { - return DecimalData.fromUnscaledBytes( - ((BytesColumnVector) vector).getBytes(i).getBytes(), - precision, - scale); + if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) { + return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale); + } else if (ParquetSchemaConverter.is64BitDecimal(precision) + && vector instanceof LongColumnVector) { + return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale); + } else { + checkArgument( + vector instanceof BytesColumnVector, + "Reading decimal type occur unsupported vector type: %s", + vector.getClass()); + return DecimalData.fromUnscaledBytes( + ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale); + } + } + + public ColumnVector getVector() { + return vector; } @Override public boolean isNullAt(int i) { return vector.isNullAt(i); } -} + @Override + public void reset() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).reset(); + } + } + + @Override + public void setNullAt(int rowId) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNullAt(rowId); + } + } + + @Override + public void setNulls(int rowId, int count) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setNulls(rowId, count); + } + } + + @Override + public void fillWithNulls() { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).fillWithNulls(); + } + } + + @Override + public void setDictionary(Dictionary dictionary) { + if (vector instanceof WritableColumnVector) { + ((WritableColumnVector) vector).setDictionary(dictionary); + } + } + + @Override + public boolean hasDictionary() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).hasDictionary(); + } + return false; + } + + @Override + public WritableIntVector reserveDictionaryIds(int capacity) { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).reserveDictionaryIds(capacity); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public WritableIntVector getDictionaryIds() { + if (vector instanceof WritableColumnVector) { + return ((WritableColumnVector) vector).getDictionaryIds(); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public Bytes getBytes(int i) { + if (vector instanceof WritableBytesVector) { + return ((WritableBytesVector) vector).getBytes(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void appendBytes(int rowId, byte[] value, int offset, int length) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length); + } + } + + @Override + public void fill(byte[] value) { + if (vector instanceof WritableBytesVector) { + ((WritableBytesVector) vector).fill(value); + } + } + + @Override + public int getInt(int i) { + if (vector instanceof WritableIntVector) { + return ((WritableIntVector) vector).getInt(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setInt(int rowId, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInt(rowId, value); + } + } + + @Override + public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void setInts(int rowId, int count, int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, value); + } + } + + @Override + public void setInts(int rowId, int count, int[] src, int srcIndex) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(int value) { + if (vector instanceof WritableIntVector) { + ((WritableIntVector) vector).fill(value); + } + } + + @Override + public long getLong(int i) { + if (vector instanceof WritableLongVector) { + return ((WritableLongVector) vector).getLong(i); + } + throw new RuntimeException("Child vector must be instance of WritableColumnVector"); + } + + @Override + public void setLong(int rowId, long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLong(rowId, value); + } + } + + @Override + public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex); + } + } + + @Override + public void fill(long value) { + if (vector instanceof WritableLongVector) { + ((WritableLongVector) vector).fill(value); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java new file mode 100644 index 0000000000000..fcdedfbc9d71d --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent collection's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.CollectionPosition}). + */ +public class CollectionPosition { + @Nullable private final boolean[] isNull; + private final long[] offsets; + private final long[] length; + private final int valueCount; + + public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { + this.isNull = isNull; + this.offsets = offsets; + this.length = length; + this.valueCount = valueCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public long[] getOffsets() { + return offsets; + } + + public long[] getLength() { + return length; + } + + public int getValueCount() { + return valueCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java new file mode 100644 index 0000000000000..fe95419ac3218 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +/** + * To delegate repetition level and definition level. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.LevelDelegation}). + */ +public class LevelDelegation { + private final int[] repetitionLevel; + private final int[] definitionLevel; + + public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) { + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + } + + public int[] getRepetitionLevel() { + return repetitionLevel; + } + + public int[] getDefinitionLevel() { + return definitionLevel; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java new file mode 100644 index 0000000000000..5438b67973238 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.position; + +import javax.annotation.Nullable; + +/** + * To represent struct's position in repeated type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.position.RowPosition}). + */ +public class RowPosition { + @Nullable private final boolean[] isNull; + private final int positionsCount; + + public RowPosition(boolean[] isNull, int positionsCount) { + this.isNull = isNull; + this.positionsCount = positionsCount; + } + + public boolean[] getIsNull() { + return isNull; + } + + public int getPositionsCount() { + return positionsCount; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java deleted file mode 100644 index 6a8a01b74946a..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayVector; -import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; -import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; -import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; -import org.apache.flink.table.types.logical.ArrayType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Array {@link ColumnReader}. - */ -public class ArrayColumnReader extends BaseVectorizedColumnReader { - - // The value read in last time - private Object lastValue; - - // flag to indicate if there is no data in parquet data page - private boolean eof = false; - - // flag to indicate if it's the first time to read parquet data page with this instance - boolean isFirstRow = true; - - public ArrayColumnReader( - ColumnDescriptor descriptor, - PageReader pageReader, - boolean isUtcTimestamp, - Type type, - LogicalType logicalType) - throws IOException { - super(descriptor, pageReader, isUtcTimestamp, type, logicalType); - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayVector lcv = (HeapArrayVector) vector; - // before readBatch, initial the size of offsets & lengths as the default value, - // the actual size will be assigned in setChildrenInfo() after reading complete. - lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE]; - // Because the length of ListColumnVector.child can't be known now, - // the valueList will save all data for ListColumnVector temporary. - List valueList = new ArrayList<>(); - - LogicalType category = ((ArrayType) logicalType).getElementType(); - - // read the first row in parquet data page, this will be only happened once for this - // instance - if (isFirstRow) { - if (!fetchNextValue(category)) { - return; - } - isFirstRow = false; - } - - int index = collectDataFromParquetPage(readNumber, lcv, valueList, category); - - // Convert valueList to array for the ListColumnVector.child - fillColumnVector(category, lcv, valueList, index); - } - - /** - * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating - * if there is more values to read (true). - * - * @param category - * @return boolean - * @throws IOException - */ - private boolean fetchNextValue(LogicalType category) throws IOException { - int left = readPageIfNeed(); - if (left > 0) { - // get the values of repetition and definitionLevel - readRepetitionAndDefinitionLevels(); - // read the data if it isn't null - if (definitionLevel == maxDefLevel) { - if (isCurrentPageDictionaryEncoded) { - lastValue = dataColumn.readValueDictionaryId(); - } else { - lastValue = readPrimitiveTypedRow(category); - } - } else { - lastValue = null; - } - return true; - } else { - eof = true; - return false; - } - } - - private int readPageIfNeed() throws IOException { - // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); - } - return leftInPage; - } - - // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper - // TODO Reduce the duplicated code - private Object readPrimitiveTypedRow(LogicalType category) { - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dataColumn.readString(); - case BOOLEAN: - return dataColumn.readBoolean(); - case TIME_WITHOUT_TIME_ZONE: - case DATE: - case INTEGER: - return dataColumn.readInteger(); - case TINYINT: - return dataColumn.readTinyInt(); - case SMALLINT: - return dataColumn.readSmallInt(); - case BIGINT: - return dataColumn.readLong(); - case FLOAT: - return dataColumn.readFloat(); - case DOUBLE: - return dataColumn.readDouble(); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dataColumn.readInteger(); - case INT64: - return dataColumn.readLong(); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return dataColumn.readString(); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dataColumn.readTimestamp(); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { - if (dictionaryValue == null) { - return null; - } - - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - return dictionary.readString(dictionaryValue); - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case INTEGER: - return dictionary.readInteger(dictionaryValue); - case BOOLEAN: - return dictionary.readBoolean(dictionaryValue) ? 1 : 0; - case DOUBLE: - return dictionary.readDouble(dictionaryValue); - case FLOAT: - return dictionary.readFloat(dictionaryValue); - case TINYINT: - return dictionary.readTinyInt(dictionaryValue); - case SMALLINT: - return dictionary.readSmallInt(dictionaryValue); - case BIGINT: - return dictionary.readLong(dictionaryValue); - case DECIMAL: - switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return dictionary.readInteger(dictionaryValue); - case INT64: - return dictionary.readLong(dictionaryValue); - case FIXED_LEN_BYTE_ARRAY: - case BINARY: - return dictionary.readString(dictionaryValue); - default: - throw new AssertionError(); - } - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return dictionary.readTimestamp(dictionaryValue); - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } - - /** - * Collects data from a parquet page and returns the final row index where it stopped. The - * returned index can be equal to or less than total. - * - * @param total maximum number of rows to collect - * @param lcv column vector to do initial setup in data collection time - * @param valueList collection of values that will be fed into the vector later - * @param category - * @return int - * @throws IOException - */ - private int collectDataFromParquetPage( - int total, HeapArrayVector lcv, List valueList, LogicalType category) - throws IOException { - int index = 0; - /* - * Here is a nested loop for collecting all values from a parquet page. - * A column of array type can be considered as a list of lists, so the two loops are as below: - * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.: - * [0, 2, 3] <- index: 0 - * [NULL, 3, 4] <- index: 1 - * - * 2. The inner loop iterates on values within a row (sets all data from parquet data page - * for an element in ListColumnVector), so fetchNextValue returns values one-by-one: - * 0, 2, 3, NULL, 3, 4 - * - * As described below, the repetition level (repetitionLevel != 0) - * can be used to decide when we'll start to read values for the next list. - */ - while (!eof && index < total) { - // add element to ListColumnVector one by one - lcv.offsets[index] = valueList.size(); - /* - * Let's collect all values for a single list. - * Repetition level = 0 means that a new list started there in the parquet page, - * in that case, let's exit from the loop, and start to collect value for a new list. - */ - do { - /* - * Definition level = 0 when a NULL value was returned instead of a list - * (this is not the same as a NULL value in of a list). - */ - if (definitionLevel == 0) { - lcv.setNullAt(index); - } - valueList.add( - isCurrentPageDictionaryEncoded - ? dictionaryDecodeValue(category, (Integer) lastValue) - : lastValue); - } while (fetchNextValue(category) && (repetitionLevel != 0)); - - lcv.lengths[index] = valueList.size() - lcv.offsets[index]; - index++; - } - return index; - } - - /** - * The lengths & offsets will be initialized as default size (1024), it should be set to the - * actual size according to the element number. - */ - private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) { - lcv.setSize(itemNum); - long[] lcvLength = new long[elementNum]; - long[] lcvOffset = new long[elementNum]; - System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum); - System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum); - lcv.lengths = lcvLength; - lcv.offsets = lcvOffset; - } - - private void fillColumnVector( - LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) { - int total = valueList.size(); - setChildrenInfo(lcv, total, elementNum); - switch (category.getTypeRoot()) { - case CHAR: - case VARCHAR: - case BINARY: - case VARBINARY: - lcv.child = new HeapBytesVector(total); - ((HeapBytesVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (src == null) { - ((HeapBytesVector) lcv.child).setNullAt(i); - } else { - ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length); - } - } - break; - case BOOLEAN: - lcv.child = new HeapBooleanVector(total); - ((HeapBooleanVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapBooleanVector) lcv.child).setNullAt(i); - } else { - ((HeapBooleanVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TINYINT: - lcv.child = new HeapByteVector(total); - ((HeapByteVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapByteVector) lcv.child).setNullAt(i); - } else { - ((HeapByteVector) lcv.child).vector[i] = - (byte) ((List) valueList).get(i).intValue(); - } - } - break; - case SMALLINT: - lcv.child = new HeapShortVector(total); - ((HeapShortVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapShortVector) lcv.child).setNullAt(i); - } else { - ((HeapShortVector) lcv.child).vector[i] = - (short) ((List) valueList).get(i).intValue(); - } - } - break; - case INTEGER: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - lcv.child = new HeapIntVector(total); - ((HeapIntVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) lcv.child).setNullAt(i); - } else { - ((HeapIntVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case FLOAT: - lcv.child = new HeapFloatVector(total); - ((HeapFloatVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapFloatVector) lcv.child).setNullAt(i); - } else { - ((HeapFloatVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case BIGINT: - lcv.child = new HeapLongVector(total); - ((HeapLongVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) lcv.child).setNullAt(i); - } else { - ((HeapLongVector) lcv.child).vector[i] = ((List) valueList).get(i); - } - } - break; - case DOUBLE: - lcv.child = new HeapDoubleVector(total); - ((HeapDoubleVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapDoubleVector) lcv.child).setNullAt(i); - } else { - ((HeapDoubleVector) lcv.child).vector[i] = - ((List) valueList).get(i); - } - } - break; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - lcv.child = new HeapTimestampVector(total); - ((HeapTimestampVector) lcv.child).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapTimestampVector) lcv.child).setNullAt(i); - } else { - ((HeapTimestampVector) lcv.child) - .setTimestamp(i, ((List) valueList).get(i)); - } - } - break; - case DECIMAL: - PrimitiveType.PrimitiveTypeName primitiveTypeName = - descriptor.getPrimitiveType().getPrimitiveTypeName(); - switch (primitiveTypeName) { - case INT32: - lcv.child = new ParquetDecimalVector(new HeapIntVector(total)); - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - case INT64: - lcv.child = new ParquetDecimalVector(new HeapLongVector(total)); - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - if (valueList.get(i) == null) { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector) - .vector[i] = - ((List) valueList).get(i); - } - } - break; - default: - lcv.child = new ParquetDecimalVector(new HeapBytesVector(total)); - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector).reset(); - for (int i = 0; i < valueList.size(); i++) { - byte[] src = ((List) valueList).get(i); - if (valueList.get(i) == null) { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .setNullAt(i); - } else { - ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector) - .appendBytes(i, src, 0, src.length); - } - } - break; - } - break; - default: - throw new RuntimeException("Unsupported type in the list: " + type); - } - } -} - diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java deleted file mode 100644 index df7c5d85bc4ab..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}. - */ -public class ArrayGroupReader implements ColumnReader { - - private final ColumnReader fieldReader; - - public ArrayGroupReader(ColumnReader fieldReader) { - this.fieldReader = fieldReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector; - - fieldReader.readToVector(readNumber, rowColumnVector.vector); - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java deleted file mode 100644 index 6d743530fccc7..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; - -/** - * Map {@link ColumnReader}. - */ -public class MapColumnReader implements ColumnReader { - - private final ArrayColumnReader keyReader; - private final ColumnReader valueReader; - - public MapColumnReader( - ArrayColumnReader keyReader, ColumnReader valueReader) { - this.keyReader = keyReader; - this.valueReader = valueReader; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector; - AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys()); - keyReader.readToVector(readNumber, mapColumnVector.getKeys()); - valueReader.readToVector(readNumber, mapColumnVector.getValues()); - for (int i = 0; i < keyArrayColumnVector.getLen(); i++) { - if (keyArrayColumnVector.isNullAt(i)) { - mapColumnVector.setNullAt(i); - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java new file mode 100644 index 0000000000000..7ec70490dc45b --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.NestedPositionUtil; +import org.apache.hudi.table.format.cow.vector.HeapArrayVector; +import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; +import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; +import org.apache.hudi.table.format.cow.vector.position.RowPosition; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; +import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField; +import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.MultisetType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.Preconditions; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code + * Row}). Resolves nested structures using Dremel striping/assembly; see the + * striping and assembly algorithms from the Dremel paper. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream: + * + *

      + *
    • Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code + * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code + * HeapMapVector}/{@code HeapArrayVector}. + *
    • Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a + * {@link RowType} may contain {@code null} children — meaning the corresponding logical + * field is absent from the Parquet file. Those slots are passed through unchanged and do + * not contribute to the row's repetition/definition-level stream. + *
    + */ +public class NestedColumnReader implements ColumnReader { + + private final Map columnReaders; + private final boolean isUtcTimestamp; + + private final PageReadStore pages; + + private final ParquetField field; + + public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) { + this.isUtcTimestamp = isUtcTimestamp; + this.pages = pages; + this.field = field; + this.columnReaders = new HashMap<>(); + } + + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + readData(field, readNumber, vector, false); + } + + private Tuple2 readData( + ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException { + if (field.getType() instanceof RowType) { + return readRow((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) { + return readMap((ParquetGroupField) field, readNumber, vector, inside); + } else if (field.getType() instanceof ArrayType) { + return readArray((ParquetGroupField) field, readNumber, vector, inside); + } else { + return readPrimitive((ParquetPrimitiveField) field, readNumber, vector); + } + } + + private Tuple2 readRow( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector; + LevelDelegation levelDelegation = null; + List children = field.getChildren(); + WritableColumnVector[] childrenVectors = heapRowVector.getFields(); + WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length]; + for (int i = 0; i < children.size(); i++) { + ParquetField child = children.get(i); + if (child == null) { + // Schema-evolution: the logical field is not present in the Parquet file. The slot + // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector + // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by + // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null + // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays + // NULL instead of reverting to the type's zero value. Skip contributing to the level stream. + childrenVectors[i].fillWithNulls(); + finalChildrenVectors[i] = childrenVectors[i]; + continue; + } + Tuple2 tuple = + readData(child, readNumber, childrenVectors[i], true); + levelDelegation = tuple.f0; + finalChildrenVectors[i] = tuple.f1; + } + if (levelDelegation == null) { + throw new FlinkRuntimeException( + String.format("Row field does not have any non-null children: %s.", field)); + } + + RowPosition rowPosition = + NestedPositionUtil.calculateRowOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If row was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors); + } else { + heapRowVector.setFields(finalChildrenVectors); + } + + if (rowPosition.getIsNull() != null) { + setFieldNullFlag(rowPosition.getIsNull(), heapRowVector); + } + + // Hudi-specific: collapse a present row whose every child is null into a null row, so that a + // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the + // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely + // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the + // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. + int rowCount = rowPosition.getPositionsCount(); + for (int j = 0; j < rowCount; j++) { + if (heapRowVector.isNullAt(j)) { + continue; + } + boolean allChildrenNull = true; + for (WritableColumnVector child : finalChildrenVectors) { + if (!child.isNullAt(j)) { + allChildrenNull = false; + break; + } + } + if (allChildrenNull) { + heapRowVector.setNullAt(j); + } + } + return Tuple2.of(levelDelegation, heapRowVector); + } + + private Tuple2 readMap( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapMapColumnVector mapVector = (HeapMapColumnVector) vector; + mapVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 2, + "Maps must have two type parameters, found %s", + children.size()); + Tuple2 keyTuple = + readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true); + Tuple2 valueTuple = + readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true); + + LevelDelegation levelDelegation = keyTuple.f0; + + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If map was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1); + } else { + mapVector.setKeys(keyTuple.f1); + mapVector.setValues(valueTuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), mapVector); + } + + mapVector.setLengths(collectionPosition.getLength()); + mapVector.setOffsets(collectionPosition.getOffsets()); + + return Tuple2.of(levelDelegation, mapVector); + } + + private Tuple2 readArray( + ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside) + throws IOException { + HeapArrayVector arrayVector = (HeapArrayVector) vector; + arrayVector.reset(); + List children = field.getChildren(); + Preconditions.checkArgument( + children.size() == 1, + "Arrays must have a single type parameter, found %s", + children.size()); + Tuple2 tuple = + readData(children.get(0), readNumber, arrayVector.getChild(), true); + + LevelDelegation levelDelegation = tuple.f0; + CollectionPosition collectionPosition = + NestedPositionUtil.calculateCollectionOffsets( + field, + levelDelegation.getDefinitionLevel(), + levelDelegation.getRepetitionLevel()); + + // If array was inside the structure, then we need to renew the vector to reset the + // capacity. + if (inside) { + arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1); + } else { + arrayVector.setChild(tuple.f1); + } + + if (collectionPosition.getIsNull() != null) { + setFieldNullFlag(collectionPosition.getIsNull(), arrayVector); + } + arrayVector.setLengths(collectionPosition.getLength()); + arrayVector.setOffsets(collectionPosition.getOffsets()); + return Tuple2.of(levelDelegation, arrayVector); + } + + private Tuple2 readPrimitive( + ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException { + ColumnDescriptor descriptor = field.getDescriptor(); + NestedPrimitiveColumnReader reader = columnReaders.get(descriptor); + if (reader == null) { + reader = + new NestedPrimitiveColumnReader( + descriptor, + pages.getPageReader(descriptor), + isUtcTimestamp, + descriptor.getPrimitiveType(), + field.getType()); + columnReaders.put(descriptor, reader); + } + WritableColumnVector writableColumnVector = + reader.readAndNewVector(readNumber, (WritableColumnVector) vector); + return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); + } + + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { + for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { + if (nullFlags[index]) { + vector.setNullAt(index); + } + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java new file mode 100644 index 0000000000000..a18520c3b5cd5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java @@ -0,0 +1,638 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.hudi.table.format.cow.utils.IntArrayList; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; + +import org.apache.flink.formats.parquet.vector.reader.ColumnReader; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; +import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; +import static org.apache.parquet.column.ValuesType.VALUES; + +/** + * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure. + * + *

    Vendored from Apache Flink 2.1 (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package + * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList} + * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code + * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf- + * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it + * a faithful copy of upstream. + */ +public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); + + private final IntArrayList repetitionLevelList = new IntArrayList(0); + private final IntArrayList definitionLevelList = new IntArrayList(0); + + private final PageReader pageReader; + private final ColumnDescriptor descriptor; + private final Type type; + private final LogicalType logicalType; + + /** The dictionary, if this column has dictionary encoding. */ + private final ParquetDataColumnReader dictionary; + + /** Maximum definition level for this column. */ + private final int maxDefLevel; + + private boolean isUtcTimestamp; + + /** Total number of values read. */ + private long valuesRead; + + /** + * value that indicates the end of the current page. That is, if valuesRead == + * endOfPageValueCount, we are at the end of the page. + */ + private long endOfPageValueCount; + + /** If true, the current page is dictionary encoded. */ + private boolean isCurrentPageDictionaryEncoded; + + private int definitionLevel; + private int repetitionLevel; + + /** Repetition/Definition/Value readers. */ + private IntIterator repetitionLevelColumn; + + private IntIterator definitionLevelColumn; + private ParquetDataColumnReader dataColumn; + + /** Total values in the current page. */ + private int pageValueCount; + + // flag to indicate if there is no data in parquet data page + private boolean eof = false; + + private boolean isFirstRow = true; + + private Object lastValue; + + public NestedPrimitiveColumnReader( + ColumnDescriptor descriptor, + PageReader pageReader, + boolean isUtcTimestamp, + Type parquetType, + LogicalType logicalType) + throws IOException { + this.descriptor = descriptor; + this.type = parquetType; + this.pageReader = pageReader; + this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.isUtcTimestamp = isUtcTimestamp; + this.logicalType = logicalType; + + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); + if (dictionaryPage != null) { + try { + this.dictionary = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + parquetType.asPrimitiveType(), + dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } catch (IOException e) { + throw new IOException( + String.format("Could not decode the dictionary for %s", descriptor), e); + } + } else { + this.dictionary = null; + this.isCurrentPageDictionaryEncoded = false; + } + } + + // Not invoked directly; callers use readAndNewVector instead. + @Override + public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { + throw new UnsupportedOperationException("This function should not be called."); + } + + public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector) + throws IOException { + if (isFirstRow) { + if (!readValue()) { + return vector; + } + isFirstRow = false; + } + + // index to set value. + int index = 0; + int valueIndex = 0; + List valueList = new ArrayList<>(); + + // repeated type need two loops to read data. + while (!eof && index < readNumber) { + do { + valueList.add(lastValue); + valueIndex++; + } while (readValue() && (repetitionLevel != 0)); + index++; + } + + return fillColumnVector(valueIndex, valueList); + } + + public LevelDelegation getLevelDelegation() { + int[] repetition = repetitionLevelList.toArray(); + int[] definition = definitionLevelList.toArray(); + repetitionLevelList.clear(); + definitionLevelList.clear(); + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + return new LevelDelegation(repetition, definition); + } + + private boolean readValue() throws IOException { + int left = readPageIfNeed(); + if (left > 0) { + // get the values of repetition and definitionLevel + readAndSaveRepetitionAndDefinitionLevels(); + // read the data if it isn't null + if (definitionLevel == maxDefLevel) { + if (isCurrentPageDictionaryEncoded) { + int dictionaryId = dataColumn.readValueDictionaryId(); + lastValue = dictionaryDecodeValue(logicalType, dictionaryId); + } else { + lastValue = readPrimitiveTypedRow(logicalType); + } + } else { + lastValue = null; + } + return true; + } else { + eof = true; + return false; + } + } + + private void readAndSaveRepetitionAndDefinitionLevels() { + // get the values of repetition and definitionLevel + repetitionLevel = repetitionLevelColumn.nextInt(); + definitionLevel = definitionLevelColumn.nextInt(); + valuesRead++; + repetitionLevelList.add(repetitionLevel); + definitionLevelList.add(definitionLevel); + } + + private int readPageIfNeed() throws IOException { + // Compute the number of values we want to read in this page. + int leftInPage = (int) (endOfPageValueCount - valuesRead); + if (leftInPage == 0) { + // no data left in current page, load data from new page + readPage(); + leftInPage = (int) (endOfPageValueCount - valuesRead); + } + return leftInPage; + } + + private Object readPrimitiveTypedRow(LogicalType category) { + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dataColumn.readBytes(); + case BOOLEAN: + return dataColumn.readBoolean(); + case TIME_WITHOUT_TIME_ZONE: + case DATE: + case INTEGER: + return dataColumn.readInteger(); + case TINYINT: + return dataColumn.readTinyInt(); + case SMALLINT: + return dataColumn.readSmallInt(); + case BIGINT: + return dataColumn.readLong(); + case FLOAT: + return dataColumn.readFloat(); + case DOUBLE: + return dataColumn.readDouble(); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dataColumn.readInteger(); + case INT64: + return dataColumn.readLong(); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return dataColumn.readBytes(); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dataColumn.readTimestamp(); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) { + if (dictionaryValue == null) { + return null; + } + + switch (category.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return dictionary.readBytes(dictionaryValue); + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTEGER: + return dictionary.readInteger(dictionaryValue); + case BOOLEAN: + return dictionary.readBoolean(dictionaryValue) ? 1 : 0; + case DOUBLE: + return dictionary.readDouble(dictionaryValue); + case FLOAT: + return dictionary.readFloat(dictionaryValue); + case TINYINT: + return dictionary.readTinyInt(dictionaryValue); + case SMALLINT: + return dictionary.readSmallInt(dictionaryValue); + case BIGINT: + return dictionary.readLong(dictionaryValue); + case DECIMAL: + switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + return dictionary.readInteger(dictionaryValue); + case INT64: + return dictionary.readLong(dictionaryValue); + case FIXED_LEN_BYTE_ARRAY: + case BINARY: + return dictionary.readBytes(dictionaryValue); + default: + throw new RuntimeException( + "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return dictionary.readTimestamp(dictionaryValue); + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private WritableColumnVector fillColumnVector(int total, List valueList) { + switch (logicalType.getTypeRoot()) { + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + HeapBytesVector heapBytesVector = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (src == null) { + heapBytesVector.setNullAt(i); + } else { + heapBytesVector.appendBytes(i, src, 0, src.length); + } + } + return heapBytesVector; + case BOOLEAN: + HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapBooleanVector.setNullAt(i); + } else { + heapBooleanVector.vector[i] = ((List) valueList).get(i); + } + } + return heapBooleanVector; + case TINYINT: + HeapByteVector heapByteVector = new HeapByteVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapByteVector.setNullAt(i); + } else { + heapByteVector.vector[i] = (byte) ((List) valueList).get(i).intValue(); + } + } + return heapByteVector; + case SMALLINT: + HeapShortVector heapShortVector = new HeapShortVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapShortVector.setNullAt(i); + } else { + heapShortVector.vector[i] = (short) ((List) valueList).get(i).intValue(); + } + } + return heapShortVector; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + HeapIntVector heapIntVector = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapIntVector.setNullAt(i); + } else { + heapIntVector.vector[i] = ((List) valueList).get(i); + } + } + return heapIntVector; + case FLOAT: + HeapFloatVector heapFloatVector = new HeapFloatVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapFloatVector.setNullAt(i); + } else { + heapFloatVector.vector[i] = ((List) valueList).get(i); + } + } + return heapFloatVector; + case BIGINT: + HeapLongVector heapLongVector = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapLongVector.setNullAt(i); + } else { + heapLongVector.vector[i] = ((List) valueList).get(i); + } + } + return heapLongVector; + case DOUBLE: + HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapDoubleVector.setNullAt(i); + } else { + heapDoubleVector.vector[i] = ((List) valueList).get(i); + } + } + return heapDoubleVector; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + heapTimestampVector.setNullAt(i); + } else { + heapTimestampVector.setTimestamp(i, ((List) valueList).get(i)); + } + } + return heapTimestampVector; + case DECIMAL: + PrimitiveType.PrimitiveTypeName primitiveTypeName = + descriptor.getPrimitiveType().getPrimitiveTypeName(); + switch (primitiveTypeName) { + case INT32: + HeapIntVector phiv = new HeapIntVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phiv.setNullAt(i); + } else { + phiv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phiv); + case INT64: + HeapLongVector phlv = new HeapLongVector(total); + for (int i = 0; i < valueList.size(); i++) { + if (valueList.get(i) == null) { + phlv.setNullAt(i); + } else { + phlv.vector[i] = ((List) valueList).get(i); + } + } + return new ParquetDecimalVector(phlv); + default: + HeapBytesVector phbv = getHeapBytesVector(total, valueList); + return new ParquetDecimalVector(phbv); + } + default: + throw new RuntimeException("Unsupported type in the list: " + type); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static HeapBytesVector getHeapBytesVector(int total, List valueList) { + HeapBytesVector phbv = new HeapBytesVector(total); + for (int i = 0; i < valueList.size(); i++) { + byte[] src = ((List) valueList).get(i); + if (valueList.get(i) == null) { + phbv.setNullAt(i); + } else { + phbv.appendBytes(i, src, 0, src.length); + } + } + return phbv; + } + + protected void readPage() { + DataPage page = pageReader.readPage(); + + if (page == null) { + return; + } + + page.accept( + new DataPage.Visitor() { + @Override + public Void visit(DataPageV1 dataPageV1) { + readPageV1(dataPageV1); + return null; + } + + @Override + public Void visit(DataPageV2 dataPageV2) { + readPageV2(dataPageV2); + return null; + } + }); + } + + private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) + throws IOException { + this.pageValueCount = valueCount; + this.endOfPageValueCount = valuesRead + pageValueCount; + if (dataEncoding.usesDictionary()) { + this.dataColumn = null; + if (dictionary == null) { + throw new IOException( + String.format( + "Could not read page in col %s because the dictionary was missing for encoding %s.", + descriptor, dataEncoding)); + } + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getDictionaryBasedValuesReader( + descriptor, VALUES, dictionary.getDictionary()), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = true; + } else { + dataColumn = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type.asPrimitiveType(), + dataEncoding.getValuesReader(descriptor, VALUES), + isUtcTimestamp); + this.isCurrentPageDictionaryEncoded = false; + } + + try { + dataColumn.initFromPage(pageValueCount, in); + } catch (IOException e) { + throw new IOException(String.format("Could not read page in col %s.", descriptor), e); + } + } + + private void readPageV1(DataPageV1 page) { + ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); + ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); + this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); + this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); + try { + BytesInput bytes = page.getBytes(); + LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount); + ByteBufferInputStream in = bytes.toInputStream(); + LOG.debug("Reading repetition levels at {}.", in.position()); + rlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading definition levels at {}.", in.position()); + dlReader.initFromPage(pageValueCount, in); + LOG.debug("Reading data at {}.", in.position()); + initDataReader(page.getValueEncoding(), in, page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private void readPageV2(DataPageV2 page) { + this.pageValueCount = page.getValueCount(); + this.repetitionLevelColumn = + newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); + this.definitionLevelColumn = + newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); + try { + LOG.debug( + "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); + initDataReader( + page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read page %s in col %s.", page, descriptor), e); + } + } + + private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) { + try { + if (maxLevel == 0) { + return new NullIntIterator(); + } + return new RLEIntIterator( + new RunLengthBitPackingHybridDecoder( + BytesUtils.getWidthFromMaxInt(maxLevel), + new ByteArrayInputStream(bytes.toByteArray()))); + } catch (IOException e) { + throw new ParquetDecodingException( + String.format("Could not read levels in page for col %s.", descriptor), e); + } + } + + /** Utility interface to abstract over different way to read ints with different encodings. */ + interface IntIterator { + int nextInt(); + } + + /** Reading int from {@link ValuesReader}. */ + protected static final class ValuesReaderIntIterator implements IntIterator { + ValuesReader delegate; + + public ValuesReaderIntIterator(ValuesReader delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + return delegate.readInteger(); + } + } + + /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */ + protected static final class RLEIntIterator implements IntIterator { + RunLengthBitPackingHybridDecoder delegate; + + public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) { + this.delegate = delegate; + } + + @Override + public int nextInt() { + try { + return delegate.readInt(); + } catch (IOException e) { + throw new ParquetDecodingException(e); + } + } + } + + /** Reading zero always. */ + protected static final class NullIntIterator implements IntIterator { + @Override + public int nextInt() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index 3572b117a6313..1826419db5d44 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -18,7 +18,9 @@ package org.apache.hudi.table.format.cow.vector.reader; +import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; +import org.apache.hudi.table.format.cow.vector.type.ParquetField; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.table.data.RowData; @@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,6 +42,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -46,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -53,7 +59,6 @@ import java.util.Map; import java.util.stream.IntStream; -import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; @@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable { private final MessageType requestedSchema; + /** + * {@link ParquetField} tree per top-level requested column, used by + * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List, + * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for + * nested types. Entries are {@code null} for primitive top-level fields. Built once per split. + */ + private final List requestedFields; + /** * The total number of rows this RecordReader will eventually read. The sum of the rows of all * the row groups. @@ -158,6 +171,20 @@ public ParquetColumnarRowSplitReader( checkSchema(); + // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across + // row groups). Only columns with nested logical type get a non-null entry — primitive columns + // still use Hudi's specialized ColumnReaders. + MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema); + List requestedRowFields = new ArrayList<>(requestedTypes.length); + List requestedFieldNames = new ArrayList<>(requestedTypes.length); + for (int i = 0; i < requestedTypes.length; i++) { + String name = requestedSchema.getFieldName(i); + requestedRowFields.add(new RowType.RowField(name, requestedTypes[i])); + requestedFieldNames.add(name); + } + this.requestedFields = ParquetSplitReaderUtil.buildFieldsList( + requestedRowFields, requestedFieldNames, messageColumnIO); + this.writableVectors = createWritableVectors(); ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices); this.columnarBatch = generator.generate(columnVectors); @@ -340,12 +367,13 @@ private void readNextRowGroup() throws IOException { List columns = requestedSchema.getColumns(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = createColumnReader( + columnReaders[i] = ParquetSplitReaderUtil.createColumnReader( utcTimestamp, requestedTypes[i], types.get(i), columns, - pages); + pages, + requestedFields.get(i)); } totalCountLoadedSoFar += pages.getRowCount(); } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java index fdfe5d6fa3a33..1abc6ed56c0db 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java @@ -26,12 +26,16 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS; import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY; @@ -252,21 +256,115 @@ public TimestampData readTimestamp() { } } + /** + * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard + * timestamp encoding defined by Parquet's + * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy + * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations. + * (The older INT96 encoding is marked deprecated by the Parquet format spec — see + * + * LogicalTypes.md — but is still supported here via {@link TypesFromInt96PageReader} for + * backwards compatibility with files written by older Hive / Spark / Impala versions.) + * + *

    Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a + * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use + * {@link Int64TimestampColumnReader} for batched-vector efficiency. + */ + public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader { + private final boolean isUtcTimestamp; + private final ChronoUnit chronoUnit; + + public TypesFromInt64PageReader( + ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(realReader); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + public TypesFromInt64PageReader( + Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) { + super(dict); + this.isUtcTimestamp = isUtcTimestamp; + this.chronoUnit = chronoUnit; + } + + @Override + public TimestampData readTimestamp() { + return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit); + } + + @Override + public TimestampData readTimestamp(int id) { + return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper( boolean isDictionary, PrimitiveType parquetType, Dictionary dictionary, ValuesReader valuesReader, boolean isUtcTimestamp) { - if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { + PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName(); + if (typeName == PrimitiveType.PrimitiveTypeName.INT96) { return isDictionary ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp) : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp); - } else { - return isDictionary - ? new DefaultParquetDataColumnReader(dictionary) - : new DefaultParquetDataColumnReader(valuesReader); } + if (typeName == PrimitiveType.PrimitiveTypeName.INT64) { + ChronoUnit unit = resolveInt64TimestampUnit(parquetType); + if (unit != null) { + return isDictionary + ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit) + : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit); + } + } + return isDictionary + ? new DefaultParquetDataColumnReader(dictionary) + : new DefaultParquetDataColumnReader(valuesReader); + } + + /** + * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the + * column is a plain INT64 (not a timestamp). + * + *

    Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and + * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} + * encodings. + */ + private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) { + LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); + if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + LogicalTypeAnnotation.TimeUnit unit = + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit(); + switch (unit) { + case MILLIS: + return ChronoUnit.MILLIS; + case MICROS: + return ChronoUnit.MICROS; + case NANOS: + return ChronoUnit.NANOS; + default: + return null; + } + } + OriginalType originalType = parquetType.getOriginalType(); + if (originalType == OriginalType.TIMESTAMP_MILLIS) { + return ChronoUnit.MILLIS; + } + if (originalType == OriginalType.TIMESTAMP_MICROS) { + return ChronoUnit.MICROS; + } + return null; + } + + private static TimestampData int64ToTimestamp( + boolean isUtcTimestamp, long value, ChronoUnit unit) { + Instant instant = Instant.EPOCH.plus(value, unit); + if (isUtcTimestamp) { + return TimestampData.fromInstant(instant); + } + return TimestampData.fromTimestamp(Timestamp.from(instant)); } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( @@ -281,10 +379,10 @@ public static ParquetDataColumnReader getDataColumnReaderByType( } private static TimestampData int96ToTimestamp( - boolean utcTimestamp, long nanosOfDay, int julianDay) { + boolean isUtcTimestamp, long nanosOfDay, int julianDay) { long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND); - if (utcTimestamp) { + if (isUtcTimestamp) { int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND); return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond); } else { diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java deleted file mode 100644 index 79b50487f13c1..0000000000000 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.table.format.cow.vector.reader; - -import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; - -import org.apache.flink.formats.parquet.vector.reader.ColumnReader; -import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** - * Row {@link ColumnReader}. - */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector; - WritableColumnVector[] vectors = rowColumnVector.vectors; - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowColumnVector.setNullAt(j); - } - } - } - } -} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java new file mode 100644 index 0000000000000..0f5e00779a2f5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +/** + * Field that represent parquet's field type. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetField}). + */ +public abstract class ParquetField { + private final LogicalType type; + private final int repetitionLevel; + private final int definitionLevel; + private final boolean required; + + public ParquetField( + LogicalType type, int repetitionLevel, int definitionLevel, boolean required) { + this.type = type; + this.repetitionLevel = repetitionLevel; + this.definitionLevel = definitionLevel; + this.required = required; + } + + public LogicalType getType() { + return type; + } + + public int getRepetitionLevel() { + return repetitionLevel; + } + + public int getDefinitionLevel() { + return definitionLevel; + } + + public boolean isRequired() { + return required; + } + + @Override + public String toString() { + return "Field{" + + "type=" + + type + + ", repetitionLevel=" + + repetitionLevel + + ", definitionLevel=" + + definitionLevel + + ", required=" + + required + + '}'; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java new file mode 100644 index 0000000000000..f91dcca965d64 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's Group Field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension: + * entries in the {@code children} list may be {@code null} to denote a Row child that is absent + * from the parquet file but present in the requested logical schema (schema evolution). This + * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees. + */ +public class ParquetGroupField extends ParquetField { + + private final List children; + + public ParquetGroupField( + LogicalType type, + int repetitionLevel, + int definitionLevel, + boolean required, + List children) { + super(type, repetitionLevel, definitionLevel, required); + // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for + // schema-evolution missing children in ROW types. + this.children = + Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null"))); + } + + /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */ + public List getChildren() { + return children; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java new file mode 100644 index 0000000000000..f6af6f9ff479e --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.parquet.column.ColumnDescriptor; + +import static java.util.Objects.requireNonNull; + +/** + * Field that represent parquet's primitive field. + * + *

    Note: Vendored from Apache Flink (FLINK-35702, {@code + * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}). + */ +public class ParquetPrimitiveField extends ParquetField { + + private final ColumnDescriptor descriptor; + private final int id; + + public ParquetPrimitiveField( + LogicalType type, boolean required, ColumnDescriptor descriptor, int id) { + super( + type, + descriptor.getMaxRepetitionLevel(), + descriptor.getMaxDefinitionLevel(), + required); + this.descriptor = requireNonNull(descriptor, "descriptor is required"); + this.id = id; + } + + public ColumnDescriptor getDescriptor() { + return descriptor; + } + + public int getId() { + return id; + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java new file mode 100644 index 0000000000000..aff1c32917cf9 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector}, + * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's + * nested-Parquet reader (FLINK-35702). + * + *

    The accessors are wrappers over the existing public fields so legacy callers continue to + * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the + * Dremel-style read path is exercised end-to-end by integration tests in + * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes / + * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes). + */ +class TestHeapColumnVectorAccessors { + + // ----------------------------------------------------------------------------------------------- + // HeapArrayVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapArrayVectorAccessorsReflectPublicFields() { + HeapIntVector child = new HeapIntVector(4); + HeapArrayVector vector = new HeapArrayVector(2, child); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector replacementChild = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setChild(replacementChild); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(replacementChild, vector.getChild()); + assertEquals(2, vector.getSize()); + + // Backing public fields are kept in sync — preserves backward compatibility. + assertSame(offsets, vector.offsets); + assertSame(lengths, vector.lengths); + assertSame(replacementChild, vector.child); + } + + // ----------------------------------------------------------------------------------------------- + // HeapMapColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + + HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values); + + assertEquals(3, vector.getOffsets().length); + assertEquals(3, vector.getLengths().length); + } + + @Test + void heapMapColumnVectorAccessorsReflectInternalState() { + HeapIntVector keys = new HeapIntVector(4); + HeapIntVector values = new HeapIntVector(4); + HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values); + + long[] offsets = {0L, 2L}; + long[] lengths = {2L, 2L}; + HeapLongVector newKeys = new HeapLongVector(4); + HeapLongVector newValues = new HeapLongVector(4); + + vector.setOffsets(offsets); + vector.setLengths(lengths); + vector.setKeys(newKeys); + vector.setValues(newValues); + vector.setSize(2); + + assertArrayEquals(offsets, vector.getOffsets()); + assertArrayEquals(lengths, vector.getLengths()); + assertSame(newKeys, vector.getKeys()); + assertSame(newValues, vector.getValues()); + // The Flink-2.1-style ColumnVector accessors return the same underlying child. + assertSame(newKeys, vector.getKeyColumnVector()); + assertSame(newValues, vector.getValueColumnVector()); + assertEquals(2, vector.getSize()); + } + + // ----------------------------------------------------------------------------------------------- + // HeapRowColumnVector + // ----------------------------------------------------------------------------------------------- + + @Test + void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() { + HeapIntVector intField = new HeapIntVector(2); + HeapLongVector longField = new HeapLongVector(2); + HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField); + + WritableColumnVector[] originalFields = vector.getFields(); + assertEquals(2, originalFields.length); + assertSame(intField, originalFields[0]); + assertSame(longField, originalFields[1]); + // Backing public field is kept in sync — preserves backward compatibility. + assertSame(originalFields, vector.vectors); + + HeapIntVector replacement = new HeapIntVector(2); + WritableColumnVector[] replacementFields = {replacement, longField}; + vector.setFields(replacementFields); + + assertSame(replacementFields, vector.getFields()); + assertSame(replacementFields, vector.vectors); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java new file mode 100644 index 0000000000000..04f5809b9dac4 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.columnar.vector.BytesColumnVector; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector; +import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ParquetDecimalVector}. + */ +public class TestParquetDecimalVector { + + @Test + void testGetDecimalFromInt32Vector() { + // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true + HeapIntVector intVector = new HeapIntVector(1); + intVector.vector[0] = 12345; + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromInt64Vector() { + // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true + HeapLongVector longVector = new HeapLongVector(1); + longVector.vector[0] = 1234567890123456L; + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + DecimalData decoded = wrapped.getDecimal(0, 18, 4); + + assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtLargePrecision() { + // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path + BigDecimal original = new BigDecimal("12345678901234567890.1234567890"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 30, 10); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalFromBytesVectorAtSmallPrecision() { + // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the + // dispatch must fall through to the bytes branch rather than require an IntColumnVector. + BigDecimal original = new BigDecimal("123.45"); + byte[] unscaled = original.unscaledValue().toByteArray(); + HeapBytesVector bytesVector = new HeapBytesVector(1); + bytesVector.appendBytes(0, unscaled, 0, unscaled.length); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + + DecimalData decoded = wrapped.getDecimal(0, 5, 2); + + assertEquals(original, decoded.toBigDecimal()); + } + + @Test + void testGetDecimalThrowsOnUnsupportedVectorType() { + // A large-precision request must have a bytes-backed child; any other writable child is an + // illegal combination and must be surfaced via Preconditions.checkArgument. + ColumnVector unsupported = new HeapShortVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported); + + assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10)); + } + + @Test + void testIsNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + intVector.vector[0] = 1; + intVector.setNullAt(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + assertFalse(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testWritableIntRoundTrip() { + HeapIntVector intVector = new HeapIntVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setInt(0, 42); + + assertEquals(42, wrapped.getInt(0)); + assertEquals(42, intVector.vector[0]); + } + + @Test + void testWritableLongRoundTrip() { + HeapLongVector longVector = new HeapLongVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector); + + wrapped.setLong(0, 9876543210L); + + assertEquals(9876543210L, wrapped.getLong(0)); + assertEquals(9876543210L, longVector.vector[0]); + } + + @Test + void testWritableBytesRoundTrip() { + HeapBytesVector bytesVector = new HeapBytesVector(1); + ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector); + byte[] payload = new byte[] {0x01, 0x02, 0x03}; + + wrapped.appendBytes(0, payload, 0, payload.length); + + BytesColumnVector.Bytes out = wrapped.getBytes(0); + assertEquals(payload.length, out.len); + assertEquals(0x01, out.data[out.offset]); + assertEquals(0x02, out.data[out.offset + 1]); + assertEquals(0x03, out.data[out.offset + 2]); + } + + @Test + void testResetDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(1); + intVector.setNullAt(0); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + assertTrue(wrapped.isNullAt(0)); + + wrapped.reset(); + + assertFalse(wrapped.isNullAt(0)); + } + + @Test + void testFillWithNullsDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.fillWithNulls(); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } + + @Test + void testSetNullAtDelegatesToChild() { + HeapIntVector intVector = new HeapIntVector(2); + ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector); + + wrapped.setNullAt(0); + wrapped.setNulls(1, 1); + + assertTrue(wrapped.isNullAt(0)); + assertTrue(wrapped.isNullAt(1)); + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java new file mode 100644 index 0000000000000..ea222dad576a5 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.reader; + +import org.apache.flink.table.data.TimestampData; +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.OriginalType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when + * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702). + * + *

    The factory is exercised end-to-end by integration tests through + * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece + * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each + * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} + * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values + * using both the values-reader and dictionary code paths. + */ +class TestParquetDataColumnReaderFactory { + + // ----------------------------------------------------------------------------------------------- + // Type dispatch + // ----------------------------------------------------------------------------------------------- + + @Test + void valuesReaderDispatchInt96TimestampUsesInt96Reader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MILLIS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(OriginalType.TIMESTAMP_MICROS) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + @Test + void valuesReaderDispatchInt32DoesNotUseTimestampReader() { + PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader); + } + + @Test + void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(), true); + assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader); + } + + // ----------------------------------------------------------------------------------------------- + // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches) + // ----------------------------------------------------------------------------------------------- + + @Test + void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_123L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMillis), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + assertEquals(0, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS)) + .named("ts"); + long epochMicros = 1_700_000_000_123_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochMicros), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochMicros / 1_000L, ts.getMillisecond()); + // 456 microseconds remain → 456_000 nanoseconds within the millisecond + assertEquals(456_000, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS)) + .named("ts"); + long epochNanos = 1_700_000_000_123_456_789L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByType( + type, new StubValuesReader(epochNanos), true); + + TimestampData ts = reader.readTimestamp(); + assertNotNull(ts); + assertEquals(epochNanos / 1_000_000L, ts.getMillisecond()); + assertEquals(456_789, ts.getNanoOfMillisecond()); + } + + @Test + void int64ReaderReadsTimestampMillisFromDictionaryInUtc() { + PrimitiveType type = + Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)) + .named("ts"); + long epochMillis = 1_700_000_000_456L; + ParquetDataColumnReader reader = + ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary( + type, new StubDictionary(epochMillis), true); + + TimestampData ts = reader.readTimestamp(0); + assertNotNull(ts); + assertEquals(epochMillis, ts.getMillisecond()); + } + + // ----------------------------------------------------------------------------------------------- + // Stubs (only the methods exercised by the dispatch + decoding tests above) + // ----------------------------------------------------------------------------------------------- + + /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */ + private static final class StubValuesReader extends ValuesReader { + private final long fixedLong; + + StubValuesReader() { + this(0L); + } + + StubValuesReader(long fixedLong) { + this.fixedLong = fixedLong; + } + + @Override + public long readLong() { + return fixedLong; + } + + @Override + public void skip() { + // unused + } + } + + /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */ + private static final class StubDictionary extends Dictionary { + private final long fixedLong; + + StubDictionary() { + this(0L); + } + + StubDictionary(long fixedLong) { + super(null); + this.fixedLong = fixedLong; + } + + @Override + public Binary decodeToBinary(int id) { + throw new UnsupportedOperationException(); + } + + @Override + public long decodeToLong(int id) { + return fixedLong; + } + + @Override + public int getMaxId() { + return 0; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java new file mode 100644 index 0000000000000..2b71bae4dc152 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.format.cow.vector.type; + +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link ParquetGroupField}. + */ +public class TestParquetGroupField { + + @Test + void testChildrenWithAllNonNullEntriesAreRetained() { + ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1); + List children = Arrays.asList(c0, c1); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertSame(c0, group.getChildren().get(0)); + assertSame(c1, group.getChildren().get(1)); + } + + @Test + void testChildrenMayContainNullForSchemaEvolution() { + // A ROW field present in the requested Flink schema but absent from the Parquet file is + // represented by a null slot in `children`. The group must allow this (Hudi-specific + // extension over Flink's ImmutableList-backed equivalent). + ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List children = Arrays.asList(present, null); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children); + + assertEquals(2, group.getChildren().size()); + assertNotNull(group.getChildren().get(0)); + assertNull(group.getChildren().get(1)); + } + + @Test + void testChildrenListIsUnmodifiable() { + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child)); + + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null)); + assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0)); + } + + @Test + void testChildrenListIsDefensivelyCopied() { + // Mutations to the caller-supplied list must not be visible through the group. + ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0); + List mutable = new ArrayList<>(); + mutable.add(child); + + ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable); + mutable.add(null); + + assertEquals(1, group.getChildren().size()); + } + + @Test + void testNullChildrenListThrows() { + assertThrows( + NullPointerException.class, + () -> new ParquetGroupField(rowType(), 0, 1, true, null)); + } + + @Test + void testEmptyChildrenListIsAllowed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList()); + + assertEquals(0, group.getChildren().size()); + } + + @Test + void testFieldMetadataIsExposed() { + ParquetGroupField group = + new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList()); + + assertEquals(2, group.getRepetitionLevel()); + assertEquals(5, group.getDefinitionLevel()); + assertFalse(group.isRequired()); + } + + private static LogicalType rowType() { + return RowType.of(new IntType()); + } + + private static ColumnDescriptor descriptor() { + PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f"); + return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0); + } +} From 1025464a3d1e97bd4e5ef1dfa9a1c6d631a84b8f Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 11 Jun 2026 12:56:25 +0800 Subject: [PATCH 060/255] docs: RFC-106 - Record Level and Secondary Index Support for Flink Writers (#17610) (cherry picked from commit 84c1f591481d41bc8ffac313e5ae992db19ed3d5) --- rfc/README.md | 213 ++++++++++----------- rfc/rfc-106/index-compaction-flow.png | Bin 0 -> 329243 bytes rfc/rfc-106/index-write-flow.png | Bin 0 -> 375082 bytes rfc/rfc-106/rfc-106.md | 260 ++++++++++++++++++++++++++ rfc/rfc-106/rli-access-pattern.png | Bin 0 -> 96912 bytes 5 files changed, 368 insertions(+), 105 deletions(-) create mode 100644 rfc/rfc-106/index-compaction-flow.png create mode 100644 rfc/rfc-106/index-write-flow.png create mode 100644 rfc/rfc-106/rfc-106.md create mode 100644 rfc/rfc-106/rli-access-pattern.png diff --git a/rfc/README.md b/rfc/README.md index 20b1cd53529b1..512ee1bc9b012 100644 --- a/rfc/README.md +++ b/rfc/README.md @@ -34,108 +34,111 @@ The list of all RFCs can be found here. > Older RFC content is still [here](https://cwiki.apache.org/confluence/display/HUDI/RFC+Process). -| RFC Number | Title | Status | -|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------| -| 1 | [CSV Source Support for Delta Streamer](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+01+%3A+CSV+Source+Support+for+Delta+Streamer) | :white_check_mark: `COMPLETED` | -| 2 | [ORC Storage in Hudi](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=113708439) | :white_check_mark: `COMPLETED` | -| 3 | [Timeline Service with Incremental File System View Syncing](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=113708965) | :white_check_mark: `COMPLETED` | -| 4 | [Faster Hive incremental pull queries](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=115513622) | :white_check_mark: `COMPLETED` | -| 5 | [HUI (Hudi WebUI)](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=130027233) | :x: `ABANDONED` | -| 6 | [Add indexing support to the log file](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+06+%3A+Add+indexing+support+to+the+log+file) | :x: `ABANDONED` | -| 7 | [Point in time Time-Travel queries on Hudi table](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+07+%3A+Point+in+time+Time-Travel+queries+on+Hudi+table) | :white_check_mark: `COMPLETED` | -| 8 | [Metadata based Record Index](./rfc-8/rfc-8.md) | :white_check_mark: `COMPLETED` | -| 9 | [Hudi Dataset Snapshot Exporter](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+09+%3A+Hudi+Dataset+Snapshot+Exporter) | :white_check_mark: `COMPLETED` | -| 10 | [Restructuring and auto-generation of docs](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+10+%3A+Restructuring+and+auto-generation+of+docs) | :white_check_mark: `COMPLETED` | -| 11 | [Refactor of the configuration framework of hudi project](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+11+%3A+Refactor+of+the+configuration+framework+of+hudi+project) | :x: `ABANDONED` | -| 12 | [Efficient Migration of Large Parquet Tables to Apache Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+12+%3A+Efficient+Migration+of+Large+Parquet+Tables+to+Apache+Hudi) | :white_check_mark: `COMPLETED` | -| 13 | [Integrate Hudi with Flink](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=141724520) | :white_check_mark: `COMPLETED` | -| 14 | [JDBC incremental puller](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+14+%3A+JDBC+incremental+puller) | :white_check_mark: `COMPLETED` | -| 15 | [HUDI File Listing Improvements](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+15%3A+HUDI+File+Listing+Improvements) | :white_check_mark: `COMPLETED` | -| 16 | [Abstraction for HoodieInputFormat and RecordReader](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+16+Abstraction+for+HoodieInputFormat+and+RecordReader) | :white_check_mark: `COMPLETED` | -| 17 | [Abstract common meta sync module support multiple meta service](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+17+Abstract+common+meta+sync+module+support+multiple+meta+service) | :white_check_mark: `COMPLETED` | -| 18 | [Insert Overwrite API](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+18+Insert+Overwrite+API) | :white_check_mark: `COMPLETED` | -| 19 | [Clustering data for freshness and query performance](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance) | :white_check_mark: `COMPLETED` | -| 20 | [handle failed records](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+20+%3A+handle+failed+records) | :arrows_counterclockwise: `ONGOING` | -| 21 | [Allow HoodieRecordKey to be Virtual](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+21+%3A+Allow+HoodieRecordKey+to+be+Virtual) | :white_check_mark: `COMPLETED` | -| 22 | [Snapshot Isolation using Optimistic Concurrency Control for multi-writers](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+22+%3A+Snapshot+Isolation+using+Optimistic+Concurrency+Control+for+multi-writers) | :white_check_mark: `COMPLETED` | -| 23 | [Hudi Observability metrics collection](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+23+%3A+Hudi+Observability+metrics+collection) | :x: `ABANDONED` | -| 24 | [Hoodie Flink Writer Proposal](https://cwiki.apache.org/confluence/display/HUDI/RFC-24%3A+Hoodie+Flink+Writer+Proposal) | :white_check_mark: `COMPLETED` | -| 25 | [Spark SQL Extension For Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+25%3A+Spark+SQL+Extension+For+Hudi) | :white_check_mark: `COMPLETED` | -| 26 | [Optimization For Hudi Table Query](https://cwiki.apache.org/confluence/display/HUDI/RFC-26+Optimization+For+Hudi+Table+Query) | :white_check_mark: `COMPLETED` | -| 27 | [Data skipping index to improve query performance](https://cwiki.apache.org/confluence/display/HUDI/RFC-27+Data+skipping+index+to+improve+query+performance) | :white_check_mark: `COMPLETED` | -| 28 | [Support Z-order curve](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181307144) | :white_check_mark: `COMPLETED` | -| 29 | [Hash Index](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+29%3A+Hash+Index) | :white_check_mark: `COMPLETED` | -| 30 | [Batch operation](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+30%3A+Batch+operation) | :x: `ABANDONED` | -| 31 | [Hive integration Improvement](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+31%3A+Hive+integration+Improvment) | :x: `ABANDONED` | -| 32 | [Kafka Connect Sink for Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC-32+Kafka+Connect+Sink+for+Hudi) | :arrows_counterclockwise: `ONGOING` | -| 33 | [Hudi supports more comprehensive Schema Evolution](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+33++Hudi+supports+more+comprehensive+Schema+Evolution) | :white_check_mark: `COMPLETED` | -| 34 | [Hudi BigQuery Integration](./rfc-34/rfc-34.md) | :white_check_mark: `COMPLETED` | -| 35 | [Make Flink MOR table writing streaming friendly](https://cwiki.apache.org/confluence/display/HUDI/RFC-35%3A+Make+Flink+MOR+table+writing+streaming+friendly) | :white_check_mark: `COMPLETED` | -| 36 | [HUDI Metastore Server](https://cwiki.apache.org/confluence/display/HUDI/%5BWIP%5D+RFC-36%3A+HUDI+Metastore+Server) | :arrows_counterclockwise: `ONGOING` | -| 37 | [Hudi Metadata based Bloom Index](rfc-37/rfc-37.md) | :white_check_mark: `COMPLETED` | -| 38 | [Spark Datasource V2 Integration](./rfc-38/rfc-38.md) | :white_check_mark: `COMPLETED` | -| 39 | [Incremental source for Debezium](./rfc-39/rfc-39.md) | :white_check_mark: `COMPLETED` | -| 40 | [Connector for Trino](./rfc-40/rfc-40.md) | :white_check_mark: `COMPLETED` | -| 41 | [Snowflake Integration](./rfc-41/rfc-41.md), supported via [Apache XTable (Incubating)](https://xtable.apache.org/) | :x: `ABANDONED` | -| 42 | [Consistent Hashing Index](./rfc-42/rfc-42.md) | :arrows_counterclockwise: `ONGOING` | -| 43 | [Table Management Service](./rfc-43/rfc-43.md) | :x: `ABANDONED` | -| 44 | [Hudi Connector for Presto](./rfc-44/rfc-44.md) | :white_check_mark: `COMPLETED` | -| 45 | [Asynchronous Metadata Indexing](./rfc-45/rfc-45.md) | :white_check_mark: `COMPLETED` | -| 46 | [Optimizing Record Payload Handling](./rfc-46/rfc-46.md) | :white_check_mark: `COMPLETED` | -| 47 | [Add Call Produce Command for Spark SQL](./rfc-47/rfc-47.md) | :white_check_mark: `COMPLETED` | -| 48 | [LogCompaction for MOR tables](./rfc-48/rfc-48.md) | :white_check_mark: `COMPLETED` | -| 49 | [Support sync with DataHub](./rfc-49/rfc-49.md) | :white_check_mark: `COMPLETED` | -| 50 | [Improve Timeline Server](./rfc-50/rfc-50.md) | :x: `ABANDONED` | -| 51 | [Change Data Capture](./rfc-51/rfc-51.md) | :arrows_counterclockwise: `ONGOING` | -| 52 | [Introduce Secondary Index to Improve HUDI Query Performance](./rfc-52/rfc-52.md) | :x: `ABANDONED` | -| 53 | [Use Lock-Free Message Queue Improving Hoodie Writing Efficiency](./rfc-53/rfc-53.md) | :white_check_mark: `COMPLETED` | -| 54 | [New Table APIs and Streamline Hudi Configs](./rfc-54/rfc-54.md) | :x: `ABANDONED` | -| 55 | [Improve Hive/Meta sync class design and hierarchies](./rfc-55/rfc-55.md) | :white_check_mark: `COMPLETED` | -| 56 | [Early Conflict Detection For Multi-Writer](./rfc-56/rfc-56.md) | :white_check_mark: `COMPLETED` | -| 57 | [DeltaStreamer Protobuf Support](./rfc-57/rfc-57.md) | :white_check_mark: `COMPLETED` | -| 58 | [Integrate column stats index with all query engines](./rfc-58/rfc-58.md) | :white_check_mark: `COMPLETED` | -| 59 | [Multiple event_time Fields Latest Verification in a Single Table](./rfc-59/rfc-59.md) | :eyes: `UNDER REVIEW` | -| 60 | [Federated Storage Layer](./rfc-60/rfc-60.md) | :eyes: `UNDER REVIEW` | -| 61 | [Snapshot view management](./rfc-61/rfc-61.md) | :eyes: `UNDER REVIEW` | -| 62 | [Diagnostic Reporter](./rfc-62/rfc-62.md) | :eyes: `UNDER REVIEW` | -| 63 | [Expression Indexes](./rfc-63/rfc-63.md) | :arrows_counterclockwise: `ONGOING` | -| 64 | [New Hudi Table Spec API for Query Integrations](./rfc-64/rfc-64.md) | :eyes: `UNDER REVIEW` | -| 65 | [Partition TTL Management](./rfc-65/rfc-65.md) | :white_check_mark: `COMPLETED` | -| 66 | [Non Blocking Concurrency Control](./rfc-66/rfc-66.md) | :white_check_mark: `COMPLETED` | -| 67 | [Hudi Bundle Standards](./rfc-67/rfc-67.md) | :white_check_mark: `COMPLETED` | -| 68 | [A More Effective HoodieMergeHandler for COW Table with Parquet](./rfc-68/rfc-68.md) | :x: `ABANDONED` | -| 69 | [Hudi 1.x](./rfc-69/rfc-69.md) | :white_check_mark: `COMPLETED` | -| 70 | [Hudi Reverse Streamer](./rfc/rfc-70/rfc-70.md) | :eyes: `UNDER REVIEW` | -| 71 | [Enhance OCC conflict detection](./rfc/rfc-71/rfc-71.md) | :eyes: `UNDER REVIEW` | -| 72 | [Redesign Hudi-Spark Integration](./rfc/rfc-72/rfc-72.md) | :arrows_counterclockwise: `ONGOING` | -| 73 | [Multi-Table Transactions](./rfc-73/rfc-73.md) | :eyes: `UNDER REVIEW` | -| 74 | [`HoodieStorage`: Hudi Storage Abstraction and APIs](./rfc-74/rfc-74.md) | :arrows_counterclockwise: `ONGOING` | -| 75 | [Hudi-Native HFile Reader and Writer](./rfc-75/rfc-75.md) | :white_check_mark: `COMPLETED` | -| 76 | [Auto Record key generation](./rfc-76/rfc-76.md) | :white_check_mark: `COMPLETED` | -| 77 | [Secondary Index](./rfc-77/rfc-77.md) | :white_check_mark: `COMPLETED` | -| 78 | [1.0 Migration](./rfc-78/rfc-78.md) | :hammer_and_wrench: `IN PROGRESS` | -| 79 | [Robust handling of spark task retries and failures](./rfc-79/rfc-79.md) | :x: `ABANDONED` | -| 80 | [Column Groups](./rfc-80/rfc-80.md) | :hammer_and_wrench: `IN PROGRESS` | -| 81 | [Introduce Primary Key Sorted Table](./rfc-81/rfc-81.md) | :eyes: `UNDER REVIEW` | -| 82 | [Concurrent schema evolution detection](./rfc-82/rfc-82.md) | :white_check_mark: `COMPLETED` | -| 83 | [Incremental Table Service](./rfc-83/rfc-83.md) | :white_check_mark: `COMPLETED` | -| 84 | [Optimized SerDe of `DataStream` in Flink operators](./rfc-84/rfc-84.md) | :white_check_mark: `COMPLETED` | -| 85 | [Hudi Issue and Sprint Management in Jira](./rfc-85/rfc-85.md) | :white_check_mark: `COMPLETED` | -| 86 | [DataFrame Implementation of HUDI write path](./rfc-86/rfc-86.md) | :eyes: `UNDER REVIEW` | -| 87 | [Avro elimination for Flink writer](./rfc-87/rfc-87.md) | :hammer_and_wrench: `IN PROGRESS` | -| 88 | [New Schema/DataType/Expression Abstractions](./rfc-88/rfc-88.md) | :eyes: `UNDER REVIEW` | -| 89 | [Dynamic Partition Level Bucket Index](./rfc-89/rfc-89.md) | :eyes: `UNDER REVIEW` | -| 90 | Add support for cancellable clustering table service plans | :eyes: `UNDER REVIEW` | -| 91 | Storage-based lock provider using conditional writes | :hammer_and_wrench: `IN PROGRESS` | -| 92 | Support Bitmap Index | :hammer_and_wrench: `IN PROGRESS` | -| 93 | [Pluggable Table Formats in Hudi](./rfc-93/rfc-93.md) | :hammer_and_wrench: `IN PROGRESS` | -| 94 | Hudi Timeline User Interface (UI) | :eyes: `UNDER REVIEW` | -| 95 | Hudi Flink Source Based on FLIP-27 | :eyes: `UNDER REVIEW` | -| 96 | Introduce Unified Bucket Index | :eyes: `UNDER REVIEW` | -| 97 | Deprecate Hudi Payload Class Usage | :eyes: `UNDER REVIEW` | -| 98 | [Spark Datasource V2 Read](./rfc-98/rfc-98.md) | :eyes: `UNDER REVIEW` | -| 99 | [Hudi Type System Redesign](./rfc-99/rfc-99.md) | :eyes: `UNDER REVIEW` | -| 100 | [Unstructured Data Storage in Hudi](./rfc-100/rfc-100.md) | :eyes: `UNDER REVIEW` | -| 101 | [Updates to the HoodieRecordMerger API](./rfc-101/rfc-101.md) | :hammer_and_wrench: `IN PROGRESS` | -| 102 | RLI support for Flink streaming | :eyes: `UNDER REVIEW` | -| 103 | Hudi LSM tree layout | :eyes: `UNDER REVIEW` | \ No newline at end of file +| RFC Number | Title | Status | +| ------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------------------------- | +| 1 | [CSV Source Support for Delta Streamer](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+01+%3A+CSV+Source+Support+for+Delta+Streamer) | :white_check_mark: `COMPLETED` | +| 2 | [ORC Storage in Hudi](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=113708439) | :white_check_mark: `COMPLETED` | +| 3 | [Timeline Service with Incremental File System View Syncing](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=113708965) | :white_check_mark: `COMPLETED` | +| 4 | [Faster Hive incremental pull queries](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=115513622) | :white_check_mark: `COMPLETED` | +| 5 | [HUI (Hudi WebUI)](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=130027233) | :x: `ABANDONED` | +| 6 | [Add indexing support to the log file](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+06+%3A+Add+indexing+support+to+the+log+file) | :x: `ABANDONED` | +| 7 | [Point in time Time-Travel queries on Hudi table](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+07+%3A+Point+in+time+Time-Travel+queries+on+Hudi+table) | :white_check_mark: `COMPLETED` | +| 8 | [Metadata based Record Index](./rfc-8/rfc-8.md) | :white_check_mark: `COMPLETED` | +| 9 | [Hudi Dataset Snapshot Exporter](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+09+%3A+Hudi+Dataset+Snapshot+Exporter) | :white_check_mark: `COMPLETED` | +| 10 | [Restructuring and auto-generation of docs](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+10+%3A+Restructuring+and+auto-generation+of+docs) | :white_check_mark: `COMPLETED` | +| 11 | [Refactor of the configuration framework of hudi project](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+11+%3A+Refactor+of+the+configuration+framework+of+hudi+project) | :x: `ABANDONED` | +| 12 | [Efficient Migration of Large Parquet Tables to Apache Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+12+%3A+Efficient+Migration+of+Large+Parquet+Tables+to+Apache+Hudi) | :white_check_mark: `COMPLETED` | +| 13 | [Integrate Hudi with Flink](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=141724520) | :white_check_mark: `COMPLETED` | +| 14 | [JDBC incremental puller](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+14+%3A+JDBC+incremental+puller) | :white_check_mark: `COMPLETED` | +| 15 | [HUDI File Listing Improvements](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+15%3A+HUDI+File+Listing+Improvements) | :white_check_mark: `COMPLETED` | +| 16 | [Abstraction for HoodieInputFormat and RecordReader](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+16+Abstraction+for+HoodieInputFormat+and+RecordReader) | :white_check_mark: `COMPLETED` | +| 17 | [Abstract common meta sync module support multiple meta service](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+17+Abstract+common+meta+sync+module+support+multiple+meta+service) | :white_check_mark: `COMPLETED` | +| 18 | [Insert Overwrite API](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+18+Insert+Overwrite+API) | :white_check_mark: `COMPLETED` | +| 19 | [Clustering data for freshness and query performance](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance) | :white_check_mark: `COMPLETED` | +| 20 | [handle failed records](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+20+%3A+handle+failed+records) | :arrows_counterclockwise: `ONGOING` | +| 21 | [Allow HoodieRecordKey to be Virtual](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+21+%3A+Allow+HoodieRecordKey+to+be+Virtual) | :white_check_mark: `COMPLETED` | +| 22 | [Snapshot Isolation using Optimistic Concurrency Control for multi-writers](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+22+%3A+Snapshot+Isolation+using+Optimistic+Concurrency+Control+for+multi-writers) | :white_check_mark: `COMPLETED` | +| 23 | [Hudi Observability metrics collection](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+23+%3A+Hudi+Observability+metrics+collection) | :x: `ABANDONED` | +| 24 | [Hoodie Flink Writer Proposal](https://cwiki.apache.org/confluence/display/HUDI/RFC-24%3A+Hoodie+Flink+Writer+Proposal) | :white_check_mark: `COMPLETED` | +| 25 | [Spark SQL Extension For Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+25%3A+Spark+SQL+Extension+For+Hudi) | :white_check_mark: `COMPLETED` | +| 26 | [Optimization For Hudi Table Query](https://cwiki.apache.org/confluence/display/HUDI/RFC-26+Optimization+For+Hudi+Table+Query) | :white_check_mark: `COMPLETED` | +| 27 | [Data skipping index to improve query performance](https://cwiki.apache.org/confluence/display/HUDI/RFC-27+Data+skipping+index+to+improve+query+performance) | :white_check_mark: `COMPLETED` | +| 28 | [Support Z-order curve](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181307144) | :white_check_mark: `COMPLETED` | +| 29 | [Hash Index](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+29%3A+Hash+Index) | :white_check_mark: `COMPLETED` | +| 30 | [Batch operation](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+30%3A+Batch+operation) | :x: `ABANDONED` | +| 31 | [Hive integration Improvement](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+31%3A+Hive+integration+Improvment) | :x: `ABANDONED` | +| 32 | [Kafka Connect Sink for Hudi](https://cwiki.apache.org/confluence/display/HUDI/RFC-32+Kafka+Connect+Sink+for+Hudi) | :arrows_counterclockwise: `ONGOING` | +| 33 | [Hudi supports more comprehensive Schema Evolution](https://cwiki.apache.org/confluence/display/HUDI/RFC+-+33++Hudi+supports+more+comprehensive+Schema+Evolution) | :white_check_mark: `COMPLETED` | +| 34 | [Hudi BigQuery Integration](./rfc-34/rfc-34.md) | :white_check_mark: `COMPLETED` | +| 35 | [Make Flink MOR table writing streaming friendly](https://cwiki.apache.org/confluence/display/HUDI/RFC-35%3A+Make+Flink+MOR+table+writing+streaming+friendly) | :white_check_mark: `COMPLETED` | +| 36 | [HUDI Metastore Server](https://cwiki.apache.org/confluence/display/HUDI/%5BWIP%5D+RFC-36%3A+HUDI+Metastore+Server) | :arrows_counterclockwise: `ONGOING` | +| 37 | [Hudi Metadata based Bloom Index](rfc-37/rfc-37.md) | :white_check_mark: `COMPLETED` | +| 38 | [Spark Datasource V2 Integration](./rfc-38/rfc-38.md) | :white_check_mark: `COMPLETED` | +| 39 | [Incremental source for Debezium](./rfc-39/rfc-39.md) | :white_check_mark: `COMPLETED` | +| 40 | [Connector for Trino](./rfc-40/rfc-40.md) | :white_check_mark: `COMPLETED` | +| 41 | [Snowflake Integration](./rfc-41/rfc-41.md), supported via [Apache XTable (Incubating)](https://xtable.apache.org/) | :x: `ABANDONED` | +| 42 | [Consistent Hashing Index](./rfc-42/rfc-42.md) | :arrows_counterclockwise: `ONGOING` | +| 43 | [Table Management Service](./rfc-43/rfc-43.md) | :x: `ABANDONED` | +| 44 | [Hudi Connector for Presto](./rfc-44/rfc-44.md) | :white_check_mark: `COMPLETED` | +| 45 | [Asynchronous Metadata Indexing](./rfc-45/rfc-45.md) | :white_check_mark: `COMPLETED` | +| 46 | [Optimizing Record Payload Handling](./rfc-46/rfc-46.md) | :white_check_mark: `COMPLETED` | +| 47 | [Add Call Produce Command for Spark SQL](./rfc-47/rfc-47.md) | :white_check_mark: `COMPLETED` | +| 48 | [LogCompaction for MOR tables](./rfc-48/rfc-48.md) | :white_check_mark: `COMPLETED` | +| 49 | [Support sync with DataHub](./rfc-49/rfc-49.md) | :white_check_mark: `COMPLETED` | +| 50 | [Improve Timeline Server](./rfc-50/rfc-50.md) | :x: `ABANDONED` | +| 51 | [Change Data Capture](./rfc-51/rfc-51.md) | :arrows_counterclockwise: `ONGOING` | +| 52 | [Introduce Secondary Index to Improve HUDI Query Performance](./rfc-52/rfc-52.md) | :x: `ABANDONED` | +| 53 | [Use Lock-Free Message Queue Improving Hoodie Writing Efficiency](./rfc-53/rfc-53.md) | :white_check_mark: `COMPLETED` | +| 54 | [New Table APIs and Streamline Hudi Configs](./rfc-54/rfc-54.md) | :x: `ABANDONED` | +| 55 | [Improve Hive/Meta sync class design and hierarchies](./rfc-55/rfc-55.md) | :white_check_mark: `COMPLETED` | +| 56 | [Early Conflict Detection For Multi-Writer](./rfc-56/rfc-56.md) | :white_check_mark: `COMPLETED` | +| 57 | [DeltaStreamer Protobuf Support](./rfc-57/rfc-57.md) | :white_check_mark: `COMPLETED` | +| 58 | [Integrate column stats index with all query engines](./rfc-58/rfc-58.md) | :white_check_mark: `COMPLETED` | +| 59 | [Multiple event_time Fields Latest Verification in a Single Table](./rfc-59/rfc-59.md) | :eyes: `UNDER REVIEW` | +| 60 | [Federated Storage Layer](./rfc-60/rfc-60.md) | :eyes: `UNDER REVIEW` | +| 61 | [Snapshot view management](./rfc-61/rfc-61.md) | :eyes: `UNDER REVIEW` | +| 62 | [Diagnostic Reporter](./rfc-62/rfc-62.md) | :eyes: `UNDER REVIEW` | +| 63 | [Expression Indexes](./rfc-63/rfc-63.md) | :arrows_counterclockwise: `ONGOING` | +| 64 | [New Hudi Table Spec API for Query Integrations](./rfc-64/rfc-64.md) | :eyes: `UNDER REVIEW` | +| 65 | [Partition TTL Management](./rfc-65/rfc-65.md) | :white_check_mark: `COMPLETED` | +| 66 | [Non Blocking Concurrency Control](./rfc-66/rfc-66.md) | :white_check_mark: `COMPLETED` | +| 67 | [Hudi Bundle Standards](./rfc-67/rfc-67.md) | :white_check_mark: `COMPLETED` | +| 68 | [A More Effective HoodieMergeHandler for COW Table with Parquet](./rfc-68/rfc-68.md) | :x: `ABANDONED` | +| 69 | [Hudi 1.x](./rfc-69/rfc-69.md) | :white_check_mark: `COMPLETED` | +| 70 | [Hudi Reverse Streamer](./rfc/rfc-70/rfc-70.md) | :eyes: `UNDER REVIEW` | +| 71 | [Enhance OCC conflict detection](./rfc/rfc-71/rfc-71.md) | :eyes: `UNDER REVIEW` | +| 72 | [Redesign Hudi-Spark Integration](./rfc/rfc-72/rfc-72.md) | :arrows_counterclockwise: `ONGOING` | +| 73 | [Multi-Table Transactions](./rfc-73/rfc-73.md) | :eyes: `UNDER REVIEW` | +| 74 | [`HoodieStorage`: Hudi Storage Abstraction and APIs](./rfc-74/rfc-74.md) | :arrows_counterclockwise: `ONGOING` | +| 75 | [Hudi-Native HFile Reader and Writer](./rfc-75/rfc-75.md) | :white_check_mark: `COMPLETED` | +| 76 | [Auto Record key generation](./rfc-76/rfc-76.md) | :white_check_mark: `COMPLETED` | +| 77 | [Secondary Index](./rfc-77/rfc-77.md) | :white_check_mark: `COMPLETED` | +| 78 | [1.0 Migration](./rfc-78/rfc-78.md) | :hammer_and_wrench: `IN PROGRESS` | +| 79 | [Robust handling of spark task retries and failures](./rfc-79/rfc-79.md) | :x: `ABANDONED` | +| 80 | [Column Groups](./rfc-80/rfc-80.md) | :hammer_and_wrench: `IN PROGRESS` | +| 81 | [Introduce Primary Key Sorted Table](./rfc-81/rfc-81.md) | :eyes: `UNDER REVIEW` | +| 82 | [Concurrent schema evolution detection](./rfc-82/rfc-82.md) | :white_check_mark: `COMPLETED` | +| 83 | [Incremental Table Service](./rfc-83/rfc-83.md) | :white_check_mark: `COMPLETED` | +| 84 | [Optimized SerDe of `DataStream` in Flink operators](./rfc-84/rfc-84.md) | :white_check_mark: `COMPLETED` | +| 85 | [Hudi Issue and Sprint Management in Jira](./rfc-85/rfc-85.md) | :white_check_mark: `COMPLETED` | +| 86 | [DataFrame Implementation of HUDI write path](./rfc-86/rfc-86.md) | :eyes: `UNDER REVIEW` | +| 87 | [Avro elimination for Flink writer](./rfc-87/rfc-87.md) | :hammer_and_wrench: `IN PROGRESS` | +| 88 | [New Schema/DataType/Expression Abstractions](./rfc-88/rfc-88.md) | :eyes: `UNDER REVIEW` | +| 89 | [Dynamic Partition Level Bucket Index](./rfc-89/rfc-89.md) | :eyes: `UNDER REVIEW` | +| 90 | Add support for cancellable clustering table service plans | :eyes: `UNDER REVIEW` | +| 91 | [Storage-based lock provider using conditional writes](./rfc-91/rfc-91.md) | :white_check_mark: `COMPLETED` | +| 92 | Support Bitmap Index | :hammer_and_wrench: `IN PROGRESS` | +| 93 | [Pluggable Table Formats in Hudi](./rfc-93/rfc-93.md) | :hammer_and_wrench: `IN PROGRESS` | +| 94 | Hudi Timeline User Interface (UI) | :eyes: `UNDER REVIEW` | +| 95 | [Hudi Flink Source Based on FLIP-27](./rfc-95/rfc-95.md) | :white_check_mark: `COMPLETED` | +| 96 | Introduce Unified Bucket Index | :eyes: `UNDER REVIEW` | +| 97 | Deprecate Hudi Payload Class Usage | :eyes: `UNDER REVIEW` | +| 98 | [Spark Datasource V2 Read](./rfc-98/rfc-98.md) | :eyes: `UNDER REVIEW` | +| 99 | [Hudi Type System Redesign](./rfc-99/rfc-99.md) | :eyes: `UNDER REVIEW` | +| 100 | [Unstructured Data Storage in Hudi](./rfc-100/rfc-100.md) | :eyes: `UNDER REVIEW` | +| 101 | [Updates to the HoodieRecordMerger API](./rfc-101/rfc-101.md) | :hammer_and_wrench: `IN PROGRESS` | +| 102 | [Spark Batch Vector Search in Apache Hudi](./rfc-102/rfc-102/md) | :white_check_mark: `COMPLETED` | +| 103 | Hudi LSM tree layout | :eyes: `UNDER REVIEW` | +| 104 | [Unify schema evolution on schema-on-read](./rfc-104/rfc-104.md) | :eyes: `UNDER REVIEW` | +| 105 | [Trino Hudi Connector — Shim/Bundle Refactor](./rfc-105/rfc-105.md) | :eyes: `UNDER REVIEW` | +| 106 | [Record Level and Secondary Index Support for Flink Writers](./rfc-106/rfc-106.md) | :white_check_mark: `COMPLETED` | diff --git a/rfc/rfc-106/index-compaction-flow.png b/rfc/rfc-106/index-compaction-flow.png new file mode 100644 index 0000000000000000000000000000000000000000..a81e1d4f1f2392da61da45d8b49927ceb76c5b8f GIT binary patch literal 329243 zcmeFZcT|(<);11^1r$*n8&yXE6=@Z7iiEWhF&KM5L}={pq%dhy+PQMC|&m9l(~a`R`4E57FS;mX}1zyX3zD|MB;9 zyykV|hR7M<`z{f2QCX4g!d-wrQ&G8peZL}lN@Uw#+r>mg5`09&|GAG1@G1QF9QYF+ z^N&xlS7LwfAwhbz?eFhm*M+-U%X2n>4~f95PQfA~hm?hXqStPpToMs67rFM+#XAp0 z7ibb0o(`^>3pugc&$XXR*x{(JhnD{Ws9d5 ziMgSCYz<$mk=`Y<#X}yAWg)h7>%!sbr9K*?7Sy>IT5a`^$1vs=%<>P@4c`%a49l7e z`g(S&hwLKN?);v~|DG0iJBeK+Xi0lM?z^+}&qDyiD{j$>|7-Za&HR5m#{U}r|F+3L zxBRci|Ht_HU#R&{!1n)ljCU}UMG(UlM*}!a8ghueFrvfwv)q-)ox*jSBI-JuT6VqL zAva-&1x>GnES%U1uU0_D02=n(oO|JFIFlV*5JR|Za&4E5NWhC;3WvDEPv5BjM21qy ztBI|!E z45MafEDrl6*nK;=K&(A8nt@B_#_}6Kv<;afJHRt=C5MDQ|sMBpLj=al%cg%@H zk!rNe+LA%(hsYr!kvaLv8g?R&Py&V;GmC98H{6Df-HgKRT_2JTrTM%>jka8Zg<97< zV7^px$g1z!e+ugaA_WSDxPSlhP3U}-$<+VM_W$O(M*O}DKffm+d2CjrQ5Z8@il3`? z38soZN9?-Sos?C&K*zQEdj#k0FiNl8aWI0&XYJf6ZI6BMRDpkJ$X!Io(Y0mc>85~X zx(a7yDe7N{V9hB6tmW``Ok-S2_y0%Y_&03&BZcF~RM;qvTyO>gZOC;c^X16sbi!y| z#1C`#Pv~_Wa14`y=0PcZ2$Y=SKZe(5VSv19v73 z*uXp0U$)@szm-&%3Lg7qI~~saGSa?CBZh4rygJKxog6|d3uuU@mgZ02D#B1QKy>yt z@s{$7mtZjByLkci#@?c+#(4IvqVo5m#&6_WIu1k-7q-!7%fnkIt|a^D8rDi;5B7bl zh7)*JU9}iB>hdK2BQbA%$P^K-6k#+@#a36ujK8^w(mZJwvjrb@e<~?oIR@^G97qV0 zi9i4mq5%Y)O}4Gc@~M8GhXH~Y?7iX?zvrK++{h)xczXDMnu!d`)y3q%2_N@6Wu>w{P3Px~9e5RzR=YzpB6`S7k~KZ#Y>?^~cy-c>#z9k8Y6Q2692$Ds z{&?0Q-T9?wD|QE43R9J-`$udpnXra?R|^$))~;m1dc$DdJI(q5Lc~&W-rd#2OEA0N zLY*=~rOLzGbC5j9Mz=}%`?{-(uzP9i1p4eMsdrt8)*5M+uDm4xX%OZ#n0f6zVgqcp zu1<>bsxYO2hrUsxL^BPEhOt% zC*=embcX}~+-B3bAtEF9Zyf#~xLf{MVE+#K-q~WQtz7!AecXQxJiC2MgBI>yff5&| z>-7!vI&Ae2t%?5^K_varQM%p`s~5b*)B7Vt-xHwGyG6FZUqZVzPRs0zq`C}Q!L~&C zz0w&nZ5uEpgtXY#fY^w4g?^CNs035Xk5!$?vI8{e$! zvSXuZx28)$rs0-Ox%ZlTpE!N5Ps&~u^RegK55?J!k)!h=`zXjy5-*1*|B^mi^ zGJAkDyr)O1VT+Rw*ts1Lt<-1U9@+w({531`%7BJu`6=$R)D|6bpQDZW64XGoW=ru3 zjAHE6{(X@~*EM1T{)f2!Z$C8#0r%8yP<(l-E@}BIVAoqg$3vcM$#_;kpblm=T&V zEgM(}FoG{+;GaLYzuvhcJz^#o!3-hC)ktVTveGGA@^buBz@jn^byDs*VWZ)$a&S3C zL9Qm5j2JwSJ8Ck~y;(q4-q8EOIRL3XO;d(3;hP`AX$H0zlGlUkeYfP^!~J5d0du2D z@m=1B2py|wBGkCUZtt{K*b2ssYqmFm%> z-q}4uU>G=YyU_RBbz|m(cB^VLT$h5-%nIEFtL1Q)`d>~8VCN`-IzqiP}`AkgE83M?Nn%c1n|2jz@SnLmA zq|g7O2K6_L0rUfcAJ)(7)_ESafcVVN`ulO=s1MM_(G*m8Y1@c5+V$zqdN*U#6fjhx z>x2!BmDJEB$_rZ<*3QY9`+!A!66HaMH7iQPIRnY1v7_husHw1ea*SG<^NI-;`Cj;I zw%EF9=@MA+c;qz7o1JV%%YM`2w}t45u>hhU{L0~CmeYXSW>Xq0Fz69FzA;W$ z{QH+}lm$r5>ya=$qT_;Rx{m7B1$0l|%93cH`A+UHWLz9tXT#@qeSg zmP%lLWg@=w@YPv$&s($Vp>wy2ob2~rJeEgD0m{+-VyE${QvdK6@v+ zhb}rqZUvEsZ;N?Ff+wb|XBmC;wT|0)1W%>7x)u$8@*E)XFy8#5Y@|m04u=0BQJaHc z*3yJ3oVLn_7n`7tsFiy5{rmqcLF#-bA;hS~5rB`YuUvvHuD5Ncbz$h?b$Fz{-wM)g9>Y`xpMQQ0)RY~EgP|W}B^rfz)Q31N`YyQ3z*}waK*kn$i0TYb zR?Czbf3=SXWYK>-RmIgSufhp_Uwm5j^Yo>a;xbz`3;O-<@cf*8x5rP@9L#z{{eUXG zQq3#EW3(**I|US78GIt%0G`jDy5W+Q%~L;9=gF&SfF{x4fz_G#CAHHd z`BK@^moM8G*Oq{EXa*@6vdMtI*S3 z);nZD;{?l%Pm6G2fD!*u^Wl@7toK=>)IVn~3SQD@>x^&1QJ<;q_7|pci3F-5jtnKJ z1=N2RJ{#dc3}*%B&-c9DTus-UGYuA~PWLa=b*$<|{xApRfW*x3Qc5QEmz=)V_qi)? zqvzQq24>|lfFs9NTOv(I;50urfe|s4nDt#387>iPH3_Qa~;be;ZF)1B^U z!3u9Gb9*_|2&i;7tHR0vDJ02Yr9+43E75>rlpV}!s4opqx6F(Bs*6#KYm3BVOkC3u z^Fr0|bP#ruPD1jyj8OW6AGbiw!}0*5?k>06PCIkzwkV_0+Say^ zj9X$FwbO0edMLGTiXjsA10Gg=*ocs?EM{agDeEwKeS*JmOQQGA{s^^SpCEEPNfSv9 zU**2=79n?|0;`kiF1>1KjU?BpZy_AAEP-%H94e7&>o7czyQcgT4yf&%K>m;?A}hM? z#dTgW<9uZig4d;;TgX865&>tw_`B-amsK!2Nx)j|vqhVVfyT;Ioj95uLpgVzB$~@T zIuPaOes&8&aMC3JOW!QLo|O?5v%F%y#`yToNbS2pVD4V_m)8tNJm&)5Jp7wjxb=fJ zz%hBzsGe^rb~ax(p&!1<9@xT(aFQ9{pg1=JzKXfK#r7`4`BLF5@(ql+Ar{5DTVVqn zP~i<)Pm73&j*8uJ^^r(S=)U1f4@&^6XISb$nMgDO^r^u-(2Xs4)GqmF zfJ?nEGKd$6fu26LfmhcgUOez`68sM#Y?=4SGDk7)@a6Pf`!3XNr?loWztrN4IA4O3 znS19IwlMckC4jkq>Uy*JxVihpb_$9P^qM9r3*~H?d$urbAR&EzxGPkX+ILs$kpD`Y z#U4n0|H6+NTXq9io=D7z5s6Vh^ZXT9F*LIZ!C|T!UT@w4V#y&A*mth)(PLeYRru@!0les%nmqkEO1ouw?ap9PYFFS~&GbMyu-Ri;xky#-p>1`JH}7Z8wLmnJ-Nq7Pc! zlKZ{~#5@3wdTcFt3BV-{fL`hQYE&V7GJynFffn5;s<)NUw+3#!;D#!l>XblgAUPnRCK3I#7 z8g%kjHz88T=TmjIg3aXsHiz8@``v@AH65<=HeEr{)@RVBg=~FGKKTT_7=lvR4j!;=F~ny1&qWZUntRFU6{cNJpKV8{m|azM@!W42k`HEk~?CzKXDA~ zwHYyp%N>BQT8*c*Xq3-PmL1&gi9`MDY)|1^E@SqN9`{D|Eb0beSVC@?U zS*ku%+Z9R~Xs)V1swhF+_49~!Ju?)W6+hZLt+~;p>Hix_)Ndo*CFq(uJQN!_woNG` zeza=kX!pY_9O=y^>Ar|VdRKtEkn5|n|1Jjt-C+lJYWEr&PiQD3_T@19kxNgM7G(Bb z-^V=TP5hTorrN`mC(MmD$!|w9wN*JMDwUhNdfY`25Ab-p;!cMTUT()QST5Stzj%C zZi6d&%8|Y$!mJS?O%kfB?L+m`x`kBV@x#&I4ijPgK^P+-IE&h^f|WaVbDU6#684*G zMZNn^e&6xj=Xd*OjZue4>>8#WntMlo&xH6eX{F~6@d4HT?74_NK7~~_`>~eegEwGI%>6LfFIYhepn*9;E zHABq}d`#iM`7Mm0UDAIYh}=M)+2+h+#Y5zIRTjeHgQL2993)L$xl8zA>Hk<7%4U zkZCXLj{~;ZZe2otp@l_r!os3AInnfn35CVkb8p5uNIK^R2R!T;6H5nL=>x)6PsHm! zTQg*x|CT!C;V~!B_Rb_M#+M1Ga0nI#LhH6kb9%>o*lGl)ub9yL6c7uQm^pqYeWSBC zk0AbTdG@_A_k8kVhsQA(9G6w*yBLUdiCUl9nY|nkMx=^T_l7nL9WmbmreD{SSV#Oa zrP0twh-?Q$MIwz+2OxJu0wvDEG^(NXxx1rSpk>v)f~NHPNo7~XzD#ql*uz?ZHX3Bg zsqMZIHt@#AuX&&xUc+pS-e~n>Rn#kX9NBBtn$&o$%apeuD21*uc7-&b-agha0;L=Pv7#&adMnJ|28!K{y+dgA9x(JZ4c<9qqm-V$9o5p%^fAgFZgq0@Nr zKGWqF$svbbLvC>Z3ahmGeCl|`OB=&tBf;8)x1v6N!b6GY$U${S7NDI-c4&=I$ zIsNDreNEQ=wi-4DxiM98z^yU}&HGYFTtqHXX#zH3(OXguqXVNl^K`IfPCUJcJ-9C7 zE$Su<2UM8#4<36MEy*tTm#{f@Zn%yq0%!tC`@NK$dX4_e+Ar z3ju>JfYQ;c^P(oREbn>d?=0mvorjOWtX8b-{O?hY;%G+#nV}-Xg&>{I!L4wS0 zHrucW`|_ZbH89F54+}M8dzcLlOk5&!5ack>eGObJzoUDjqh7_EiJ;B94e@)F__sK0 z0vTD|6+V+QH1#chrEQ2TU@@_qTr82i(N12VH1-mrc?3>!p7roqoe|LbI(l=Rc~=_Y zSe8Umv?jYPJw9x0aRIfiAxdaKaBQg2Jx|^GyG!(G74dOTzIMxIqU~uPwXJQd+{fD? zf}m)#lOo~m;L#uuV{@Z>b0KO)KN=SuJD0riG5J~i;M^l`@>@4!$nt0Ay!b+@UCo73JiJjuWzKjcKj`xp{GN=<7MOV8G9qiLl6tYPOKDLAckR zDm5J+({6qNVIu~ZG18G^=Qr>R9iYAztmDTEN-G(yd-So+-3$IGWsKC1H3wzHtsxW= zf!CUb<}Y}M(OIr^T|9;kKgGX(L0dVd3^=tMhRd%?Zzv_Q%3!obNPD`)*KX%7yU?NF zJpeH8Bm&_Yy&!X&2tc<>;Vi@qNLb-Ohbtf~8S&+fw1k((B`*MQ`30QWKD1y}<6?VC z?5NDor96(1(9+$99k0DQD_VPIo%Hadg<@}a6I)0*u3b65Gd8~|ZE8z}dx5xYwe7Cl;AvxtRnHv+@xrLN!l)8Nr=MG2O>$!C#5wSg^Cg%8v(14N zq!86OpU1{X;q%Gi_xw69aelRp{JHZz!Gh5(y3OV1=7^-@oIA`~Lj9CF4dB;)khkpx ze_#m^pS+L^!q;%_do5cnF#zL-n6-n6nw+1QeOa^xFAU|z%`&M5WtTJrWAddfbAIXt z4wa~>%s|fBp`BUx>1_Gp;C!_*!J%%G^eC0{oN0Y>Es;Bb&GqfOJ@YBvWhHt&J^CxK z;&5aWC@jJ(TAt$%icv4V$^qe8h~C|~&MvG>%Za2^AxRm6XNCorsl5ps1pix~!lw@= zGja+&l)gi}{;`d7cRG&e9v7`SBk_Jp`4E9>Mtc^U3k`3@FUV znr4T zA1ierQ(h{|gMcgB5Ynpl^PSq)G0ePYd6c4~5`HFFMni6ebNRr_+|{@E>*s!cF+Y2-2#7S<-2t=#xn+dYgkQX1E-3yL;u}H#vJ4@ts9l9t;||B(KX%_ zWHmLgT5^ z{8DcQ`>gk&cV>bKGsOVrP{Q#y;tv#g!>#2dS&tmW?n>|{dy3X88m>P!Ce#g-`G&66 z*!O&QjCyqs}d#71KXcJkwnVT^YU( ze)_iQI8Dx^WBA5|k{Pm;85nJnR)e3pRgC&palI>RWWSD|OBcWCFsTC3EH`?S$a{1j zq3LbOz3urB;>81%2c=SINa3-^s@b1xqk`(lQeFjRWFC<`@2zWX1_8Kdl|>nm$K5q) z(PBK>E@p``-##qUCA&MThUXWL*ZWwrzuBkx?#{lbBuKKzXh|?+b-dhYJq46?sBE&Q ztvPDX5LJ)zfN*!Ia=us58}%Q{)QbIL&1Oqx!@;Gk-U=)fi0d8{^Q1v0>4dpEZkNB~ z62qxGVsQ4XL(KXJnmW-;(~#7Ty%`l( zm{}>lMD`Hdx!bQiOdw^&By)3pIEk)>@J)SWpZ*Pp1GoK}PmH{M<# zQuv-Q{^s$J#~zOvLSp0i4@LUhpi90-+}laUcN!V{S$IB^cL*pD@hTkR? zOdVX#oX~lBKQVk^xShPT9zZ|Ym=m;~=%7e6E=?Dd&9x&&X*UF|gQ2R~7fS0QOpe!H z`;3^W!P3;D#^wYT)cGEbnYw$%|4l$1v^chyO5wy0c=1f%;yV#0{YR@>1&*g?vRG~p z1`+M7H4*6O1;)SS8elyerTfaVjv#NNJ{#Xx)1(!;Gi)7=_IXmc-&G9cYrkt-b z3!h58n}goS(TWg%9zYK(yHpps%sUTd+8@mV0ceK+htYYX1nf$zs&O7~TuCqi0@AK<3%;V#21GPfjd`898HQJLES$(jCR_IM(C`;Zc}w zoH_@P>(ql&{7k)vADV;}Wx3a?Y;c>0zOZm1V{?>toT(?c*)$8}JGk$Yj?dJMOTjKk z6)G29Py>cE8e}`fr?K@po?PVRi zS9+A2Nc^_P$_PVqG%Jth&)Z608=F2Z9sV)IWVeLp-u5?2{8^9(AO_rRyaxoSF*#IB z?)3A`)=4ttpgFib;jo8N)H7BMvhTUH(%xOaCMu+uJ6`Y1##=x4m$?Z(avtncGk=?N zLJ1Y$XCF6MVzW}W;{0-R)WuKcXW~FvR@sTCU&FlA8m4tszm5Woq4$!DYl7*vd=e;l zZ#agrjp#l4j^lN8DiWg%B(IJ1LZ_TpP^a3vbbH8`hIn=Hza-^9eiqnN{6yjSvvTv; zQ#CX+Yon*9UfNIKvCN4k&^Ix^%U6|@4()XKI`o?38v19lYj3Ek0>n=ZiyMvJ+X}Yq zq>vOB9|XCkDOj;u|J{v?X;#0vs5<98!RA{)<=8$ZQErO(@OMMh?h9!KoU>0Uri=Xk z*X+LZ`t+l`=4?lTCyH|f`U>`N<8|=az9_0vyZA1DjTk9FBie(CV-x~)HT4F6S%}tf zI=75Y(e&0U&J5gO52&9Kh%3z3WF!5&QM}bcGuHv$P)8G6jl8TzYX=lg5YjBvFvh1JOvu3Zn865rjiQ*m18*FXD=Flb=f>BJ!`c1-9<>bT8Yx_=KY&|2< zp`4vo;Ubm;y~&8;5Fs2n@;5nG#e__R7Ja9%;jQ$$@%rSqqv^c?Mp<_UG5oWIliwOa zGNo3IxSWI7$4Ma71B)6dXKq;1xZ3ajk_2XofJDHv&$+&JrDa- zs?gTmzP_%nW1`fy-bc3JI~npn@)Fn$pYzy0FQ~%;mzP|N*Oe=p5oi$l{ba+HrZ68Y zw9vI{Jz;)C51w>6T0V8+2cXS{Zj9eaml-SB2X0>}vNS0n0=mfsTFOZEnYFZ|L!@~k z2}4i+d^1&is_VkUGt=iO=3Vd8;4Or|DvHc&`-*ZZOCN_9hY+c>0PZgXbY25}z8r8s z4l=wiYqWne0hjZBK2I8XU&+3tHCL!vFoLXkzw}n4L?DWXGdCYX0A+**0c!d%)Lo3a z#9o!g6sL*X%cpw8^K1;izKuej;HJh?AB_nYrVCtHQzu?l{2@2G6z|Lk0K}CW>3^|i z=|4QT8lbne*19Fj=vB);#5>yNq^eo1bi60n%ctGM759hoaN5y+AZfv@ioTO1PxZVG zPw^4USts~kGlE__Rfd#@$wju;E<28A)-LIw*R2uv$;Nt5?!w=8Ym)!$NkFAC5u_Q! zAau{pkft4C^QS(O)=>lgyyn#;Dh+R>qVxNW(Nk~lXqC(@v@K+*=6+_ZRqE{B6D(!> z+%j5aSK9BN!D2t-u3rj}_@KOtU3={+)ZV<@#FTYTjrv*(EO*mNB;Zzt`b^Oy=d{)) zf7C?{kL%`TVi5)fQcEB?gJ`cfxhxY6YL(eUS5O>tZa6#o_FRN@dONO*hJSUi>+g2~ z3}vn;^KTsO?+#T!oJc8Q6%@NAu!8)N!@;MbMN@-M|6AJCducxE4Jt;bOS8QH^ev>A zWDJRSE$6k=-yW}u&e@>MSAU{M4@M}we9+G$`}2mC)5BoowG3uKA>&9Hq;*Oc_bLBv zXSEp&UJFkn#4gZi2UxYM5XV_prQ387tT z7=Z2uMnCRq5vP|#&V)sIn>9NP9;gWa|RN!TU>Yp>Oi(yjJiaDHe6k&rz%1_lb=(B#Y-RLp=VkpYu2GJk~|#W;s*2v*gA-u z3liG*-~}vFKqDl9x{KAo--{5*Ay|V^z?w8%+**=f`p`TcCra!WH zq(xp`>;=2gZ%JX+N})jX@fwgt+En!FTh&n1MW$ucq-B*MVRH1Nf6!^ho-n8GJpsB0 zAcIuP94~(C0h^Cj5E4+D4*tB!tw=EO2d!ag;XEomN;a~6&#w}s{LCns0Ol1*U2zox1Xe_WSZIG#*E!u{0K)=J_suE=qS!-cQpS z(Pf);yefk?26-V6)O!UrDRYy@<&lEk z^5g*@j(08pSveA&iS%e4LbQ#OB*_DQNodd^!SWm5`5$@le4%E8D=| zSLn)VbTm3Z-3`cqru{FS`kCR~2=?*pSdtsiKDBvb7)pr^3qVk*Yz*+!UQb7*P=+TO z->z)?OG=lRdK;w(%QTc2%83@N^EE##D^*HIEj@9)A7lDo(8Tizk1?&1H7f$kc8x806l_9 z-sEhW(o$F*XVQKb3`Tzj%GSP@Hx8Xsm=}tiw)z7dcbfx8ae&VDbq}(*{uUnvzbCbO zr%`)CMA}D>_qZ?Jn@8W<`AH**WU^Jw*rm9>uIE$*) zDh6pB&f^+D#fMS_)3j>a(IVFovP`#U-=oma4%hXwe_g_Rug8= zV%wf*RGgNlNy}&b$=xwe_7!caOz#|yjNTLdyi7$ivUz?cq4>Wus%NA&Fs^h@t6Wq7)!ASfKBi6V~H6 zm(g$g9OjkaD(xo<9URXx-5*u}E(6DBh?SMN%-?}6?Uw-R_IwhT7D)6lkIkR)p{Xy>)PyCjN&9wMaLvy+c| zDD7A%vEez}LpP~>kM`yYmj;KDx^QgDxy4bZkm$FN)6kDF&!_)>crVNKm1%o1M`6A8 zK~Hx=`?FdKt?1a$Ipj@mzW7{3!3T@l0-qUoZ*ln~6^_q!8j&H0)VLQvZ~w=+m}WJh z{DK9l9OROE@>%FMlcT-fX1tLoz0!^!K%yn%HzBL94~LRht;qdmdO8`M~UodWpU%5}oYfozok+fd~)^d>Mro3mzk5>Wgq{MuZ zg!0iEs=WR}vd=8Z1oUDBZ>?YB1;G;Aq+@iWea{!am!G@2THN3U= zCfi^2yLGCh$qsOt>I;W#+S!u7fqua>xi)hTPRs82qh&4`+FpCR1g0B5zV6`5*mP#; zv{PO;Iw_>0rfyT5#_jR)Yr3I*@V3AXHLwH8-mKS4edq`!!pSC!L2_dG@hZ@sCjKXM zng;wIHU}qN7?i8{l045kY&tNVSDz;bry2e$t6(1}LaWUmUeWD)Uy9bTHDUEXxzyJE z{Lkvw1> zxYA6 zC|jqa-U(B4ukN-#BUUlcRiLlu#aT))tEcsA(t*AfV;d|y$6=6Yw$i2RM+dz4ZdXSI z{3-a$>*$aGM3M{7STSU<^gfhUIpwlhx%g73^79)^H3Nt7VSpThv3gjp=#yz`GIiSa zt3_(9Y4{yxg-hW3?XHU3h_n04nngM)VtM2doYuqCZ3k{zeLU@Xwg{A0toO=Z6axn*b+c0>8_G%<(r^&p z7bo>Ih*pnhL`R)NP+qsj+w(oeyrH~eGrj3b(pxOf>kWnrbv%o_&X==Il8bLIAy2)S zXumgHC+} zw0xW$4(%sp>Z-REbnMOnDcQkpjhzkc_3Fa;)8uhQQ7j@C72M$)efC`%lriBH+?7&^ zVpbs?&1P(i&FO6v0oF6c6!gmj`z3YDfroR3cdXVMOQQ5|QkR`^nrr68pvu2jK$IHh=@P!b3$o!95K|JV`qz_v&mAHFd2NfljR9H>VfmS5IHekFCpmK3od%mo}Me&fvy{ZCc!qOF6R(H1Bb-h%&U}`GJbIJ@=t} zNjp#$)UqUzlFmx;GFFo-U!x47IYnKDMRVQ_3_FGr$(}DxY-#d>^5OP)fAqm9j2>0H zV9|qrjg|u62#o}j@t`arU&ov zQJWLt<=B1}DqPDV?W_0UgnGL*UAQ{}!bEbz``XPEV9T zDv}c`NA*1m@a&oPj-tRs@WQP3Xvy=w;cpDQqtC30Si*Si`R2> za#jb_$-mZe#hf@%aliL*gxQai@O$OSI7O$J$mqA#iyx?p5~Ld0Gzt04#sVpl0;NLX zh7hYP8_XX&gs)B>%q!k_G)%sh8Jl>;BH#(P&+=YU-0$bs^s4)@wwJEs=J2coNxs}v z*Eji(Jys~C+xpUt1;K7w>Y-FqP06P?5(Bg?v5w_x&GBZD8}vjkc$)K6nRvH=6XO41&v3KUe1NkmTxRkFUv8d6i~4-V$mLIaiQ! z4yD^N=NC6>pG1N6r#p8WgZPq`omtSc-bIh@q$PNK#N+SKd=YMDB>`QKb`Mr7I}vJSR%PNj zd&i@TWN2yh#0f8LmGh>hs?@H=%pdCtL<)*guQp>T) z=4sQNVsymmr*rlqKKbkor`CA>eT!S6k#6%F2fa#y-wFb=L1Dw)8E^YN&QbMzsn-)z zvCz-Hw`lUWX3EAu+wl}V>Qps)WcGGE(siV6(+?r26rl?;dAt_@_f&^gx$vPprVI|6)*3W(L23q16{?qQ*62$d#3@w+K0U zM#)u%MHM~>x4&UPR4n20t0@g>y@jZBHG<#$zS2w;? zGw8aO8sGZp{7_piAA*U>@z~7WmOi)9?(#&EmY|mXoETQ2lBtfS$aR965 zok(uUHuZ!zTrL=Js16c6>kADh9Km?LvvaulPI>p|G6 z2T;!#;CXRBF7Zj`HyUTzp7du6{wgqwFP`&-sHb)2ChVElc3S+-8&6;(FOJz>zhvAV z8++!VuG-t`UcfO+vT81~?DUo2zcCD0$XF#v$=Bo+>FojCEvVFsXE?|)zNs}htvk<@ zHbTTeD`K}n(IbzXS*qSy0|^TlZBGMFe}6TEJqe$@R%)8aLilemSVemC8@Mj;#799& z`1nUd@3VG?Xb50Dht~^j1|S6i2I^eJt63hL(P7=EABHo2&iL%o`3OiyobL9eHjw@$ z_WQpwkNXnz8-Qu+6Y-ikRXAcdhcWAY#Ojr*PPy_)U3<#D-g*%ygn!)gGi&`!ISmKe z<>O^~4^wX1wK=Yp%?${u!Bik7;r)hda~!pr2h2 zpaJu2HwMf<)#~!piG{;T6%AB2a{;{m%?ctDWL>uW)tQdGK$5nj0~mu`k^W(-<6=J@MtF`HG}B3*L6^$n&pC$McI&F z4NAM91M$bKRIK74opihV%I z9ZH_Fpp;rxcKJrnp7%;wse$5;*HA{;s?^&p%m!J@Iq&Ir%GR8X_I6^c3CqXT)TNVC ziP~rX5cxMU%9olWKc-A1D4p;0)iFz$Iy}he{`uf!aRqq$3$L1<_~JtYW%vUsnfhvu zU7+^`@Pe9Z2){(ozfOI0JEyP=LJe{O>(2wt^U++hP|EEYYe<06QL8z8D?SwG^1i%& zI^(j9k7lSHMZ47BiLZ;q%Jii;A&Z?xock*4rh0OTEn0iv`OM)xz5`ck&=ZFZJfp#B zp(Vl9t$WikW+SAcDDH~~8j=G88P0{vACcPi4hcx-YB%x;ZA{aTr+NQaCK_`>`KB56 zEZ5p{iE-8S_>3}8dxneMykUI~o4Vole#Uquc7b|EIG0n`|9Z$t@9HsFlCUYC@4UF} zoW4Om^_z!xDK~KX(G#^mY5mf z=cIWc#-QJJ=Iv;ImFkjoATp=)CR12YTdMD_+j+0!ax_)=JkcC?1FGETPBO2IZcC z^8Kw&oxS+>ZXjmPVUtqPTR#RG=>FCoJ3?_9p}e}2Lw3W^E~raZc|L;*a}Ru5Eu~Cr zD8q-k$=(1;WO)1x){y1QFRt9fyThUkt?D2%yzj6)GD(^xboPl>A)mLjocGf2E;fPe zvD43QAlTzO1cn>O62_(y6ApVMO1@>_2*d4vD-Q#zT6?xsn zuqq$fJ~s474v-H(548l+15Nk}X(cZi9{-1_>kLRj@7{tdt+bn&2*KP`hPhBf+$;A&#l65k_uluu|MwGLln+0C=Q+=L z&Nx?!MKd-^fAw z?SI?V^&ipfV9#5nE=bzv+E8|NasC6QFRWVn`e?E9Fv`kn<6(QkA)#e_NE>`lvmCE> zbc;B%tgpL$cZ0Td{=Vk|>{^*x_ZBU$*#;AA*0eidn;jRtwR{jAN)1By`a2VyMz=tH z_TSz9W0TcK0rIb%vD$<5iYjD^ZeJJ>WLV#V*;-!Qh&}U8FXWU%rs(m~T{!v~+_qsp z3hw27F-gPLXz+4wxVv%AJnV1ca_<7oFQ7>k<1(;6d5(vk@DA%ZgCn)T%2@X1!KTg) zZ3d}9g>k(7^V$7qwx?+@+fPC%IAPIo(mI%i?#_X57>`kFae{@B)zAA<+C6FFQ2oME zqeUyYOymilP2!nv_A#w*T^P$&)YqW64eb0hao2w1*_TR^|I@M60cuZQ&wopHJC&Gy zr@QykuHUN^tezT*Hn+zqJQ%*=NRzbUG%_QdNRV0Oz0&SesU=zNrO~NDZM9&xd23`KA`p3m zGkD=`;)v{uLvbPjHT!Y6Og>$=_4mC6Ju`~!m&)Dm#vzHKGK+B~eK-D*SN!B-B04kz z+IBK7oL?@otxL=F^SyS}K39|rvtlN3+$WJ?@!)%!{h4X7Vc-r#L1e1`+x=7*CHZX9bk+q74VKKTh!+M`dVVv)3OJ?5_a4Ta@8ToKG)>UO~%Q zsz*>mkOQXGKd5^-Z<>=+Q~N`^c}}0!hSPjlI}{a!I< z`sZgGK6OWn+cA!eLi3i|jQ-1%%=6fUtl)MI9e8(#3K^tOpnIhU?(o8RMS76~Zo&t# zs0lBO%Rzc}wDkJu!!PYh7e~)TR;0!v?APzz%^8d=c(d}KqOdHMY`3Dkd~F_<@JnW> ztVx%6h2wiBTHHT&`LStO(t+)^By7#a!Vs#=`K1TG{Y9Ecao}i_adQeu9r@t&Ggu(_ z>a`0DQ@E0&?PC{sn2&AP{j%stCx%QUHp!9~O_}#_+^E~^IP5~aQhZ%KpkO#f>zp;h z=ZQ%$``&B`X$x+8pT|U>2)EM?rtJ3Q$_GCy*tHs*u_rQ$*mg3dIqQV05F$L)mzWiV zj1KH*tEs1tlF+B-&nm9(MWpp|gwu=-md(2wJ^wNl!X|ygcMu0B03TNQd#Z<++1{vJCC14i23RRY z!D;BQuaCr8BnRb4n)UJjxW2|<6_U72Z$G82zN7u+g8`Lna6A4cVw)9GX8n15!R~>N z?Q^+U59rf3pFkl|B#8H5rdWu0Pe+7AHHLHVp8l!Wt$Bo15sZ=HRJh^UFc*wM zvI)W1y-x{J2?SDQZ+(KyCUT?>(v)(?xkC>*;A4_gxfJ`0=+C*_10Aamj`a7WRyZfT z+EwFF58tq13aOW5&$2f;#Qw+e+1oGZSix0)U|jR z?(r4O%U@jku40En z%?*L#m@N0e{bqSwH|BKXbjD_rX?aJH9KyJH8;5-Ib_$O}?@rz}`lQmEC)0ZCyP&2s z{3m+B0u=Ld4&PMfl=#j{DA7BohT^n!Ju+c4lf*L3acW95FLz8`*r(qg{C@fN@GJQV z_Nu1{uDQ(X+d{W=^3+h%)H)qkRwj&tNF~=#Mii{$)RzhtEudEN2s5Md%eoyk@>lB9 zj}EF7t!wt`vJO@zrNw)&%wD2}ehF=K#teG@>Bvu?!Q${Q>jmOUcQW)XAU7QJ@_!6z8C8|*V-h{DEz>hz!~g3e6Si|rx=!!WHC?{G^sh`XQE%&Z3tg= zXY(e@5DwfgPX}anV>hAe?6=3P63iq7jHfh%|C+^FTW0lz&6Y?7?W*sTgx??5gKQYN zqeY!=52yGXAd=N2)pb>S;V%8^bFjB@k!=cltryNV;zqbvX@`jeM8ENEJ3K zIDN4#{aFF!5xRM>yr-Bl)FZ=L_QWN4=L|$HPq!0s)m41-XexRk9kCN|g-wrqhu^xb z3#&ukcK+AN^ub5q2BB$o#)yBgc;asrlPt$Eyc_foGY#dWy-vYNu>#H_UF_b+K^%J? zw|S+qYzFGMS_qp~u~KS#vtn}fs=!)XW}RYuVf59@0^a9uy?xN7x=714xL5FJ$*v_t zWXyZPqw29yFk9;cy$3sHL9!WMzi{T=EBS_S^OYhQnkc{LNz9+KrT$vI>4P7gs89mj z`QdD7nx0%tQAKaZkfy-p__xIFcBXFV)nMA?rV>teYa$x?-(X!ft2vDXI_TDKG}o7= zX({ce=;2Mb8A{2Wb_?P(&b8n1Sxy!Y+D(4oT7Huk=dhQJyPY#JoV=-ctA}L2F|P@z z;%N#@$QpjcMnKGULwDb}?;aa^ZZkj4##PUmu;Y2zEv1wbX+zq~TM`U4cXNcogP*Y^ zf1>V@o3`87rG=={%gbu`houWio9fT*WtGrUyK&1iIvY)uqI|7YwpA9H%QL%Ew?Tc) zfd?k3##xDoeGv+GCQ}=i1ib0ZnFu@RE zUL&e7|HL_2MjlfUtn^3lT(9k8NU#&7-u?Io2_-o4azc1;P?W@WtELxs__xhHRoVyY zoIG~D=&S48%*S)4Z#(S`Xv_9juy|D)N>Vi4&r?9Z5Goese-Ibu{sqJHg>z$z3*vMO z8U!l7gKp^h>k#@|x|E;IaD3)CjPB@`BUy2U%iFMin_G!zrha8AD^4DDI-cYW6CUJg}?Co z?v?-D3L)Q7NZQiOcs_NFvt#=Rn414FtGXhdd$E9fDhzD?SBzDEbsCD2m=i=%K{ARQ5X`8M9E3Ib5StWn8kkIgr{K=Q7xSawz zZ=I572GkWT-y@&vR6p{m0J~^WtZ=19GHrq*I^Ol^=;x0rpS5~ybedL{fuH}pzgctt zn2F3lk`=wAH?Bn3!c6OfKea11xAdzhPBfy$hZi@lfr)N+Nz1cochR19D5U#4sZaai z?0Y`p=BFYGD+tAO&F5yo2bzNpz(>J3wh0b94uz*fvsd-*k8+k$b1w@V?+Nx_=njlD z6j}6HepPCP)1*1X>-Q?Mnj1E&J@?`17u|2w&<{xd&X^EW&lKMzMzFG5_RolNKl>^n z@5=ceJR>g5rm_yFZsioYRH_~YH-oj0*fmuY($Q=^+o#jS$5EP*&Xn}X_f@#PB!>3r zhnmrxrJ!Ph!9m zh_2Sb{kMNUI)3-|HY`Kc@$nx52a`=-@F+{F`bkSn9__&mtY8rFS4BnFv*_bVE!#?M z8fA_f7z;JC6@@c6HOykbUh;(ctKQl->~m2mZ}NJ|R2}0DH^x&sZP*0?-xgL52%(`3 zB+ohL(a?T?{`Lu;sz}GKp4i24ju+gq+Ia*n)d+Rk&&G_MOH3k0qDwL~z6cxs;9OmhugU#s8Vzpin z-Py)@CL4Ckx4mjQilUx7qn)d}W$%mVu*37#bv2a#Ng5zvRYjp1fNLcJd(4K%f4X)3 zjJ>3B$TV0c?5*S^0&Q-)F`>O9d3G5s(m?R7C*FJ{_(yZd({?u~oc!coR3b4)XqTtj<}-ZDn}kdjENp_5w6LXasG@hbgV?q?ezE92s5rp$>ei^7OqDan+4788_+RiYy zHD*9Q!^=acyGC@NySu<<&Ig^-u}uxR8hg!(Yn;|Ihv;1^oh#7&VffxhQRpG-r`^jd z&}thrZ0(YKNR`kV_W;Kj+dNS(UWdOvwv$f$*efwP2={V6FJVbwFHYKr#q?jg0@>x; z#|xj(dqOK!*3UEN4Flf6)5f*Ote%E)UFFL6seDq>$D{b^t<}>iA>qo7n3pIMlrFq- zoc?fn$>$OH4EVGREFK!+Gj1Q0D_@SD?2xxKmO=D_?7hv>z;|Q%ZO2q1D3{!^;xAq> zgvNu+kGM7p>>f2F3=9hiX*>gQVP`YkVO6*>dM1JGy~>}Q5y|whsT!8--iJz)V8P;M zO0uc7y&z^)kF?p;k<`OFa_04O4HU5hjmsU=IUUu%y982uPF3{E_uis1NaZYDv8}RO zY1x!Tv}x3=3`OttI54$&>gZ;6h< z_)pm6onJ8-%K^Ir78SQ`Ao!Pvrxje9(Vx_1d(B*X_2#iwOY9gB>f zu}DZ@+U6;4b_KIFra$U)m%3S4X$2n2OOIMeMU00UsF@kjA1p%5c5>EKSBLw7r`5Np zP#~_9{VbdPY;;ocz-QU-dUf~Ka$Tn|ZfUm^7bE~;}bxb_zsLrejw_=N(=v}?q5fH!Pagn^)<3CbeEVzqb z{N~3y*4pA1O|=HqP^3ScL#i<}ZZ}MJXrW~LEDlhvG4<>BAZIS9S?@7b26baG&J>LJ zT$;^hV7-4o^)01*LQUSSwtM}>4L2@2^Vjx(&ysB_Mi)o2xN&s*f^DaBU?1IQ!?>ug zJjhbh`Q9gDVkR95(w|k?9DjFV6t>&?%CH9Y+9fiZ=wOkCGGg`fd00zSjBb?ZWmt8( zfNQ&*$6vtd#ZA*V7@saP}7tj8#>_K>E55gaw z0GJSe?D%%BS0)vxfpe*669;`+|J-)IG!(Tf=D zDbyVpzB~JUmX58Vt+G#97aboLnkwR`EZ%SQs=X@Hq@vCP8q_JH5ZvQ)5ymVG7%$#w zz9)9rOjF!vOc73u_xFtp+@`C+gr(y^IB7(GPj6G#L=|QF2gfUgwb6KKVh}nrh_A4r zX(smd%bna?tRU>VW18`soY9r42!o~`yCqtQkCpGi^T|!!ps_>4z739;n#dP(^$tH@ z9gLnzREIw^uG<1c^s2mH78=3*4SoMXz*8T8Zd-2rweU|*mbaMRd?Sb6?TUpQu#l{2 zjf~}zlkQ69EIe`z^$3V9!yxC5Zx;a#(-P*a;22?WrrJ*vZ{dvLm8KE)j)r|SY2#)ba+5=SIBGV1C* z#P%lH(SF|pqI z$+LZHc#6M=E-Do#y9c*whs}d5pFzjWz)E|tW7bbofV}6+Yt4`7GeTJTU-p_2Ppd8$ z!pEEOCL|1NuWSAd_SDZ(5}-Yhs+HNj6Bn}Ju6hZgMO*}#-#7d{)k56MXpB-{y!j{2pdp&+c*8H zz(iGWTdp$;=jMwYnT$52~$}v3$ikk7kNG;#B(O zEt1T^BmxMJWd~jyd(#UbE&AFsmB@+i%-8I2hLZy|+=akX&n0SKWJF_phJ(9ZSDQ%0s>CqHaURl(9J#mr@ zT61JEHrMR$P^&7vSuPh|uKzbVZqJIYY}wcF)M_qnc6FE}_HoWbrVh9iOCBwX8QV2% zyJ=+9vGANj$8hR{;Y1EX5in%Ik6}kYG%Lb=?*q)i8uG`Qb1ua|C6BlUHMxfWwj!U` z61Yok?c+PHw}w8IEgV*{)*&SO!NzjtVnS)-?WZS?vC)}q$_WVe_U!# z974wW#Ax*^7;j$bV2=6z^0+;593q4%Z#JChbfwGBP!sggx*r2Bt`UGP)PnPAwN$6c z4Sxyj(v$x2EUf`^^hDe9hi!;>bzBbAy=7aMs#Rsaw;(!4ef08%2DLt#V$^!UeI)d# znNe?*f!W9+aT~fYt`oT@>3qEDd?ZM-x(KSFPaOi)nD+$D9lUsGn6fvtfNO}}ZTKC5 zrv*Is_hi)lH@vRfjxCNJ*}3~dtLDJi=}IcuF2Oq-QLC2=+2bh3jv!>eyAGS8>2cOl{QP^!(Xi%p-L86A0Y zn;8~+8b5ugO|Koz0?QNdyB$@M9Sf7mwxvTb|eB%>UL6PKdVGD zUk!Hnfk3Apo$GiB9A#JDjI@2s(D|E*hMhUVRR!>7;{Z)jO)|CB9KRgS^lt6t%?5Wg zTWMklml$w7dM>TJ+~qV{0sX~ku6>I$X3v}#ZioU0>sI<-x#$<8IPQu($sabg6}`-b zt>0B@y6xXEot4jgH4u0^QuvfQRbm2qWN_XnByji1wb;zG?EKdV6U}(Wo*n&EV4mtFLvROG#{Y<;|}}HbiDEr`?3^Kp4r$X|gY~IkE_` z=&j&iNuadjrz6fTDnaPro zNJOq96OljTMQbb62PYQkbd^LTAG>E+1_xSeYB~_z;nrOnQMP=hyv>_(w$OPf;uC9q zyXZYrghrLNTtg{AC@h0YFWGM zGP-_^c{%;M5JaSz5-Sw@^^c0Wx|-j%viXfH|1br5YyZv{+qsp?i%a}-g8k?3%Ais3_`KhkvJ+p68RqcU zd5ipBy(BW~_x+GUsj9@DS62A+0u8NKL3Cb7LOwW|<0oTqO3f+k8r4WYNQIxB{DkTI zs&LWEy@d@U*u#h_zo=?JL7b=>p22T_o^M~1pM7m@K|#wE2b4?IVJ|~ z5bQ4FI7)5v#>A9R$!A$$J%=%LW5t3MfEbG$KgE5>vy7e#msMteKjF&vhv*&(qO?C@ z(6G-xVt?;av@iFs4;Ezkhf59=;_tTL4<;ya8Bv9SDW@j2A`QA<=;n)2b7jCm0Sidj85b>pul^pzvaay<$(FRA-%1sR3s^0nc-b3x9$#Ipr+#stWY;~&%yS*ORRb$ z9L6deR?&6g8&jiUbMtBJIir@6stw*K1aAGOiOPo9?a&kRV*kPnSsRCHAa1PPUN$59 zOCPO4tnHFwIe9Xy{zUQU)?{?Q<;t37(`bvIc+{S4>8N`phe8BlP}=$d4XjPNSpCXV zjQ+v5ll#EpDjw(SqGJ-^h2y4;S}QNm<~*pJ{wD!s>zHt0=11Y|M-#Y=ue zET?ps*5gvH&4iZHyfXFTZ@5K?o2m2rF=}$R9US*-QVo|JJ?|jZc=VhW+_I@@1E#$P zqLX&4yG@|I|AqX01=TN!BArF|Pk)?koj@xMoX!i*mP%@F&~sJ5K~&Fc_4Ba-4J*Qy zB@b>BxrYGVWHa!E`cPHJDBD|y{v;3*8A6>RH(iU%WNjf`xBW=_+#z5i>EJ~l$W~K> zE37^N5N$W{9t$dj^Sn z;Ow9AM7!kJgo9TM~sWMd&tXqUvI^3;6c3z*aBW#{}0HQPsINrd1y zX*a9<44C^;=79S`#NETYOXaYTN6rAEjl4VR7SSocL|~$fxa)xmsxp$SOMoF1QM1y- zuNm~*<&HR;Pev@OfCS%P1(=Swtt$=N4MR`IYK=Y#Iz_t_k1E;lMmGE_Cv8(ie~elm z2sD*mkET|fNeFQ2SnB5>v`PXHfs1-Ly_?^{u_s8pJttkn`4^W%yiw_1-o7SIgB=GS zH~404&@X{#X#*Ww2Vm(U>bKjx3c$;}Ui~Q9P+8@AE){Ybm$hgkGbqmCX@0ganNThp zBB0>#D=i037ITePyKm~g58-#JXw~100kb%d5OIOIjQ_Y|J<=G#N7pI3UNk`u1H~i) zt(aS%vYVd3SD%~XF|Kxg-=Xi``_z23&Ui(Bm8W_vpOdtH%7Zi0Q5H()LX#)@j7`VX zjurQn-@75xwTk)TqP2MooMn#0KfW?#6n6*Z}TtalKMfpI%kH`pi92) zeR8-~z(&NY_P+W}SoGvP(P(R?d}JRN9W5P1IqVEFx3b zpsC^95?Lqh6W2=~0}Lh)ivX^3Ik3kY5GWPdEUEpw@GshaeG%3x&z&<*0S~Ol28Nx=CZNx_81Vn>Srf36F3UQ+_Uu1$_z1C#^Rn$6f_ zN+E!OQj(uyXG*rzM8ntMu9j^L-n|vkVvNr@j_MqaowK^48h&o%+xSwD)U{~$FJ*9e zx661>VqLmVJ)OE!ro|Wip6*|uiRc-c6EMwBi;n4bIxCCy`hb&r7A#cSAu88u;6c4> z5mP0EyO}#({52+{qZ{G3ka{p#zPX@ypH7YT`BQ3iwG-Lf2Rzh%L4Pf{M?N}P1TCti z9_#RQq$j)7oD#&S%I)z@VOI%H&A|IzfEOkPhe~NCy`OZxD?Kw>mzQy2?IGE}=%cKF zkw;-(D?cNnBia#_fX^2!j5y`rKc_469Jp44W~vujtmRH~pZ=J`k*@wd_sA1oD-6Ro z>;EXyBMm^1&XEgQ6PLq8{*4gKe)_szAT)*!ocT4sZV?PbK!`1j{{N$|Sjvl8E z92y7ZwAw;>k)`IE!fU^*#?@EkNq{7v*b*03{A{`x@Xmzn3jC|zgkvW0x_sX0nhFz? zgWobu@R6K(d#R&rEk`*lEF}9@2p-}6p2Xyz1;q~uO@!>-KRfoj3*y+EpG`v0njP1j;} z+0BZ7SmYXH81X*3pyztiwps9xPXyskZV&G4Z0zI-OU%`)i!mnBqW1^;#r465b$Gc_ z-e!<*4G5*&YXXr^){wO~eYqPx@(aX3LsTI|oa)<&@xx>*#v=dm%OP&e)F0`-+y`6{ z??v^QVSO?OU$an_vX|=tpge8mW5S1wFLeppiK1htu*##;F$ZOBByOImAI}0174(^h z>zn;UtJwaRGoT!@w%o_MajmK0I0tDC0x zDPTjJHF=kDIaRr4)_TDk+qHgAB}B7;GdlZl{l^zcYscP^0y+60PtcALt$146j2>su z2yyh|ey+O=QuBTEC@)@z*jKK5JwL)Q>zlgS&TW~8Q{whlzV?;4uPqKFZ6WmV7G{n| zAGT+VYUp~4jDI8V;epkGo&U+M|Lte}rPLD2z)u1?Ru_X8Iahy08A9f~ zy41kl-EV#_wX)gc7%+S_K-3Emg3`6M!~ivw_PJF72-FLd4w<07qBwR)VoZ4DIh?nl zGy-*+Y5MVgc9{6BL(lW)D!J_w2oOlj5?GUb5;ai$`zgl|VQI;vW7{TeL2*YFL~_<& z@ipJukXrGjTaD@o2PUe_^qh1IiD|#YI^pG(QW^DW-Vt?<*>iG*|6D=WSEprLCw-a^ z??7LM$x+lKOsidtTvjeS9W{;(exv8Ga3b$Bro^jpL?^J>BF`Mkr>Gn>4ZGLx4O0S2 z!xHsgi;M<)60&iQx+Q0HZH`q8^a-1K8fhjCeSvUxCiMym1L36s?X?^qt7;k8QcpJS z&<8hlK8-@*D1Za~4wd z?`Qm_-&^q-x~q(aL&bNtpDD?X2CA%in#q1ni=NQNNwc3esJ^RYqf_O|9sd!^IY-X4 zl>Io^!5ZqQn_+P>$4)Z_xn&gf=r@qyI=}@W$QY}X>U=LVSeVv*}l+i(Y zyq}ZRYox3z9GNd01aR?^jVHzgR|vZ&6AKg7+yWx`PGH&nJ}KoI3RTF0%<()Ifm^;QPeLb;pT8q&<-i#uVcw{{$bADbuxv*L`i!=Y;r6RO*A&dQw?{eK96H{#yy8mK%|Ae9=y90Idq4_bNM<>1sdAzUy zABL_TG;T?OJA0dYk8_Dd8#Fdw>P%udNi9qb(#i9RcTOXxH9*QjQ68o{&FT-vc#=+L zHKt-gU32xKWqtQ~@%_lj%YYo6nvtGU0A~^Q>F}f4p~u!ZU`2aM3;F!W zw%O||->Tb&m;|cDmLM#r2m01p8=Sfql|~U;nb5tsUg3jDVW+yy09F@A)mgW@W0C#8 zb+2O~m?itO#eV^L@_4$i{Ak!jJZ2QRLl-zG>*T$qfVp@6HR`IUBp?}O69PVrj0LpryRL{o<5f{)X$Z@ z-tXNQx*m69u@8rMZ)JOorY(-$1SC#6I}(eM@!f)Ekodlz!aoY5yEij7Pd~Y#Az->e zGkUbGO)>8!`)$oNQiF>MHsj931`GKZwhG9~`zQ4~^mbKz^&lp^?+Pml3R1=eiqpbdbSM0#~A08Jtb%y=~@%Qewq0>-U-eZ6ExX05ur{MrnK5#%ukw8KZ-p zs!^_?lwVJyd{!7wvp3DFIYmMJZ@jAo@irs2W9>3OZzf7RBbcwCwGsO?BgDphC2|lj z#bto{y>w(l;< zegIz;RS+bfP1Bgb?dTiDWdA1Y)dB(^dT??@S=ELt{_&L^u*D+^Q;c1?LrExfLH9S> zE{Urixph8QkSPgEoS8kh)kp+72Yym~tNT833@ym)-f_2(N1c#@(_h(vt#$n+NeNvo zq-nVs)Sc;gPy+M9GeIuS3KGAQV5jw3d$ik(Qt+GbmuAC-o08m$IjketH2``wDBb6t z%i*)50=Lz~l{ZeHI@D(7&Xw!I@Yr6LK)(8Lp48UtRgUb3Js+BB{@hP8*(x3`?lSnY zDXQ=^45mNhKAG;Tfnhd#VlUKB^6ZM_hYxzEs3jj{Z>h24U|s5<987qJT`W94zvmHF zSC^;S`sTPaEtI4LRx{UW|KC)O_MvvWThz=6Os)ACBvYKDZ;Zp?+iL9-8m?OXS#_C|k7p z$AzM@Klvl_xg#wT)k2jWben{DJZjak;49v%>M}}UZ-;icU6_@Ry_sJ^oU1zee9=tY zfUf*L({a#8<`fbu*d@uNzqDAxu!E$qD{lv0nQInuf+2o7 z;(ess;ai>{@6z){rNUenNq~yJ@+8}N4TpJoYIhxoc=`#nw;iEbDszzGj`OaWUi@r3 zkwS~WO4&9A`H5g2&Yp|zwcx009YwWX-fy)w>~II$9%796lg_>dc;_=Mjp?9$8mau& zY;H|?LQc5O6Um9n%Kyf}S{`@=XBRJi@z-9?M!>{c5^Ux_J?>Jn$;6ZA(9@%8^p5PI zL2hz?z@eq^92UamXgwo%F8}YZ7`j95$N-Ma%UJTUy{Kq)DQFMpWCTM1&Rp4w3V}_& z;lE~7G&fxfoG~lSH1K8AN%r*Z+Y|YbsAeG#Zg!bc7_WI((^*+_*>|D}z-THidZ$N2=!zeYH~7u__=@)E8VrN@k4LPGMO|0z!MFNYHAnKd{ZWMS3(9>~ zu4K*AmaP4;Xx^h1bv1~NUzUzw_1tTk*%e7}6X>3*rMGNHd9FL&J6}%}Y=}X&ZHa%o z7j8gEEZhQIKN#;V_r3S1&cks=y7Ba}=4jjTEInZ({L$@{?B42E!#q`dhinG*V3hoh z^3C<+phq?kacoF^BB7}k?^#6m9oGrmTO5oJ&xyyL>e+qn8R*9CB_cAWhM5Yln@`Ea z5z~DAXF|&#Y0v-IsG+=OhKApyt!MOpdyvdjwU`Y;d%cNl^tVREpi}63>;6^1Rp(mF0eq??P;2^$5ZrY~0O+e9>_YSFh4@Z^%kK6PwRl{(dkQ#3y=PDSDPSL7jiKCoOTH)E2<=9SS+_ zo;B(57yvv>jw8)8Q+VnVBz@ovH>*XinRNeU#Y~t-YMvRIYpJ=26qF!EeZ? zR}$5mg2+YPRw(V>xP@Y%O_%z{i61XEAGJ$1`a>+bE=0qLamdY&%bf5q&7VHxL+$NhAQY!_>=_;mj8%oXS>)JecM6{`K&@gn73jaW|LIxLH z35nv3#ux>(!YBz1e)-5HQPR2w1ZF0)OB(~3?Wi6YT`j^38<_3G*n46BWvgWulC~}` zgc^=y4ednF*qfA1{4-X>Yehy3KGw*LjD)+)q{Bx?F?t=#(VWU+c_`%mUzG*m0$0Qy z=ees#O<6{7HV$X$%hw&+_wX#=3CfzVVi9cbeNvB96;{1{IO@&Yq*bdDrE^wr?oW%T z_goGOf-ZC1I)G^u(H88_r;)4J$vdu2JstKzc=VCLkMGe}uXA@AOEcjTALYh=8`ak* z0U~A_<&s1W-N^Q)UuyCz8s$CVCXq8gkD#B7vx*@>C`I2fg{N9W1vi;eC` zVtg*Zoqp{R9twrz_^{+)11PcYJjYtoiJNWkiE&<*JF>ZuvCt14*YqPdw2lJ&Wm6E#Pav>aR=-<{SClF9T z&>#C8kJ?IB;?<$Z<#p(O=)pI-FRmecBMmvPp!fn~mIigYw*aQ$zp_FN%A9rH~ zZt-jw;Qedx8sP1l%M<3UUa$Fj&c|V*my!&Y+VyZUEJ146D&J3i zi>|e&x#e_tzpTBMFeQ8w+>b$w|UV2dImH_G$BWKWi zK5R9OpY@%1wbrjuT0te##UcLF?pxaVvAdFPHUSmYAH4nM+!E$>7B)Ia|6k3OU3f5O z_)4^RG-|CKrd*Iv`H$Y}sxYQrH1tgd6cxr@`&(1p=vfR|lWQv~&6E=&Tl*m)e}{;I zYt=%9#25p2K*T_hWBh^JOQf%2U?KAN8txx3m%H0I0dqH8g_OogX3DwaZ7$1MN#^ ze;xRgfgKdP0c(aA)>>a$hefYGf<9er}>ZGGsW1z98-x&S>u(3}_fW5GnThS@YlIo&n6O zBpfJA8Xew|`HbW|h*7)ylj{Lk753ovego8!yA9yA+VMsG2*W?j++^*j`z&aNvu+Cyt-qbPIYHAf=v|YxeYYE4#)z545_Qf<%P8}?r?C{O)8ugs{~D<-1X3D@C-lUSKI@d zH`fQ|L~1x4YcswD3Sj<%i zv^y;p_0HxNKN(0Z9ut+YY~uNY5^ei1D2k{pq{IKn8bWc);#+j(f77+Z{jv}6Qp4y< zcnQD;by?i<&_`a|l5l;oyzd}O%1iT@yOJjEHxKWuv=%Ys`HqQ#pSE}3dnCC3=4~6w zX}8VkaGZ!?Xw{nNp$oEu=MBGShO1JArf-8t<43)2^a^lgY$c!6Ij&QH-Z7+I#PCxN zlqQlNuoO-0HHq_Lb_zxhv#O}2uG7j6ydEO=+tY4jyt1#H*e_LzRCO$W0=`R3_1$H5 zKwU|}LYO*>KBbkk^is27;YYA2wHNRN`*=!pU}D1b{E_aL48i2{p{zi&=N6b=5O$4L zFl~CYBRh83yl7jqa=+lxrMVN%&esMC2CU@qRu;nXkWdaGV-5M zOfTL`3M)njIL>r@0Wu;gFI^?ke?;4P>+m_g4}D)QJQS|ys^RHgvG}^@N-FzoedZ_& z7*YmOPgTw$Uj$@-dO0Yh;KpscdT9OREM}J|Yzh#Mc?VX`d%p+H5IhlsqmJc+6)7uY zHDA3oPY(E!HI(%j)AyX;D(mvJ;juW{J-e(!ESDWQeHs%N*r+ zFOeFuGhJ@4VIL>t42CW4;E2EJJ7)4l2xgx=6~G|${5Zb{c9_>t7PmQAEhToOvSZ$^ z<6mZl(^PtIMfE5TuD`+#QBjIj{I%n;w!Ty*df`RkGy-xHZ)SKMiC^5mzn^KPaU8rokX2DRX-=O^y%F z8(%|Ib)}%mue4 zf@M}n^{^eC@1ywtv?8J=1U0y};(fO+ZuxN@1<9>TsN+Ns=rS+aUnH#cIlvlh08S?V zK~9w_zH{%GOd8N~xPmht+bws7yb{N`mjJQy$Kioeu@IFCaRFSL7_ah7$0lpwqF?a9^c7x$+Wb@F-WH z-v;2q&+j0tT#A+-e2y*A8~j#W160H1K?1~063e$mBZ)jICJpM&{aJ@6vN{BRz6gge z?$Fyx^vr~)^i;ijXNA~GwGjB+WR&Y=%{StuL`FJu*3X59o zcSb(XPFC4He9@u3*+S}ZMF`mBQOmqF*C|v^-xp1QRorVPB=7nWK)zc*ZlnE%MK<{o z`@0>aT+q&H28`ZLj;29{wU7 zdi-78>GLi!!Ru51N7i|Vv)R94Kf@?$v`SU2=TX&GZE8n!pjD%_w^B7i)!rmss%EKL ziD;ErNd(c_v}(^1v5Fe8i5>e*pWk}l zQbma3({;&9QVu!lrDsuxap%zp`PwYsfN)PU<=)CNRgyRqcff|S;daTuP0#G_jo>4l zZ+dmZjfZFW@$h6 z8l-oC`XX2Bv~<;=64?e0h4aRUPpm%c7j-2o$`zP|MN*)5M82>J6PRn2IUGQxHa~w_ zw4qRtef)k&zvIRE1<3py_jk1Jo5I}FbZ~}yv-1gEKsLa z+zGR4koB%p-#14`{rJU0glbyexqj@JDw%zCbJ;?nbI}h%};#;_xwC!o$Dl8 zFYo@^R%j7ie-l2~YGIqWVvsY`4s3mLUDAkde_ZvQRN5>ieT^3lM9e>8cpcN=1~dj8p%B6TLNniXIUrNkQUHj9 z2}KJn2xzt|nk4l@?`#Y-abY{#g!jSBliV76g%WnrU}Y1}9OJGi$fD;>57E>Cq54#_ zYPos^_7bt?&UZ&6O|~mP828U^;t1!J)Vg2^w%jM0U+c0W_Ma@mL1CXM|?7srAkZIMv`b#XdAuR)3hb zFRquTbk=OJjH$x<6e?VNXTzX3_CtI1)Z0{?ng(IP3^PUKCmxj@yrOi~YMDi@X>^lJxwX#W_U`&BSwz6+rG?Lrjlz@G8z; zHBj9?lEwb4i)`&6AS7%+Szu1u_Y#Kn#1!q(AInzvP@I3IduzkD08x4Qr61~h-{wx` z;MJoZhrcz3-f{<-`0_+KZO97a)UeNkmw(`254ii0TGAATW^Ly>!}3ML>E^fsP^!fJ zz}o=_Hk-^2>t6BZp1yJX=d))+ZeVTS?_<<9DeReeeU1ufuXSn9l6^gZjT zsj=nl{$OmJ3N8dk_9?|>6u#5rC#1Xx%~C!4LbfMk-Ph}P=qmwOy{n;4NPnZi^6Kao zIVo~#ejexp(m?nHbkPnsm{(6-DJ}EA9}Bq1K7TG~QF&v0j_$~*n{T{a z^qjF~+WLN*O4h<94AjaYWu^oS11LM%{Y4}i`hD8vnEzN{5eu{`ggcIL^s+2d`MuU1GxeLiz}1(^6jB;l}!iX4(0&kS8sHj9q3Y3jTI>4VM+2m-q{h zVJmmU=Z5ppVXjT~*uGdx|J($=VfW?f!F{hbm7F=Oee&vvQJQZ+nOa zIq!lQ@yeYYtgyz;H zEbi(*DE-eknVxt$9{jV2`IlQ{6l3(=F0C zzMe-EdSoFgdF1M!$O3aVAQ2=2=1z(8!M;27$2>XSONqwf0EX*dI_PquH~i{W zA)&eZ2K1g81=R5d#>`Df6;5jg8_oEWqT+bd@+nDUWQYe*us0_`C68OCrQOz#QQdSR zT|T%&$vSv#c@9oCq<}^8AX}!)Y5?v_=QMH<5!Hv4W)f4MxvR@~N*_k<9}XqNVV1(k zBk{J2SAEaRJahli=1Q=gvn}4BW(&LC)gY0EI5=D6L-$*)IkwtpB;0Du_qR^&ckBIe zh1jFw&Lr=Xw#^yyAo1)?03&G4k4 z;Xb1#t^1w}&y9k0hM7-o-!;9}tbFpR)=*U0Aq*YF8kS7t6kO9IIeBv*(bb^tkAz0{*=tG$yVMEhh@)X0>I}ElU&L=sD^W8o zh8H6XO!XV@?UIZc@!iWX6sV~QYPdHz!fj5u${!u|+Feh|va!noSY?KiW8M!x*|rE< z@?|oKzI)xHoIYy~S@ZRCG$faqHZ>vniCA*?B9S6xP_eyVutTCtjhF%*)HJ+kTgY_H z9_JNjw(st9PeWKIK16{@O{GtneB5uBvmaEfUvsD(8S*5SbOWDLPY-?g3w4Scs%pgL zDGIMw;Xm8ihtgYS!=q4qoJ#n@qV)0JD2Y3ES|eG<46(FpV6rgyuO6GKB|+U8^_til z>z>GD8i%;1jN1RxHuR)|@O7$n$;&%7eBw2w3zrg!1zZR4g(LXZSs<2?XSvdV5kPmiK0s_z_!tBi5FLXofzS3a8cO8-KNhSn60TA&CTckL|Qz! z^{(yhQ6k5%TX}RO7m4l_c( zUQbOcoki1^TsT9>VNS3P2z)F|1Bbfcqts@FGlGx+nRHL%G-BDGbc+^g7-t3eL z2j$O-!;H_Xpgk$$$o;#cfNZq`9@R|Q*feum+)gzQ_i#UGi&7;^zOYuj({$Im>Zkpx ze^ye92X>(;4Xc6 zlG8S@$dy1Y?9t0M>uOOBi;3*Y>=_!tq^t*WrT8`v$T7#Sx1F#`-8oSdU#A+1^B-8= zH!_?ov&GKe%27DF*6@D1!eeh>$oBcm=6X4=!^u5 zjCr5%$^*6vZQBk+`3w?F+NzL^>mncNnp3o=d7z`NW`$cbq`!U4(|8}C#1!t%-6Ov=P_*cb70dXb_y9BUb6$4tx+fm>+oG-c7iGQ>VHo%u% zhCu|ukYSX%zwpFpwcUr?E~gVvo`bE z0X~XFV(r1t9^n7`-Ju(M<4%sK64RQH$c<#Q$K=Wj-Y-Z0JI&twK^2H{lOFv+6)550 zoa95iuorw)>?^n&AVyE*-lSl}!N^OotbPpdhp-|hD?mes(_Fd@NezNP50D@V8}!5O zI`GNV+UcKi^h+H=t2D`Es{Mxd!3OL|_f)g{9cAsX1+JHv z0uV6)+a`E$#B9yJytb#0BPMI$Rm$L?)!T3dXILFHiDx_pg&lLdBWgX8wHIgLsCc3{ z=XzlRSRdej)^je*HgJe^s=6~pfNlkwm-FYh%Dvq1xKijz4GUE@a8p8fB&0Z4W{z7o zkD!mj9&^N07kyfRwhXQ7EzK`{6?nSnG2qh-sMQF1HlKA|(Ra!H_x28fb} z>P`X00Fq|ApubVWWwX+Wy)@Wi7MeXaZ5>AXtKkm(6HgP1Zi~w+6Zq9bbG}Oej!~%M zg&s3ukuz#H>NP~pmE3>jPEV7bSvSqTdc(g%=*qXJjQ)kF@X8m@ik~!p3EHxatn=>@ zmtAT2G~X6Ux)u(4K=NHkPC*ahHo2f{_xou|YgT|pntJ>=B(rD!Lg;2|&q$NFxzilx zeh(a1O#6{w(FtsTZKIGmFDyWqVX{*Ar$|(fe5h(J&K=c$_gC(X0ec@Sn91^nymYD%E;Y82gaiYq6tP|0c5%DepqpX>c5Orb>h=ob z4^6F(*uwA##^$cL-(6mIb2ez2C+wp*nz;q%8_l?PhkN#gc%xo#$eg(Bi;LgbFtl+@ z)gftaHyJEXsbm%nwOeym!i^*P!#FzP?oPaTj>_&pDEs?qgw_Fm(;HWj06-wnmrVf0X)z#!HlxjJnS=_EYa z2HV?US62n&3&SV#Dy?h~5wy2?ekXBfVh+t|_iXu%BbwvvjQii=ziMe!SVl|u|5#I< zlW#fG@-kfeG!TC+|58Oc3DWzh8$Vik?e`l~wHHT7y*PgJaUVckcu407tAF2B%@=Bq znwMyep-Td{(afArmlBy^`t z1&DMnJ6V7Hz-ni5LSTe5+%AAVm`r6YSb={aMP|pNuwifkhbxJ1WucO4`LD4|LZFCw zOq=Gqkj_)|Y?JuJ1sCR1rFv{eO&n=SJdJYN4!Dd2q`9)^q2BTngQDXLI>N_VCOaBU zx_DLQ8jcj&y^t{d7HV=Q{9E30@JD*M85Ea?94ldeYQv z|K9a!^yfOTvIl}gxv}$FRuYRWR z{w5}C_;Ez%t0UFsVWRXnD=GT->2V%&l1Z&y;e#E1GzE_J^wC{)0gRc~?~_>Su#vw1 zuf+WyCGU@IzebahKP)sdgucR|msMSqbw*F1#5#j>;IIds$@IG9ZwvUF;!@MbRKBuD zoT~tn6?pv62-7N{0AK)+Or7LNwBdKSB0FnQflz1=5aE^JM`4Z5(dfaJ#WYBZPAfO^ z!lKbtDiURxAr%Vg*K(wL9pV6FLW9FkWqA($!IYCt$phCUy z75iMn=hHhCZ#!h4lI)wduZXHTskuIJEBa@aXM($V`mw@MUVKZ3TF(}mkdUJWmu@cL zCS2ahpl1wsDC(Q3{JGx+d}EhvBHrEz7LOob@&p+ds`$R=8F$C>k0RIgbbMQOPUqjd z?*X^MRmPjLMjT#od0fZ4?<9@z%I!EQEKkfZret6O>c-#uRbv1%ua$1E)wvNHud zPF}jA^5(}O;Uj`yUmsi1mWsM;CK=FDG&&a|m>(yvv7NvEHGvdTza3PqgfCcvVY+>B zMC%Sw1A@6e3S;gQ9lq5Q_I9b*Zg}v3Px$-s?}Wnz3F;_Jb3;@KU=tb5jR^fW&VcG| zN?*GT@H&2)t0ACiZ2s3vR(FOgEPMpqjEScN?Vp&J`y*isrgGV1#%Ev~g1t51FB!cC zG;0KQM^R`_s!P6aH0;xeIHVohI}fpswfrxV`d_0p zjaOo_jA%zUlBu(FP}%DwpKo>inLq9Dv|jVt))TOwJUpUrP+YsroN?;Vk(kfS+5M42 zL+cE1Knm-S@*wQz`Rr33Njw=okWp5&4G0f%v+!iSk8(-40Su9FB>8#1`!0>6cD z21O1%*8usG(6@_=uot4d95_cC6aaaGLvoK$zcV31Tfp{K7;_`Q3=|suE^a6Q-pi2t zS~E`a)Dvb~6=4);;f(|}&1U9vy6Cnn%weYdxk>-iFv;@{dZ+#9+!NkFA6_zpPxBXz z3=8mY0iF0cd|YLp4)PF`wu~3yTr!o|QJbgd1dW6B@q*d2Hsy{v1~3l!NNuvDi2bYK z!QkvOwX`A`Q-!x5x(WKW*b%aoT-cQ3ZqcnC^VbzHY^l6~8Ru4&owyzO#^yy||GR-Z zeXS82_cRKGC4J9~4y7vvR|S|&8fq$@s!C1`8xW5h=pS&H;g*r!F1EMw%Qc)N*kDW} zcNFbq{;GE&KRLt*hpr@Lb$2LIpTh41FY{v+c8v&ctc8~RYy01iU`Cx;?z54Ladkkm>d{DKo*2MK*T?6dVqZle!UwDxD#FOm zCm57J!XOG`#nHrE?E%z&mcOjy7ioOCt90aMIxOi)`R5hn?$VNqh`9otv_T87;XsTB zkbU<})YfeZHG>Nu&@X%ObC5?6;=shZ`w7x?VTMz=YrD+-b5>pC{C#1SzDrgs9;6)p z#N^HLyAuuB@j53&Gh_$xuDG|`RE?i>rmtc%iAK$rjQDX=XZN%(u}r^yD;ih5Bv(w$y&5%*3(q&(~sEWEI2lHkGy;P$N!(`q0-%XJ%f;wy`G-lrO3a- z2YrIGraY-D!5LlMxjaFQJgx1F{wYdHw6k+@O5;OCU@D*22P{=PymLk}xOG@LhLgZ^-0G#PSE$=}x!Dtqg@@fmiJtbY5I zgRilM4avst-O6t3!nD?P8=Y#NGsECc&>h10i*W(RnEIuRQvAxHkjp9sf4oS%aB-w- z>6ySA1C`W}4OPI0Ff%pPb?_kT;ggBOzVvYSs~MCLvb67l(1W>4-phU&B8Co1+5`+}kOn9sbCV-b~f}eR?qwHlDzli5y5a*-bStBXhCK`hbJ9zyd4=_mC6KG+o?rjn^Ad4t{rxXAWh zlSv^9!E#_`)lN&^cf|><$tb=66|(FMy3*%}&x@J$j`)aqTAao++&A5H%%MPDO%t-p zf<8pKOlBnjE8sD>zhum_n=@V5Fhf{IeL{| zXF5`+83>U^RJM~;_ZEaz{j5K>Zaqky!jZ$VUxtSHH5LEfMdcBcAlN~e!aCzj7xHsf zfBkF)Hv1S5=7Z?27R5_sF0N zP=Mb3#6rLs?kkqhS+kkmr%krOB|*sLWw0*eH&EgbUgQk{Tvq&yWB%`wBuF#J@9__D z2W=|{-b5x5W|2HoU5c{L?uB`7YiKJ!xPFRBxX%^j@%@!kH8L2(ZSMj44S3RB4tkCF zJXO7DoWl;~eWQothM(!_6)Q0!$**6uzRwfkr0K+eL{BC~#EAw@U;bi>;TNyB{K5Ki zd$KXq-&u3ct5YIddo>$fws&u}*;Kxq(XcHlz?UNm-+9*^@2;qo{m6Lkuk6I2Pg>c>^BoNBeG@`@cmOdM<{a7p}HB*f14si`?$`z`5BTpXX3o zEZ5RAprIA(xvJ)|@V?S|#cw^CaM*qHZZmzgyTz+kXH|jjs9m6Z&5??gZ{a_P7OQ}1 zkRUjvP1#L8@8SwaCqp)W)WN&Mo(PbIus)jpBZPF<>;@lu?4+^n##mikBQA^n=cP92 z;2M_bu%y|epZj(T*e~N%W$hPxfv&)T5@Ep9b%z@~5h~VgcO$8H04jKdvI{h36DU)B zifJA&Q@jH-9m4mTzUo4f9XX*atslaEFfpx{rPDq50j=qRp$V)SbX=~7TN4N1W)$my zz`I|Nj(4?8hn0^r!-gI!sye|<2b*sI;+1D5^X3_*fTtM8`t8vI;4+3!tP08N!-I4y z;ISZEKDeP^dVZ}&B`;yqF+3xLwKG|rW@+u?!Zz@(|H7qn){37B7$+@!An`&yZD?%R zL%rw;*gAvyPYG(mG$C*d87jy=-)2Z3e4@V_*ba2IFJi*bzZK@xZr7aWL~e6r?b-kE|Lu5OZiX@vlO;kU;8Vt zLPS=!SfWfUU~Yvuq=<-C_K;ue(-BT6yDYu}K@EL1?6XV8^UFk1YVA%O6GPn*w|RqdZqNvY#9OhXs&KD6-y#9QpL*AP8%)Dbkn!nr#2C{Hm}EW8Fpy z-i9%V8*^lLv|_F4#9kM^G?y*2 zl`2=H%MS1}l$S0j-*3DIRD_H8nij&+=Fc7)AC!PMkc|&EKr=HK3-H1~*ir&z{R+#DIJDm!QSYw_q6o`prEp44%d~3Em|E?;pUzK?hDjBSTRxKb$x{C$q;FrGY zbeF}KN*+6Yc^YSo@3XC4lAo^KfpN?vso43p^-L~ew0*NGrtoAIjkq?nYA*Y;`zX9B zX>5}_$3?bD8nyV=uPATbaxJ$lRZ`?3dpfwnQKJasdMOY7(xu|Ut$50PWHoX)SIn#e zCPZj6JI006-BwWp+7!`>|MZ!3GuutNnJxPZYt4o6YjiNmwhIMh2mcEyvWwv7F?qyx ziiPh%|LHJb1VHG=35RQ+wD)7Nzjh`s9)lZ7h7sZ(KLEwxT85Em4Zx*{xE~PcyC4uV zK@+S7zaYXveG|Z96tmI~60uCVW)x$Ze67_dHpunT2|%@Nv?f}Do|#_V0WZ*F!@(xc z=;U9*R-3xcNmE0lmB+N&o|=7lGd)XT9XIslSEC9cQqCPwvC(#Tp+ngeKhxNaM7$Vy zRcZUzM|ZXm*3ypMYZ@FRue9pxSh1~|_-s{N5j}rz&9wg)@#>%=?>3=7qN~IG zwZf+;fwN?Ham+f$vm3Jb5ZunupRC^>SMHv%$izB;h0d8xw4cDJM_vfj{`bMzQwlN* z4BhNfa$%JK?9Q|$?*$g0du9pE0Fanfc9+UM5DB|wwCL03Np;@tV$ZnNUyE7p=d?j}6c!n&2s zg+@)PxRZ%~Kc+hpD_+TGj9M=LByJ1$zo575!hhewW$P}gdnHICL42V!_kjnD~Zn`&m4sKD{c>0ivgW=-Qqhc_Ey)`S~Q z=YvgtyW9uI|MFI_E8#Q&!}A?pYWDH00_0pp-n+BCFMzUTHZ|fZXq<3AVL!1|fPvYv zoLP3pBDwBsQ>QlA)du7|^q8?=t8yzjvG1NHm*Fc50lk~xW*Ul-q^`}PIhEAZ3k0xauXQy{zps(Q$H zYFZ-Xf_yL)8*T5|J7gI7L{Uk?l^&gSBg73>Yo-_ls9lTY=mN;L<|I=}~vJ`uMpJJ;CVdpNj~@_gF19f@wId7>ldmr6!t9-LD_p zbdOWU1YEn_>V=!Lh3r!$TvUfKmRJllo^pkn8AJ(xQuC{*H*>vwzTI5mmu)6}9CMww zLg8Uln2mcSx&SjHgOB;Gb5cITIpvF7DRlAp?{F;ano<4qI)v#;Nof5>*pms%f2(=w z9jr5JrmrRr=la8hQa5{i;(H!0)73q_G7{mBx@R)E9D`IVs<~6dQv(o$3pdEB0dl@c z@9ov_Bu1}!Ab2NPb^u9mFA0Wkb;o|$WCU^tQ`4MJPn$eCUBhsj&CTO;5$j8>A`D zo&+9WGZo+1JVnkUIs97um||j5rIopbuT-%48_A-t*oa|F7F4xfmZH17C6_lgd$FP8 zkbV~TOIEg?Zw1S|jgO{tGCCEUTF+A-8+4s;%oJ4d-O548QifVSs`YRi9o9~BSvOd@ z=kQ;KIK_ym6jzW81bel!puO9wZ}PTBxG`4pQOk|S%Qa?tew{rvs^>$!i#RtYwJ>^# z8DHg&#nkvs;_UWTojU2Zd^!aFakG+-k>S3Z|F~S;^z!hEnL#H}0D7IUgbpU=`r`J6 zHn66sd0kQ8E4Xc;=KA2S+8@e0OX~WV*#f)J-Gy}6Qa#%YW`w@-M`cWy}rQ-xGCQI zjyLBKTZ?ERa_X{3?mtdfnY`bRlCY8nr28iq6@!mFcJX$wuH77Bv%j*})rP zYXDu!%LJ%lHBbGPl9&ITs;2a>i#MF|t6ATeUOZUtJYPM~r+Ts=3XxSi8O4R=&ydha`J#jr~}YZ~QV5QT_4FwUB?I zo2aL+J=+mX;PDGdLApG4=#6T5$gb<3o1#Ab@jLM; zYz7ltj5TSA?En`z8~t_4f;<57^fst1N+Xfg=Ikl)xl(r86mfkR5n%>x4XzMjVK?EW zUTe6U%{w>!rCGIy^Q#llMZy5y$_kt3E}4GfjQL$ZVp@`S(*s!9RQ63SJ{PUh3l%Gu zv+Meyf{&BdRtcXv5cs7u$)X$O>1Q*ic0Ai{2Hx&fqUvR*u$G#<(MJ~=Hv6cT@CegM zqe#x{Gstz6eRx@>cf5)oqXJ2%u4N$IMojvq+Mki`>^9i`7# zlo`A(6TF=!>oHgBI8@YcnmKew6hBL}gAvNGdoD!z%}4QCF?8ez-LEu%veE1H#piG9 zJw7!^oY{1226 z24@Yzzvm=P1?-mPWv3RaGN{v&E~h?&9%h4k+gRPH_K#l0ntx=3o=F2Ul$Nny0lx&? z4q9HG8m+h~);ywhh7c%KWCv7hnnu%iJ25&(c+P|79bnZX8+!P1xZokd7LiV4dV9Nn z$PCaFM$M%Y&vfkYJBZ4ugf86PU9Ad!RFPrJ*^9M}c8FO4vXs{I@^{>cJz)&@)I#VK zd3RYzi6`@4jI`vySDRaBxKDtfm5bJ$6jw3;N_k$#f22OhYrCgc-NXBaJG+h6tP1lv ze{1UfEwj{QYbymu=A?&g0F zlhB<&$Q}tfCqZm6TV8Zo(ZOs(nsv%a;{IBJ5c5fsI+smA5^dQeqp~AbM&&l855>vaT(5a0RNj_lMa^? zUlUMl@CT3L8bmH+P7Do`ulWIga)D{!0dFQ4!}6&sWo6T;c-CRS8nvrA3mOLtZzxJUgZwB=wl>5Y=4KBviO(D7Z*MzLv=b|EM$`Hkny&FvEA<7*4LWA(cPXDs` zDYu(W)5%sPrV&%bo4O0)^-!0FoF?1wGM+Vt=H(L#aATPw5+Q%$(V&?=&CD5318Q6V z8!^0pcrvoauHWPdlX9{SZ_lN4;xBsz-^L4iT#n!*xPvx&i~ z=3EukC2+SMwYF_`cf5ox76KehlHwn z7drJR*~U0i*532imx1mO&U~V}`Ll~04>5~f8W#0qyNlrpPK1QJ^yfsuOZ3c~sH#+GiXU0dg&f<%DY z^eZM0$OkLvC~Wk7Pp~*KBAYStsDT}D+Tq8r`%Rz#$jwr+@Ec^^&AJhNkXOf$UIxVi=CTOcUROA)YFBC;hdP1f=H7dPCfRD zjdijM2vOlpd&5iFIz8niF+NcCm5h{+Al=nicYFHdX-$){;yrf$s9K9_*Ik@T2A;hV za?*tPD>Ee^=vLyZYSV?@&@;Vnuwe?*{V-2D=>~po^jD7v21m=r$QnX0UuzH=C2*NaqRjqe9=)-!Bb*E@)$>?J&DTIKHxdR2-oOHW@+)ETsX? z0YIljC%aj%GdyGwIB(>{Tu=2H13W~)0oRMdZBhB(q!Id;Z_7iDXE>2N(ZGTxuQ=IW zM)X;LSLF!1Jw5osr{{-bS%3^{xcheTs?eg^mdLdFp@!dE_zA z7r~}HLy=1dQ^gJOPB7mrFWC$jp;%&AEYR02g$Td1KZbFmBnwK$X=TDK!VI}w8JMnc z*E-`L3}c^b;_)|XVcdQyWu-qedgBkR|EF9T3vlb`}BYaUyf&- zuhQDGjr5e#jZA*)!{Kg=$XVrwb8X(6-JFqbZF=Fw)S-dDUOTaEC>S2Tb!O~$Y{ys? zR2tel;mBX5hX+I9U$x>#IO5+Y!y0PGkMzP^c3Z{__Y|6IqS))JP1h-OpSWXgI1YRe zhLy~)vVCB)IOUPj)3-}sM~Omds3_IQx(^qIs{xP=UgHpeXn>^Aoj7J?rX<~I?(&B%9OsB2nSB#oj^amQ zCAy4AxTM_+_}N=#LT6$7_rr0&#Z=bIz-piu@IcVU%5qc74UrVGpX!SQBxfh%{-e;l zJ>sHxfK}L_y6XoUFo9}lFUi%(dY#erQ5ogZc?W&7bMjuzz4N9mvXIftD!xWj=+eCZ zZhAPS?wbP<^79t*uuXJ&S>)=D`|U8ZFJLmmH($@zf{FojNE<7xxj?0CVxcKj<MMNvbL@{--n@!vZ0?)is$h>`3{A7NN~BmBnXuiPO# z{~PIyKHc$2^sk=L%aO8-aW%74%?3e^SKD#Vl6X5A`I3{YQKYLv#0>qRB{Om3RH*Odpp$P$4 z%G!wP3!0n`Wk6l5!piWlBPy$Mbun0Fb|heIh@v23>65)wm>d#-ed9fj-V&bvDonDH zsjI8DlB@HMOJ83rQ+&<5Lh8TJc{%X7%NJ02uo%tTm%`*Lr5!JCz@b+ZESB&qTzZ?K z_4GL<{O_c|!;4-+GnkOr57OTI>_&Zcf&Y$RX59lus!0L8kPL}{hj0P-DL^*hTr~oI z)(Vg&Z-H1VHv2K&?%`ps`&|Wllc(W*6W9k}3YR#GCus2y7w8*&m%-U4B`-7^^d}XM z8>AIqFIImOnH1bPv;(O%Nia9O4!DbHYW)(xBrGNjA4o_D+9L3GG z{+)l)6m-0KK73b9s9ugYp(KPz(dg_C3)mHCU;l_1pzjlt^5*;_!wA0xAy}belfVSH z08gDvkcY$-0*DRb4Hk0aF;R}U0dujHB%0O?0kT1`U=62`J-9QJe==DR@Xf5KH-?nF z`obBmIl?ioN*`b!K9?H;YO8mf39$A)Y_ap-Q0+Z4&p7s7NyPcf^UN!sxQ*sg)(gZ? zX7%ywxmX9ga)v>Mhyq!m+NWr+CeXg&dWOm`L~vMHKR`PB>dH?WQ|hZaOG{Q8?uJ>a z&w!o#EzJWSGuG2)iq4q6OWCpNLr+fcq`F_rNQZ5w^4plFJEbnYuJA)8jsXzJQR0Am z*y>_*+|u1x?mT{ScgC%46l{Qovxl0L7X<7W8*W8EjlXEzsZi!6<>E1IgH1)IV)oMZ zPKP>~sX_WFHFk587K_SsX(yO9t!i-7eQ7(R9%k0{(?ZBGo?RbNnXcqti$i`LS)Kf< zQZFhVoOuuud9J_E7=Ck87Xz(a|K8FWO#6H_5esSx8~hM)L9^GE*aorQepc|oDbsUR zfB&ntI90ja%9^y;_(C**n8N?*mX*iS#G4GA$!|k_=US~s-Mx@32K90i%ZDe+zX&}~ zEk8_b%iWBZV(I5dWNza~yda=lHX?N9-B|;eHoq3&?0UI6F!H5!|FLMI`o^Wl3^N+2 zzyIQ4VUUkM&7ytd?+e{7=-tq@s_O1dGftC0bJI1++O^;;-$A=j?1a_9Sk>;{U@%1! zw9l4^LGC-uEdgFvHU)>DQ)U%lj0foeS?Tbh4)JFm*W|R`I*2$lO3)f?s6cBYFkqS| z|D6|CJGZVgJDefyQhin_50Zj}9lJBV2dXfbeC2JO-4@{9XpML-H~gG#&UE0a+}bM1 zPG6?1?@MR$MESZ=g@&}ilQG-C8*DOF6oY)xw3rWB3YVqo&e0UeCW+stY?FBwR~W<}u9#bQz; ztNYgY2q%{=cTQ2MzU}d+z=8Wmzu5;AbbtZH5sWKZ4swb?$b@ z{PW1Ob2K5k!4I`9gO+H?R-*1+n5Y3UhaFH9HG~pCItcp;kQSrb=d(sK=sI~%u;SZ< zaHwu#J9j?yG3PhHT?P2rgET`caCZ~D!d0~eu_2DmWPbu1uPi=0_=1RyIA*UiXdLY@D@lQ*g znlwr4{UFcxkp#FmKy`Zq>Dv<&GhIm7pEtIO-qgr%9UG2ZyoMJk3{#U5Jf01Ijhjk2 zN%mCff8XplHR`>?Ez))BiIr8nz!R@IdG?_yT*QI7YpH3`aHL(PyweUOal_QGDBTJ) z*)*{qID|HZbETsb=fT}RrpHj)-LYTZjvV$O}*lNjnasbm13vssEOq$ zvtv+s@ELx4M#lNQ3m5-^ zV^6$(M#wc0eHyyNW?#*#)$N;E7;bDf^5_j#R*T+Di1eAP;-S@)emU#8I$UZ~N*?8d z*RjC=q3b=|n$Ewi(a=E@L=+UIC@Lx{C{0QV<0vXZR74acK`clUG1L@7QBe?3QRxW| zNN6GSo&ZDdNPy5mq=ydaklf5Uzd7%D?{n|-*WO+}*vz z!nJbMxZwSjn6}sqW^2b|ma80?v9N+DBZR>g7I6CX_ugoOC6~wDaJd29n5#ZcIUv-^ z(8@ZCm0sgL)`yA+O_TrnA&+DrLf>`x9tz$x=}a3l))3WUB3Sd-EXHSS5JTh zGNHV0Z&+mMxnQq}$cwn+(V`OwiC}3+qKvxu`=u-?C%}HuY8)hLR^*S+JoYI5-B-tE z2?NJk(fsdP7?I;lg^nq+Yx4{4vW~HbYp;B8lG__ikP*4Iz4TJ5#{7W!UB$oRnRQ0n z=Wi{v_1}l;BE!xPMvCM=st&cauOD7?vIcw(bR<;KgXKS3LHiR+mk!%>ySahDddf_| zenWtg_Y!UBpzERm%BdCY7EAMp&~ds3tOMeDt`wZ3G`EflW98|h0x?4-k+YLb-Lx-_t6q@cv@E)E0LG&qeY^N%B^Sjh!}m4cYEKBKA!_m; zpnjVSBy|}yvIV#!q8%(PDm%#P(+Yk~&=9c<)~nm6 z=$t<*ZUB)LIRN>j+UKlZ49J{R9Go1b?j>DPAyFiY>E>t1EWAmTm*Iv2f$n_%txqB% zREO;|9y5psC{pl{1Bl$2L`^_r;@kV7kZik`yIwY!6z5)Lj+M<=$j=o`<~ z?rS^DRZncb95?p9Kau(qRl1p@D*{O^(5fl2n6K{=uEWFohP}?NAzxK24GK-`Kn?p3 zNwq()l+Sy39#Tkgmsc(-k6UejHggfV70u*#TFd|VWFmngFUk+MH$MbA5dDes`ys94 z)vD>vj|UGI<$16ejq%niu0DH7zKv(2f#2M&kcyqV{ZYoi#_y7dV*D~|O2L7ZN++&Z8rTh;9C)*n@*zy!$H4i~|G{vt^f{4mv1x zV8wvhLVX?uJdcGgily^co=|9WbSVCd*_@Ck32U#8ok))ZYvLZ43f}n##0{64{{!KF zxD>@(LXORhvP2$H6Mo-`a#4{3by7+J63;7?^hM)2CV->YBzge7qGo4Qi0R~d#%t=Vqb@$sxC^)*grAH|l?8Q(eMGZg)~hYC zwB)ULlaVsmQ3VXw)AP4~(43?TNJs{C~v&RaUGty_vH^27kcKZfu zk*0qj4PASO%Q%2GHb2DaSJr;NYzOI<52E+}5^ZdF1`2L5Hn=!p479;3)wtJr_1mbj z#&hiI?sB6@#rWam7 z`i2)x=%=mcw3eP*8?QfzY4e`_`#c^glh1xq-*t+WP-b7{WyL1i)6ZRowh?$~0evG6 zHF>Hgv4_v~(O&AW+LG?URVtLXj$4E70lkdyJLj<`hxB529fXOYHaN71@%r!EI_jfr zP8`if9_41PK70vF;cmqy&-MSOn2$n4ZUH8BE!HIF#rBIl0i;_!e=>~P^ZL=@0O7JJ z@xlVBqvHJS5!r8nqNG6sfMKfiZ<*egL80JU#R~vZ!wCvTy!4Vh*0ixOG+B7A+osk%d^PJbIp&xR{MERB(I%e)Txsd$ze+dA#>&YeWZ%)NSH`Q`oXvD?i1_IO92vm>{s zL4}c8;Ohb5to+g}2TH+A0%xS8iGjbQ&KN-F?=EWXf)y8wvSjZOOluYIs z)<%IEiifR{%0ED&bWnib!#gbXOiEdd2=^7mDbl^XoZCClks&+oDFMSkP;7uJF0`gC zaBgT)`h)Y(f?pG;$ej&AA7Vg}WJcFQQijEfKf8050*jma;1ln{Jqv!=(y6DGbH1Wy zQlOvjVu$8I#?o1(??S}Dxb;5O%>Hou`gh*yRgnJFYTxIlU zfVY3ioBBrohi(7!PXE8bT(e(d+x=dTXud|zuYTu{>$tt*7Xv~Mh?<0g*-vD*12($V z0z@-Jasg)iMbXzMPkoLmKiT(-d@utG>6FjIX8EZ()lIy<+W_bSeDwLklk;8kToX%{ zZV~famsbi@aZX6>(4KVr`#5Zlb)E2I^T+%65VzX>CM{l4PkYa#Ot)*LzQaF|4^oQv z>#ij?dV-1K! zw68OLeW;#D1*&qr@4tH*aspUIZfZnkt`jP8u-txTx9 z`_FVo?*s>AEMybQ>16eXz)BxM$0GHx$+HoVG=%rWx8|?dlEbM@S|58E9y9XMIWM?J z9bH`k=}eOq@*Q};7OR72Eg*KV>jes&hK~aw&S@%UqK_%ik3^_dTGc z?vgmt=wWnkRp5#O_c(qG9uZD7UP8V)L1tVoN#qszR)SXplbYw^;#Owguhxgt83IN> z55@v8m|x3>Sho4Z{(|vmH1c(|)iEeT;O#h}dfSu|te^YpgKe>cG)mcsQ&_iJh16y&i)dTm&0rc6}zJ_Tt7w61xI z7V4JFN12z$)F6Mq)|^d-$lQ8E;JniweA4wPrcQ2jiWIsuEjqZNZ$gTyRU!#*$~_eG z1DFlAOO=cCoO&rz1;FzSe#qK?nu;NfZc81MMtJ2MDEd7fy&-CV85BoH_ zox{*eh}IHp-NXbnZs4q?J~-af|9KZl50b9@b{NN=nx;A zNA82p=Q{GgZJ`r(HSDRmIJn6@?PkIHVzX-eb%M-hpW>^6XBC^O9ZhO2_hPCt2!%gl zDPNL?=|>qKH$VI!e0bUn4I|zUjq3aR)BtacFtH?7#CC^wsO>z#G=JtVRMP8laFaC) z8;c9?EQR;u7Ufr|r*wPxY4R!429HxNY&4UCh%Qw|>)LOJEYW^7PH-s8wqam~n#dK# zx_;!)sRJsAH61nXrdPFS$=1Q;g@t+8rdyw;&%T)CBQENl706*&{7x~wo1&6_8T3qT z_JO4}!$#%71m^CN9!@`T*iJhr;s*on+IPgBf6SyaxxtMu%z?O|UP*ncS~)mrC{UXb znsgFPR=ocqR}^E3ZTVz=xE0)d*)=(}Y!m!42*Fi@{x5)=X}_#3b8+k%`se)~Nr$8c z$3spyy$g;cT&g=$D+&1HmSoT4Iteeulh^j6C+|#3tvLgvWtRXag5f78KVed(uQlwQ z+d|=YM#0xYt@gH16$=}7_uYr|%}gIyi^Z-*mAgLa%F!bJ^l3y}iDUC6VmgvJ>5B8> zH$OiE=od(4b{KOyU7Tc+^#6K5@e?B4g0Z18_;xXv&}x>2Xg#62qCOagN1a&SktEr(TA7$DvAgr#k?;zIHEur<&D!i<}H}AZOzSXgs~Q!W96534fS=f4lkZBK57laCv{&P|3aKtz1&}ZQ5Kx zWkCrIIXpUfdwFxtxFmKzycd->-ltrg`;5UWc9gBzDGXv&GiP7J{2eOZ$Ksst8zfvEabNF( zG5Gz-KQ?DjQSPdVR?CQUT9221@n%w14P2P1mSd}|ZM)s|<0i8(Xc>y}eUHE6Cila? z4OXWO;8qbJf*#W?bLOUSP{%1P^*p3q&hjQt<0wQ5P|tG;cb)U4kR=m?51c&m3$1zU z7@(db;|iz-7}v_DHWbVLwYSP0km8Ii^s?fI1p-uNLr+(KgNHe$Mw<;NrkxL%{d&*b zKIGE2c<&5CZRW*xN?Aa)9P`u25&ao>6vS)^=h6R$sBmgie0td_;{M4FqY~S&1gZ@_ zUs>4`@_2TCy?D|}5kB=oVg!e*HTXUXULpcIl$z|QIY$R#R!vL;6UxN&R_n|dcXwQ% z#&&|A-}2D*TB*y|wP*w)Yc>nNo^3I>;DT!Yh~+JSHaTA8Gv|fZaw~xUiPDCeBDVBILKBtaS<}gvseJ@D_ zlN|7HhJ=I?6!~<-o14qx7_j<3dXZPh9pr&SaW(~A3}zr=wR`9Ed5lqbjkZLE6(A2m zWmbd!Q;fZ=MLs5>bmy11tNM)y^mU(qwMroe=j}SOwv54a_5bs2?XmzGkEQmW>{Iwt zA{7uEd^hxTM67%d4ZSFjpM0yMbLo-DWiRM<8ysP}0tm&r%05SmnhlB>)b4LWtwqvb zasAH|)I&bf;k*5%LsRPffFX-PyhD%;zs282*QENUv;v;~q4R-|VQydT@8&%OxI>4=8 z^?PzU26MqjPk=&vuCE$j zWRB(i7P3jQ+pA?M%npo{@$8}6{(F`sC^;O(^8+A0HqFT6FU-^uMY9bLiBFe86;vP+lqkp< zn*XgPhAO~2Phk@95l{;_j+=}`7kR4&i26*L-am#{b?MTON5Jb;6#GO@{cJNaz4png zd1|?2biq(6rB^ClN_fC2L3=JsJGEYV-;E^BYlAtVe@PO18VW(rc+~!U?H_7cXheIi zy52JpAkkm`jclM&1)Xz=nZI4Jx)-_`X@vxtGU?joX!2)%0EEH@Z_TD>M5$exjK+THa5!Uj&w^&tKgn1SPWW`n{TC z0p~yP86`m}1+BJc5tzAQL01(eX14D#bl8Z1e2*Ak^|Km+j$ojXKgb?CNaS!S*! zpA!By<>}@kB4-LElmT6kUA56TlWCR7V4|@54A}*RO6@uY&=F6$c`3TvxKQeB@DCx= zPS;yfOj>4Vn=61{yI5$cLiL|e|jPR>8$Rfk7(=6 zw>sV(gmm^W`8< zyFKTt4#Y)1{&E>Angod0cFCRXCd|EmL%x980wL-jLR4;ju4PZDg(9U1N3sFpiTKy0|?+ovSCIqK(IWE+@?{>@i5W`YldNuU8}kb<~FFEiy2Zx zdejBjt#q|N$nDxZVP{;dJZmF`x~NnI`MadafilERgjP!GE!(fxApe9qf(KmamsoF- z_xSeKzp_JcbCXxv@jJ8K!yDJ3oA9ABXrzJTLQlNPaz#4Pp#QR5|MTHOMGJn&V1ly;>}~La*Uf)l^Z2Rn|tySm+r501e2HPL;m@a!h(Jz*ihA4i}dg z{brP@z~0}oe+d8roQwaRb$`!Xk?(nF5ozZXI<$@i(CA!#zK7!AkUaA#N^>IhZEEqP zCbS5pvUwdIqPq>^%tr3H74|K9@~@q&o5ag-fV3$eqFF1{Q~W%73CJu9l~&{0Q)^GL zXAVyIJoKYsweFho7Z~Dg!Qe_*=U|(AS zDSZp;x(9Nb0^+xDX~Q{j?ry2_o!4_b*v`$^weS4#Uw12#=*^T1OWg# zM6VY-i;162*SdLQWNVYsrfSxcxl<=gkz#vK`VGJ99){{V0NuLZF93IlM`Kc}^aoe1 z8tGI$dAt%QBlja`TE&VoNwwWrFs|Eq?wUz}Of!ft5Mv3t*O z?cnnJ0g3?KUjV^Z+1&{*_wr=i#o;#(zS)xr2oMbhXzLG(9S@cR?D{DFyncJE)L7#+ z;9&lZ0zZrzQ8jhSOt6u^0&1ACXQIWf`;9nh1Y47a0H^#W6!(8mG7vpG*oNp*&;f_2 zZdaaPNll^G+yE^qo+_{ZBSZx)fopM5=fvrmuD|EVd^v5m_kDbaLM&bj<^HvX<^j1f z+nMa>-T_nas=Z%9!y>I;D9$SK(4q08kUyZnsl?%kM&BJRfsDP9jRj)ScCG`p7_+2G z2bTg0gLNshEnUW?Q_^drp8X-`m*!Iiap&#EGCau*E>3pb_-4b87Eu094hS*vQ!n5> z(Kz*t!Pam*#%Osa8`OAd_0SOGxUsv6N8@*Yc$CbQLNWD)9~kvcO4E% zNqt&+@t+H+a|IH|g5A(;FMxs{;1}gpy`;Hnh`nP{bFkEyx>b;u zsvaZ)NA~XX1f$tzZ!r<~&CjeZh)dtGNh6%gl1Wk8@`2m0S#oa1pGcK_-8fziIb#D} z^qHv$(`jtR(|m4)Z{Hl_>{Fi4(y}TbxbNL{Fx(34zSCKTK)j2~$gvxDTfdt#D{k}8 zT7ZvL;&?G z4}7c|&%LLG_S@?r-h)4H0XBgT74==J`EN4kZzqDa<1Rq4+zTQsB#VV*O$e4_ZiPgKNSJQ_ z7UKgBYFwrb9X5Sk7Zj=z{MJtQE2mbHX}+)ajnjS_{wiR1&!fK;sHNy+F>tUZyR=L6 zS{@5=6SempfqQUZSJNXg?x#&O(BQM>tv{hIQ_W_z((_?Z61D$`Zl~hlC)^Y>7KHoD zAmo^P$Aw=<_TDtUd6>kJSCRQLRI~5Z>~Qh}Jy2U9SE_9-3#YFb7PygYlJIyB=b*-Z zF5qcHhDGh^4XG-L=UG%x?ok#l;$F>)-$I+$`4$01siY5J{v54)!F5~Fq=~9Es<7{pS(rXphMjle3>CJSCe?4< z)t`fkAfHeV?BXXFw0I#4TpZ3uL!l?yVP?+xrL-W}9vWLo-jNR6S=T3_zKm8HWp#pj zv9o`}OyGc+lPs80RuF$J#cxWKNekRtIjcFq}=B`J^YY9I^NyY`E%Iq&^u20 zCu0u=zq~es}9i`kU1v9pDa?4p31q>4wTlsb$TQkE!h@q;x~V+%4uJZ zW$gzQ@wF$kPm;k|IIg(pu&;A_7k4s7Xn|i4??7{A_Pp+Sh7J8zu>{aGHyV$WYI!28 z1S2w(cTv$(*>d{&qLvT!$`bRq4>$WlU^ujqC&xyok{8jgHNo0d*1fY}%jtNg)|kHy z+bMLz5arnl?c&&uEK9Cg^VMz`^0(Oa2QaUS3A>evrLA%M-^-$%Nvn4;7WKh; z!;!u*^v=e_-ztZVvFx>ltXKUS4i+YhM!-92ruV;LpewRpkO#z2P|3&5ab`1n+5NCR z9}w!UonHkG+uOO-B$hh~#%E%>g^Uep=(td_db@Pl05M0d<6-Vv&JlHVz)*O}jqU{H zr~%-NiIRgs7wI6w9rYJus6@X1WYj}A=sbQye**ZZCa%@A%1v7he^s2k{&eN_N-|a3 zJYppaT}E=mYmVqMzLvi&$E^_ZzB*vY$i)cH#&)L1xc)=!?va*`GwOwP3*F-W)9%iQ1by^u~d0 z-H=^?>$lznRsqZZh`y@v8C@;+WoUB_D|$)xe&vQ?29Y6(u2m7R?IS%?+Y|>MMVo|J zewd7Q`ovozOw4HdBRJnk>;_}0k>Kdls4GV5F{8L^Eg5hoN=I#GrpQ+MIB__{; z_8#{12-110sJm6^L-tBz|BgGCuz+GDl>Yye17GbmDeRh*dd+p{S&ViM$u!Svgewe~ zmD)}>AcWzwGm~$(U9e0Sbv?voVY3&bg;EG8sg1;9#8mkSh`+^$BoB+I)QXxPssiF6 zZ&lIpkUH_bwY%sGcC;F^(959?g@TO@DSH5L+CTh3gY->+Yw(fWPRYzZ`o{dhV6lC} zk2^Co%O8tq*uk6ukCtlxEUi->FA`%0n~P#I9Cfe148PPMCLLHvP>h#Z+b6B(dr>NN z|6Vcvtul>hTUrjqN6w@b>TTQ>N1FUBvHWmuMw4>Lp>+FalL5WDcf7=T%90G+bMpf* zD;L)$Oi`jpM8;h^6BQAox^rD25AN~DtE1dXc=mpA*0QOU(c?6l{=CeiE}c$NKeSdIw|AfPb;(lJyoakrVhHK0Q0lIsAx##QVD;o4wg1Z^V%8v2Zf%{$b z#RhnoH8N=Bv7eqK6rq>BklWr@9MrzzIx&G8cXQ~>i=E1qFZ53Tg!;3d{5)?Wu$fy= zN51ylv8;ks;kRs4SN0TH%l@t6t7{R{;x%R&-fdLd_Ss#BuI-0%?0ugHW)rKv^}Ecb zq`-d5*0zuDt*7#9NI{i^4c@pNw@}$1GrdC_V(kZ391!3hbE!D%JZvC?Hc>esT6<1o z!?XM5NJ0%TU1?{?37i@G6GWN7aIK04K2yWxNvLUR?&jjZ1weCGb?Epm_58hNqGthT zZeBZXy7pRr(@~x+a;Z8ruR~8OzKL!J2z58t5^M_AWzLIOv!N|^fIkW<)n0$ZfEq9S zl3k4o-et^lrMOdZs}g@aG%P+7(*-m?Xs!WJ6x)9y&M{|7ZtXik;US`iw{3*q4toR1`OO?v@w8Y)NJ2IF?Hd_``K*-2vV{qKHSfO~}>U#5b}OzEn69;3Z#1 z(7j`$HhNwL=>-M!FRFMt<3wRV6bt$O+_|W+ISiBP6}3T?^O zdHzL^+Lk46I_K)7-l;Xx`knWA{7$55`aW7Uwx7B%l-M7EdAy>@iPsc7Ab2!BP&Yl_ zvyjbLDTDQRO&XOp#ZC-aZhQs@v*;^<{p1p+S_uqD%-aq^+fV!`ir4C!U3fFqLEE_coUTu7 zB|;*xUw=qgIj+9HV|u^78IA1UIvulMb2g8Pm`~OIJlP~cfNegEl~)fcoEcNQxTH1bI9Ei;a%l(7XW7rYuan3K z;-hqJUFZj-AETq+t9Zz>9~;hZa9*A7ev0K3bmfpOs7^n8jIgsv3l?Y9_IH^HaEMU# zK>r6hzV|8eaaA69sEZeUNi>bk&asr`9mj0w@vOn(^*>v#VMzpN0jv0i{jlcLx}L!` zt-u^rpk2ziTQlhb>I*I;vhICZza2IQ<4br`&PJ+uw6`+?1;fkqnpcW~aQQM@;|#$l zUYlh}cfPJXOeeTjk+PJK-LSMeN{m0sy?{Ws(OG8B?fi!?qP6z3P3DQmPojUz5_v*QB5JoPUb@~Dt1aU?0?yWiDemBdcXD*B$x)vXJ4^}?0x?rANq z>K?C2qQ+8TLq%dPEr;24M)A(!S2Dc3MZI=Ay$^)mlA73%{rt#CBz2^I$23*+`Bm)Y z&wK5>;(sJV9G*>6;t9<;GYP3|`8oi|2{~+`%PjP9!i-b?0qD;QlnC)&> z{pL+k`>DQo;$(qHj;QtEAymx$G~CNA$|pGR&Ummg{a~HWfDOyh#&A0%aNeAN97xvU z&E6FjCdbEf>}M{^`mchzH67-{qWkXdhL$dgUS8psVAUp-2+jZw0!of-Al-y5e+?y4!7spfUc3mWUK$HFLu zfhG|I7^c!_hQ?iNBq7GTt2c?P-F`lp!isaCWlUvzByn&|FV|Kl{WKnm1-)|-Mb;Hm zd6KoDnCU@Gv2dj2KlS_n#{9^EH6mClh{rvny0%O=Fj4x)G#5xoK)h-Myc)qL$m~=M ze?Er7UdQq##?^;)f`4;ewo~7y8tGaLJ*t%qC2UIq~ZYj&25tQNZbLa?+? z=LC!|zB7A{i~O~w{;+6+-|>4V!+u@H{+AcceFnifKPOD982;q|RJo>j?QSOyEm`}X z1H5%Ly+~=irk8@*`$g(3@~bJBOWx@ll@aXM{Fr&L@ef5Bdwjw!M@FyUHeAJILUDT< z=UCDcvk16<2V9Z)neF`z_m9kpol)Trzz&OxzkSL>%dq6w;SxeNiV-_FjPii)$V@T z`uhCyW`g;qzS!vy;|8&E>ASDSlI=czywoA~`jQIz=B*~<3t!5Hzj*aX&$pWFv7b!> z|2}Y6I1_@6yFB*{d&!RC7^i%=;LxlQVURhjJo4Io!vQ`qzM=wny&G_#oT^gQJu=1b z1Y?39EqcQHdDIsaX7M?L=fQ~Hazu1-t~ENFN(DVN{jPayBnN1vRdLAe@bN#@E5u8) zv_3|VYC*_?@5U$$U|>mGyD?5sjOn#lT}z6(U-;Xf(L18pI=$yPVy8dVF+YhdwsJCA z`+S-Kd5#L@eTOBouA)9KSArj2kTR0IGX`6o=ueF&cfa8J=mfS+a5*GGUw92B-Of1s zs{xLw>VP??#_dDl^yh}DPI%QMJHO5!pD?bNAaYP8YyCIwM;ZB^JOjszg`&P#p>#a6 zZ_`celU`h9oumv!gsB%yYvSGit|!e#yk|mL)Ql_=`gskPixiA!u8tQO)BZ1X|EtZD z?ue`<{ZyPV!D-cCvi}yD0J(a@RR0OBeiRyV>*;02N?Xj|>Jx-ufNAFbhN=1;*qlZ_ z;D}<6=ojfTZ7%tpjKs$I zIT*jEa#zj4=u(hr1;9Eq_nx)P)~rp?fDoUzp`8S~5J$A5>#!89=b2PU;U_8Sw?tn* zQ{hq0Pu82C9FLS96wQef~Qpw5l>iJB`fzHJ;d%suqegT8us&rmjDe{*efjd7v zd~1Lan7hJ|^lT^%%A=VFju#Q}*g5|d(D(PKub}(Jd*+r1u$c>}q!p5LI>sb|PERfS z2J^0-sbOvz_}7HzKP9Y&1}q$pTRb)1)Fv186FS^)n zZc@j-TxyPBaD;OK%e1M;Zm%ZX)83r^p;c4@-@b`rjT1l@EM82p=AzaYvphT+&o%Ln z{6Z|gLb121)E>F4;F`7Cp*t=ZR>?qO;LZP)j$-Uu>Pf26-kH_{2Yx{P;jtSXtbXKI ziTu3zzK~<4XZIfgTw1eJTMbiZC+Qy0Z~`UBbZoA>pO~0DjF@c!vqj6-d;xzzO{c`J z?SFX@(?tt8upO!tstGwb?XF z57k=PB(BM56iv%ZZWQtORP|ds^~(4xAX>B)R(E#0y+6@eFL`s*t^D4(-dUL^dT`Y+ zpN1Se@X8zav4Pv+`p#vwdD?2;B=%cRUe~Z<^oZw>L4)qm?~`e-`msJmdF)F0`0fm~ z16W>c=_c-{eHG}Hdy0o&jk#T&ui~*J#B};S=Z7ke>%%;~t0msVM5AWj(lbFT8Rk4f z=_vxb(Gugz&m?#KS8~-Tv)8d`e(HEV9a^x-=HFIT~f2@?`!fb)Cw~8DILrJj zBv=N~c_>5zb8a+L;yhr|b!oC*{5)Ul`^tH7GPFyzGx{$nEr7cu79*6~u1ussw(0FKMnV;=64M{-qMM$GA*V zun5LaHeF2#T64eyj$%ylU32fPoLZcvr}z0^*wkzOkRUmb((|+U9H=r!5g?2dT2v<1 zE5)B+aWhA$#-?gsQ|hoUes^trhD|J3axwK7m0tdCNR%di^Qs=jl9B0kP}>T<4J>Y zc{E-H|6%~!7B+Aoc5rJtDCT0n;)TN=FyIv;1uMbrx-_PK@{5`8;`OH<4gEMvK?T?) zCYwN>2MMPfBdSV9#6Zi}beKhPh5d?*#_W-POX~TF=-G6|9pzK(9;gT7CiMRH{hYHm^mf1t&d_@Z$9x#yh!!Ej1f`{6A^^P`B~Otu3!|MEG}m_hNH zkh=qlW#0+iQw@Upb%|Wy6~?q8Iqfwneio^$fsgZr!l6h(F*i^ESUL z>G1Ba-%y>41dSscrtjx2;bz~1NcTU!bhX9;1+OkN2XM3dcx^xnuOb7NnA4^fG^7e1 z-EeC;!;zHZvL-8N8V_4~Ua{yi)RM#u@q6{cTEhkSIfc&rVGN_DZjV@cIL49;21^No zE?*+-fS0BftvW}m+RI0Mo~#-5+UJTNo+-``O{(Jmq{ynlkzG6r6*?`x;BIOBw z8Vp=6<31T4s%pI7ne>=s*w%=Pe|Z-JTy+_03uDs9y@4rR+^HY^#RZb47jj7I27v}k zkC|`pKcfim9bt1kQA3qMxT^+x(*}jxnTuw>A~4OgCBl+@7`y_Cq7D9pd2FoP=k~u> ztAejGPVe!3YxP>#rdaakz$X`AV_6_d&ExRQG=-`HMHCj5R=SwhRWBfF!g+c; zdH$q4h5e5)^F}^4h*w6!nwtDKyW5sLH5sGAx59n>QQl&= z3+Grei<2?{1&!k@^@jrc<~h06v7B*cooiPCe(x2(g)dh{w4L%aB0Nlc0qMttiB^9c zF0~X)&f}PaofLaMnoW}@bKB>#M-FQoduB-eh6k=L1^O(N6S00VWj4?1J6aes{N%Tk@|`a>>wyt{)Od_em))AYz_k{<2NrUNNTo{j3&Ww%PWI5R6lB}O z3;8s4ZPF1~b5`tdjs@pg638bxl}KkiL;+sXcAl&}wtNP?Tp*TS8B;-FWh1Wc{5FVU zK1}htIikKjV7J-XbsOe<+S+&!jP+%F1^$HX^Uo)m*p`vw$VP-9MVI{kyq#WVp&giU z9qj!>QPoiv_hlqcu-N`HZFWFwwa#@Wb=-Z&GDi!=9~@9dB1O_gF$eXD`1a1*$=Y)l zT-@xjU8u{5Y{g}l5A72(c6%7NS(cT{rRUn4$a&*7bYPxG_rL!Jp{dJW+wSHsb%UBN zemBwrM!5%_z#$a3*C-0t6f`Dx_#v3mXc0N5h+v7@Rw`f@01S8~qD%4+IjC_qRWen| zOysXYk=K)AbcxJ&uA;gj7d-~sEpmsXEC5Q?N)CH1QxD#M1D||@fVIELa~1;!{4uOJ z*lSk%$)F7*syi@~WCeJ`Hv9U1M-JQC_|aw3(x6BDzFzG?Ex((S^qe9~HO{HGlo_f@a(hl=*G6{YMFhYQu>; z3>-V0oCD2pL=m~Ij!+5dt|Og~y*$CN5#?9ICC*iDNX}HzjuC6VxIhlSG#;imBr#N| zEg8637K4O-@Uco7D^fp8$&9|zUVg3 z7aPDvH1HJ=B{LQCP6%0`u*{w%2d#yi3C9{_G1&xnQo&eP@t=BM4J+THqs|vvyh@(jI4bZhF4e z3mSG$Bei(nNS^uTBAq~3`f5}AArU&MqdwG1#@i((QgNHe#(-r-%FI@uo#M@}QdYtW zuxx5F=!g`Eeey)@r$J@#UcXHN#Me8LI(h|3dOZv4O zIuKpicvz1jEnl+Md!V0_Knb`0T@io?ev=gWxN{5vm6(NaLFgM}vOL1Se2tj?GOTei^}-J%z2v zMNc>fgqjQ*LM)u3^go9^Bvv$gshNVEX<*AZIype>s~`LL#KN%*3Fn0v+3hoLonIAy zGhpA)GAQVx#=FzcfwbbO$v%m?kpZFR1c9q;z#YAkmD4YZa`25E=Yfp(v}i@^$8&0R%AFy1QqOxc zVP=RHM>7$ZSrsTW9|2QCrtNgE_J!{mye+WPGMt&>f_n8oMu6RydcFoSTzOq8=5aig zl^+8yya;Iwb{WpNSRXdF+WDBkig4$WU9fxgM>t37cq3CzQ)dgJX z2W;J-KO{sX%;kYUrF4I_8jzO4B-o<+*!k)jdw1{iX_TwJU;(LnZPD=aWl;E=4v{m# z2D@)v*B!&mQoSe*@|hwlTU!Od=x6!LsJ%aN2)}6a2ePLsg1_29pK8CI+?P6#r-1d@ zrxpKqGC-p;LhO|OBSS-od~oVx=sruTOja>7r@HQBS$duHZ=4kX8uR*=S}7#PGs2IWunAWNAEQ&FzS&jhGRvjh{?IlgOD3f4XEz3?AAYn z^~=GNVCV9lZWqjZ zkvIZFxKNhJWiL=A!&QVO#xost#%H5FVyif2Q(x>@>^F4R?d24zfYDIA;493xVTLg+ zzXKRd5 z6((H?A=K#Xu$#d_A0(>xrLT*t$M;d+LXOk0-_O>zazINgWHdx%4sg(16SRg$gz5th z)TwIhHiU$^wksdLRsUL|Rvz*(SjqiMw;W~dQSe^2WOTe#fG$%jYRttezTpVq=46bq zY0ojGmtlkgVtF3;+0dptpL$J4Oxo`C5>Q%PazeJ0w36r%Lpp^{=I5Q5LjZw`Eei!?L-YcrfuQ7m+3k2uLU@Dk>noB%vu-ASwz-Z;B9_bO@m;ARSTZ1RJQ7 z(0dY)E>(I5=@1|gNJ0_fB!z%;~s;9jKNW!tb09k&3UaVe0ga7e*4GLF*fzg zVQ2%>2$L?>jBqwxZOcdG*)-o8kIAIT7WXUjG;3OjZ9QpjXs_aV=N(8f(8<#V=7GCJ zx&*DQ=OrmKB01j((?8WhFRzF6%?)X2o;IiGE%?bxKx zb7HDh5RLGLkJ1=oL~#G#9E+jclgYyET$g#M#sn#Ncmf-YpdB=tyFi8u^9nxfSK7E< zD_zW{6h5WhswP+AXNQk|=Xyo~D2Z|`hHhW0Cy32y09TbAC!(zxrA&;^2Q6U@{ zETwTOWbJH5^6|-5H!I_50YT0oh6@awAuoB|I5QZ&Ge9HxDi931ry@9+EFwemwkmk9 z@Ff>Av!(>edtCGz?3PMPn2dsY4|x1IpZNe)%7%m@kh9^sqsxvLY~!_ zz|!H24!1(o{bJ}K=h5ER)TZ=4&-rqs82fb-5TnNN)0*HS=U3p-Yeo^F4itRP+R@8X zXFkgUn}jqGlqnc=Y`F4RM>#-{ggdVo-kHB$JPar9(@_ZFx*A($7qG7VJR?;~Xo@3I zIh*{o^88Z5c54g_PTW+3LgdzHySX=&s&J7qQ5rIXrf_X@UxH)F`OY{}oaIrK%K&iO z@zIb&OO8@q)gKeC7mX%|zj5cN&z-AqX5wZ>2+BhnZjG%5Qy&$@4txgHZmM4mF(y=K zQJzgXOWz_%7>YDPrfre5M*MexunXEQ+|_S3S|VxO^ykt+@lno<%|$Z)!<6nd?Thij8VipC;h*fOS8|-Ar8IHVYXAI zysnIZCMMfv76B(q$rvrN!j#WZ8W@>8!5$W9 z#Lo9xInpFY^#yk`aI7HRkH(;~I{fjal8?P~KP%c=?odAF*j4jyUiAXu{@mC@XQGY| zmr4UeI%n^7TB$Nq^78X0c1-NmvZRuR?Oij9Io8%1)c%EMiP>=^W7LzFX)>OJD^CIde?t7jw9l9E8rvSuzn%(!BD)lwxBw$NUUy z>DLnm%NWR)rmg{Or#4Xx#UU2FFP9C+4_+CEnRi057M{c+<%#|7+CSn?&3uS0Hcbbf zYHQ9(TZi1smAE#VlkJ?t6V>WXPzzcZ$l4Docm*%z>44GJ8kA34Y^(+Mm814a^9Rd~ zK!BGr*2!os-dMBWO{?JpN$%BNh^#P` z3D3C!bb7K6erY_1bFDibMH@f1Q>?akT&}f2)91Cr^jJZ&xb>bDsVfMa>(lf?6ZhN! zz6q@f_KA^;?;}L>CLHg85C00Cw1%BH=UPbQ@MeYzfFDq<{0e&YuYTapP9Pq_^J{U| zZy~z>eL$}40-xB$%zr)1rhu{ZfFEpPLFOC%im{0B_?c(1`vpK--(MMzAqbfd#_S1b z?0!NiL~uW3e8_af7VB_w>YBVNOuG~2@5W6>-7rG(BPC*=?-FITe_4*(Mwm^)Uu%@+ zGz*6Q4r1b(gnV}xzXBT+t^37b&how<)WVaZKPXE5QCLrAf7xYu6Ae8xA zBj@ND>s+97@tk^U3UBRw;h~d|MQ}daR(F(hn?KC-je<>ta8b<6v!*-g0P=(Fl`Xhe zU);%N{1)>)CR&q_?1_NI)o8NvyDf&e`{~fcr^!9FT2bLIXl9>zG0U9mf2JWojY1Z~ zO@8b&ICvU6y6@Q$@nY*|`BYpFOj>OZB|Mx)+oaFUz==s8l?;*JI;)e!`OP4~Ikbj@ zu;I81Y6IU%xB!x8&CzvwTv9h>s%pQXdefxSI(zjAjEsXIVVkWK4Jx6ke7*r@!cF5k z+1$e(KIz221bkgfJNAKN4-Qa2?1Os0 z3qFcEpJIwAz((Fin*W~7IT(k8Sm>c%B!6VLYz2{9z6FtPg;Mq%=PWl*ewfvat*_ms zRu4|g(U3@HPQr%3%im3Ck_P5L>BPmDT=~s}l#t18JM1xF$!mmadpKMA<7CnHC|<7F zv%_=KHry2dKft8JbJqshmq1#ST>$M1tJM~u6vEIPLjir~y_~Vjn32y*lNtO4T4&!~ zXbp|B-DS*oVpnP7OJK0@zA)%i)j``4uo4Ji5oVCx@}Fcsdtej7sFj=Dxz)Z=J&~t$ zDm2v$xx#0t{I|&Mxc}R2$Vm_@;8_~opD5oZ;W79&m}RItJB+|E&)~$gTu>UJRl5ek zRPw8;-JWqn%`;};%Z%ZlI#v%Sw@ZF`Z2ls_DA|>*wiF)Uwsd~6eb3UsL|3`=Z+(^z z#<<@HZ>GA~_9J~QxU4Y+Hnz@1Bq4c!oqG}BRcesP=FLvG_NFMe)gOAh=_n=*=s?iD z3`rc;QxO2;?E3C8`GawGGrii3R)|%80cJlpYPppwTf>RT$}ZrFZa5_Y0G31p35nKG zsNi%(qs9Rz#pJPZX!MM2%NOzr@5LAH&CkgGwppZ`AQxyUt!)enK6ppk+OG%nJ}+FG z;3ASj(6-G zwg+MggS;Py1)r+OjpIT5lIw}DFpwLlSJw(vm3#_4pQ+1z+5&dSS?1Bu!&D;?W z;4g7eH28IfHOkr|L&}BknZ%>;n}@8Dhh-<6mtXQQXp1crg`Y3Karn0{w(q!2c95YF zeJTa+&%nA6Va`Thpd6;KBaKD#rces2_bqjLc>85kbG&nxiYe8 zdDg5P?F<;X)$(Cu7`ti6EWu6BPz~Pp-+XUM&XnL4e$69raZ0B$Ty~Hnj1aAsfaiSB zT?k!ec%pqPU(%rACpN+IWJEfTNevz^$8kmCoA8ht$CbMN2CkWgO$#I_!pgyI}|04Zz!$ek=MKFw>6fFiLNKYrQwz_A) z;{KQWMd#+TU(EJQ>0$HK50t{@^bS61Jg}au!!SM|dq7w_3*s#{Sb)1ZcM92`dI+KP9lkAO?u2Fy_zK;JKA z7%H`}g6i0R1d6=pUFQb0K{$3mFlNs!DyMKDt6Qw(6*opE$HWkSCoUe2Ygay&4md(G zna^c7^924n!uNw1B(+((ZcFd6{O%HZzc3Zj#~v18HpX0>0W~5zFxs&!FxI14Gf^)Bw>`DYNT2FC`1*%vj}Ek+@%?HAx=IFmO4fBSr_}CnpCTUgb3HC35DiBqy!t-pBSm!i}@L z785&)_u#AjCQ10a!)BydI1fyri%vKo5`NBX1NZzPQ)>rG+F2s_=dGBA=LpE)eArRI zzL^Ri6Ot=$>^s*-&*@J7vsqNTyfnc9V1Y}YZT zl#@(N^6tzxeoOM6ZU5HldbeS7?lV=g>hH|sg>3w+ z)jRnrz97Yhx+^7lmb^dCV0m8Rfsg-mxHx(3>NLTH`(MC@JXWQkXrP3ehchhqxTbH%nw+{q{?zLtv|06t%6K>A!godwKC`m+9Uom8Mpo~#q=~i( z4(&4k-DGUUEpN7aT~XY-?b!2Q#&smZxptY1gaW-$5ZQ8;F$QUisCn(w#Vi9KruX4z z@rZ{4mgqvT{9OgqkgM*kkn4$aAw~-htG`PNkKh=V&u1`E$-kxKRPTVG-~7b{l3$|T78^NY4O<6-ENiH5rl1Xat1li z&(3*}wYZ(MB~2DtWsNBqpnK17lsHgga6Kuhorj_Fa^jS%z_qMb#WXr zL?>=FI+{X|E-MBJlj@OuZG2m64FGBnUHm4#1F|3bBpMz6%!J+bdoa8akIKtZk9{T< zqB`-zq@nSSD*ELoHUTX+o2=^SeHhZhGACqk4vm1_v`uR7>989YZycZxeM6=U!mpLD z=fZ8OeD!8pL-5Q&(1KD=WyRh35~`Cl#lOsxr$e3ldB0z ziA(@9UMuKM)D6ug><+(MoQZK&HIkeM3z`bL(7RgQG}G!DaA7YKMF zE%y*Immc2S_-^PuRL(7rbY-GMkX&BKRGu802oE8wfN zwu|%+iT}eFW!|QWb1v!_N3}3m1A+WFG-cyz%oy)tZgqcuDiLo9v1r_yrh6Q!xC^p1 z{BmuSNCI{3xUJNutwlm{tVgEcZkMSV((oEP#?NJcI-u!byl~7L-e1XPT~wI0t>U24 zIW=ZIxa`-D+8Tv7XqAvpRrOw;cs4-x=|%u3zk(G_(U*HKW0dZ-jtaT>-;l3H(CM~) z7|NZUB3?PF8MWST20Diw_O37(Lqv`9AJq^H!Wbz=VF!NRxl8|PI1l}<@HH{`%KK|C zfvB8XdyFV#-M*yOOS{lAgu6k0IPbRT>N8J3#I-PN1Dh9v;4b&gJ%jEXfe^lYA^r0F z97fgNYG-3c)-gsgmM9i5tK{yyyo{Z_&VzAk{s)QIZkkNUmstqTNvBp%JMOxx9IZkB zfVJeoH#G}MGSi%XER7H3pZDJl!w!qHFg?s{e$$YC|5c_lJYO)(AfJ7OkU6(pVG_SlXaeSB>*e;X2c^#>~DUcO9;@qZ!ad?XEVBzj@4%2;B z?Jz*9%7$FEOc|^e504gC^z^~v!nM~5^5$@kV|i@e^Q2Vut&yoIHusk#r=k}{0IMc8 z>j1UiWV|WBwiB8ZSaM0VGhe3qV>)2G7?8hVC6RLp`)x8tiF*h?m7@(RuQT{FCEk}L zr?6~mG(8Oom%zXYr51x}D3EX8!0>luLk-E6;TyJnQM{lwfVThK9V-GMb5bWnQ8BehWPE+xEza37JdLJcjk?Dj^*F>Uj@y6S+>Tg|HeJtamXb?f_LaO zqK7`~NaFoAk)Q#uJc#}uznebyoQv0?X8N50d1_AT1u;M6|#tq5kj2*T@6SX+OZViqqo zitlm=pPLi6v>JncnRcY=$rx?-Z_Qp~V3S9ZvY>=lE{qjWCM4a(n$Kv?AaoXqrDmu< zb7bZNWmWpjPBEJ2=@Jh{OMp6%rWTQws~dM^)~s-{nlQZRdTgGKNFTpmSxWOzx1V}X zHDEk|ma7=~WN3J1tnLQr#CoTH$a~;<9~>K1MU?A|xS;9#J|MJ3%k^YslzH^pf>Ws9 z_Lk8rDr3fj-8b{JyQZ!kI~!M@65oK9i<;Vgo5KB@Wgc#9Y&rP(kq_~V6Y$rF@Wq5CU~Lh4+T z?USPOYqFL!t9kmisSNJPa+915Q^}QwCgmmPrU{L zonRmBv1sw6q9AfC#`N(c8cg?HpnltLb|)FHEQ8MHn3lGn`)k4AF`D})r7YmqkO%3- zae3iJW#09mZBe}ob3HP-ZL(;ynj^NZ6c{@9uDInty%V*eSK48<$w9^UK_K7^jz>puwrp6E`FL@1m z9=Ik?N58o|t3bZ{y%&UQmRKgVMTFAea%wX;43wmtOfO*?*u69PyB# z*@wV?l(XDpg7#R;ZF`*sw0ILZ;LCD6^`cvCP)uDv(r0WpZ+32Jb;E~{iU^rWJ3x^g zP6s+ne@8Y6Rn{;5IO$z#9b{IziYC8C{T5gU;)j1oHq3-0Z!Si8Tjl7zmZAj=L zWc*-~R<4N822q*^P3U5#ANBQRx}A@`COuv;xA)6N?|cL8{9@MM2I^Z!>afj6&BV>8 z!1O+GAGr1DBazkJZ!z3{-#)bX<3{P0GP(8bAI6h@Rp80Dj;qstVd=Myj|NUcF0{~< zsl}N4vU~gg43&rdSH0-+rB%0a^Z&?~|5}BR#DBznk%<14e{F@(lVq~)nueW2)$vbm%Ztq9|3m)G&}awLpY-e&iw+g-5%I( zgfO?dxr{dVlkzc?T-XuqLrzVOVil;2<>64;MhINmur25~LWu2QD+BneP8pj#Q=slV zLkX*?)hh@?dx%QaT{Eb1o^M68T4dk4l09-&<9D{L%EEi??{w%S&T$*ydy29!4jvzR zL!*0XPvh6S%|}mhNC|JF5T$W+P-f1+$d{Tt^q=0$UC#932FggpoZ@ejwEnDC?o5 z4N_BK7Ot@Hw10AO-mC$*551pOSv^t=3i{@9yz#Out@SqvM40qkYtKLl+sqm~c;A>+ z3aUREGJT*a)>VUg0eBl<>YdUvbCSLhG0)-O@!$4g_1vybec?^(uy@x}{<^T{zvjil zAM)C9#O*flA2CXBFfp7ySgtMPT?(+1RCA!z`Y#wjw6g|H-aUF zA__EfhHy4b5?;g3 z7dHc>`qZVl(=2-14dW?gsNd|E9p}E1B(4&pU zng$bG&T?rkp=A7jf1~DRZEH_#|2@u~CngX!Y$5gN9{#DaS-3aKg5O29Y7X=8`;AN0eOG(~}(w}HJKE=nB z|0HT`mXR-1l*>Gb+2ZAG8Ib{*d(W{GW=z2-yHC)axIb@e{{&ZUU46Vr5b%2EK2=Gh zbNfdK>KVp?3co7F)6#Gp3HsuRK}ws!qYd>9(#fSU_~{^=%`mXE_Id;3hiAr z%D5lu@V#)f*Uuk)uRIw46vbeuG^_KeYV&2-kCKeOS%R|}_a@`o5WRrzRv5I`=4B~Y z@h)rx=l7;TLP@ED#MG?cuu*t%OEf0*0LUiZVl{cGEK}Ye@$oo0BAyrF==yd1L?a`tgAw2(@z!uboFlq_) zv{|#+b|iYMGU*Wl^C$Bxsoj)HT=CU4IXPQ=CaKi!4y^`?={$v3UT+iB?n)ouK1b0k z+@bu2ma8??R5i>N&$rk%+5{srhO2zqgiZL{IfP~nqP=5~OPW^p&B6gvzUSS;LUuNiSF2*_Q}rW}5% znID(vjV%IB?(E)I$&Zm>(Qb!j4fW(Ng=}V!s(4KKkv?pVoN+9Gr{MjUHm3!CaWr#M zit=jLM8`hE_&AKnA4p7YrAsXLd(%`k>#tu~|9NgA8nigSZQ0Y5&r_BV+&kCRzehoa zC#PhPwy>Hw@@HP{Z%xeF#Vw%}kL{|(FOP8Dk_;Q&JKbm>O~X-g65(f^zqyR4!FbeC zlVB%|rUZKp@>cC6AKa6LIqrNT9RHL)v6HA)oJUj5%y^&l99-q0UAj{fk-gd#UjiER zr2Gc$H{=wJZaoSb2=2+oj4k;9=+OR?;}4$Avpk&L*5Z_}YE3*dOPcT;Pt0|@?IHaf zL~bMTJT+)O8~Om_@;3+BPD3Cam2=6JCbFothA&5mo#zhrma~Y3s>N#6_WJqzc}E3E z^LD!tFo}-3V0^auGS+7AOkig0{m@RE2W~Ds|b?Bmu{dhbJjo%%gXQ{Oe>~b3Llh9sE{9OH3xeQio-ug!(eN#B%X5zuX&5 z=W?xgTftz@7E1!cW77%DIbA*ez1{{sqaFVs%@JT{}P=v*CJ5p<@KtAfx|u=22Df>x3ytf45p$)Mo}pC zg^J`p{xF)u+?KP(2Twv?%@!7WFa_%eS->b2)p_RWkhJ+|Qt+2xP1@X?FMJ2G?~ia3 zAx8UgD>PNyAaQp5(WL2M=r7G>Md@TK1#5Dt`gqe0c~#R4QJQ5790|=q`9XbIN-N(yHMY}tl6l=4zh zY=bPS=wO2-_2T0r1~UL6U6myhSIi!y67 zMmXH~%b3N3MOHT|^La#`^)$(pG3%?x1nEfQ@9IG|@4<*0O7NnndKegTLJ(i?TId`o zvY&M$vzq-`92FxAwMMeqPV_geT(6bMxCLxih3R_^)@S^bu@wlGl?x&{)esAUyWGbr z&^#&E(_J-m9*e~4C60;L9jBR#QwW@AD@v&`8N+AA2%EBcJ-I;4*r!zGl0QHR&7i;= zH;F{~J_dUo`?;P!ULK?CQ&h%>t852-)H$ox8R8?pTYp_r+UabAiOB}Ji4N) z8mEW|jR&hGqVpRZ27daHpUSpQA<+)Qgz#Gy=2SWky`x2eB@`| ztz^L}-u1;!au8%xu48L}+PjA7`$AVXM~f{#;-7fdix8fW$p2_IAvK+ca3)^xv-vF3 zBBhCwrsuYRAZweFOaAajwq`Q4TiD~13X&XVZfg=gPg1&6;TL2W@6Z$X{)MojyvsA) zvuSS)quHFA@dDI*+52_BfaPgy+#{ zIa7c2#|?7u>u3XCYB%Dj+l*E}OkWl3BlMPKA>E3@ila%6f43M<&GV#|L-#`<;6cgCs=;}M0Y`V0mP&-$N#`>)ga?l4okd?BwTctX5tvDiFvIY{; z7(cy4GlETk+Ruk zl>T~ZIXM=v&uzN#EA@M~@z_LvVISN(WG`NyKq2U`=x{^`e6|Ry88inLmYu8*OF;2Kole;ay>**@)`&s9Ax_Ob{lJ7H6VO;ia=4Qbsr_Q! zbD;t}r1KG`wS3K3|M4;az5X7T*Qv)XeZlBmO32?0aW9Ozs;Mnv_P*=G3oDNkpw_+o zm`g&u_K>66T?DBxG-54o$$sO<_~?_@y#`o{fGE8lM%#xv4#~EJl4uEDda@ zFCWK$TMeGBTp(ck`lG`dMvJlpqm)rA38=Ts^DDeEzXrBWti$3>tO*$*@PZS^mEi!i zeN_kHt6$EaOB9{cWk48clNZ$9vNP9tpz0RPhEww>kE<7+h=$UO#htREutP^ht&+RN zX;IEsfrw{?2dn(0RMB}l3e~0YWLxlPS%y?AF!5b7^t>zcniOW+%gR=W>E8+9*A~pk zUcOpeKldU;KSePzTG{Oo!kWFOg3q;#dRTErW2n~NQyRpiElOQ2jOAx!{3KN|bpBD1 za@s@5WDX_=?_B+n6an2gWPEP$fwY9g`XImT1%1JbXCmE%-bxMqzUbP_!GDD*($R^h zJcs|GMbvk`@+i5sgw0jqN3V?(?p!&ubpF_FIgN+TiO*sv2MqU~4>`REHEMl{ubd?Zqv%6;9lO6`LTfbrJ)Ca5X<5?U}pqi5pta0T< zHC?(6n$iHH*;uWvOUnuJCdY2^$`yS-8&|nuq>ozrCKnDaKZD%`j+o^V48F=_%=7@z zzRRD8PTK z{kgxbQ+YCO>nAR9R@BJ?V7N6X$&M38%zDzMvklOI~i{Cf~^iY!Mv zor*nc!I~@7Vl^|h|2`l8{foK8a&vLolM@Zyd(&*d$0$ma)sYlVAjxRD4@qlF;3ZBM zFLObM0?-0WHQZsMBu&Z92noBJ?-(>SH=b+muzWR24|O`#aIu1=fRfD4-zReqesZzS1D#)9=ri||IFfO5fh^KNGg9YP_66qg`k|ISFH+|&I(0+D zNL^1*=X*7XsaM(-(~^&3PWWki7wnn;GG|H+^@S)tqNIKOld*K;^PNz8_ZzxgU;!Cn z1N5h;a4BA=&vG7XUH4JnJSij6sh{{BSQ&iJb`oAnefFz;&y?2w0AQNr)+f#p#i2V3 za1fAzx7|?qV{}R_^n!Rghd42Q2$rv1v=R4Bi z9DX?9_2K?m%|Fa?F)hSgoWNyu zC|QeFs0G`qWN(Z?SNp#@FWY?KR(+Mc)^|HplX+XDk*|(^0>v3c7i!y9HFU#`f<(7c z@AV%{e@zZ77r0P!@+2c0i)|qLU-!wTh-OneCT{;M+e==LDhp?kDp0KWb?^HX_GM1> zWydqhQZ`?E{DU3rOh67^Im@S@eCPNSIc_m~N2qP&a(<2qT#oMLFaM_2KQ`!W$^AR5 z<9kxOcgQRmhgJ=LJjL z56acqIjh=Lqh$AsVu$KOl8Vue%ZJgtLuaPnD!j^ji#EtV0(kDb`W2Ci6ZJV^QkT(f z=TbY>vUdkbZa<^uo>2#AKL#4-?JbTbmDrfcPX;Dq-}A@%GN zmWaYhAhvLa^;K(lt|H32lt*(X`(^^!7B{?eUfF+6G122<8{VrJv8)C^nT=U!hK22R0CH*hRvE2NBoH@~dB?1<;zp|pc!2G6J2)+BpEQ~mCJZWY#esXg! zJ6C0uWJLS4ryAxV4R)h%CfQ3a<8xUQ-1}KuQca%k2y0^!J=~Jul;D2AG@MzgK*1t3 z6Jo=^hPOp(N5qCp@H0Am-3tu<7-b$udh{wsLPsC{toy2-yUhJM!L*R}@-*j~9GOxF zyG|{pJIhxbz033Thd91;2~($2RkEGik~sKcc{wh5S-e_dHeF6QaGVhd-n}@Gs#H+g z#I~_0Uj~m4bJ(^Jf)t0|`c`MsWk^wox%8c>X~sOz)D;mp7{dVV;N_)S2$H-y@SKq{+1 zD}?T7e%knZoT-u<6R*~a-9jVNi&K*|e&(uDw*BTPmRc*-0Dy0!>WYm2#fq`=Leq`_ zAT#e{x5=%{Et_C^1xPL|1yL4-l3@qpA4@i3oA1X(-LEERj>Fwg4GhTgQ`~_fyy!#Q%$sHvHh(+AZdc< zz-{FcQbWL`kb&ow=gyn;#uMoSZNGDx4rphI((s=jvKnIiD>l1<&hDhF;{%z55=)-} z_Swa=LJwxxJ92R5VxFOuZDCv~$4&zqVFFts^_A9p1pLGi8L-`kyPAt^W4H4(4V z7Q6GvZ8Mo3e!0ie-{>aT#OKHc$+$8uSWtz@_PRymVkWl&N@rw3@YjdFtM8n8GX{x9N^JG`FY*Y2o*UM+aK0A`=8wCC#q7w)#s+n;=y zkf)*EkKhOP958FaDVfijyzPA!HTM4Om&te|f;-oTNn4lTGU7D@Uyo+PI3oAOjps3$ zH-{`=T6JwNxdR&-`?1oudSq(m zBY|#2NHtAM3c_-(jYI$Shq6DvI5^TCo>hNN$&Y!`vXhx?$Q8?h%DvxY&vyr}ZOmD; zS_m)S19q#H{!PB9C#5T9#;(Ikj-1V{sx!(-4INsOdl#)XU?@{mijCFiXWwuD$5^85 zOR>3EDrgqI?H_oUVtDZnDY-FMZS9kK9jt+h-Aak^k363{zw?qpeP^e+p8K(SI^g)N zFbCCPjLYq86lhzd#`$7q>#sc^v~1k1^(9g4hO*3Tz~be`xdJ3+R7;mXn?JeVFjZjK z@V&jr@$7V^b0cX%WB>DTB@^Rs&m8HyPsbhRnCH=L-^2QPvQc;K?hU*e8>v1<=-+fS zS(h#sM4|=f6C|ZRY3vL%Y+vyw7h3MvhBSk%(Z6gG8v}|R)AgNzrFm#enQ>a^bb(fn z)Jvd?IH)1L5ci`pj}#Iu6N2ptr~##BPibbY`Am(2E5oodpazoQ=g}`zyn|&}Alowg z&Hf7|G0wt8AJi#bwq3#Uxxgu40P*^gD%HFEBX_9jgCZHAvOr;V(VXMs>ATPuyk)30 z&2zQ;JI`BPgYTSjze{>KaL-)mq&F8d8@f}K7e;fn(i#maIw8@z!S#voqdg33%0j78 zPIzO|e_Ph+7c`>PfoB1owTK8KAWra|Ht<&5$#P$!cc3H(U-pWjV5}x~wSk+!A>v*t zEi_nJ?=gEZu`GR7h1iFr@Vlm1rkrjIm(X_GDa6GxG>_k4y3_Wd>zdn)lud>JFOxn? zwdJxj3+JofdJp(5{B=j~=&MOf>NhONI3$Yo`?f0PtUvHhs7sMe30jcOa}WgU_FWu( zJ>2$KjO7l8cvu?hhCfzX;&#*Tu&-zRQ-D%G%>j?D{kk&|KdkBl!1DvrBvj$X;6)Q@Z__Bp)pciu zw~Q}HZUbP;s{z%I*s%rcCcnV*-mz9H<6X1#WK-FXk1N|Clg8iyEf*v53zM+#F;-0* zMi&|pgaQ%jpnwUX3+E9+UW_tA880*-r~CU#&fHmD-`?{C6?)YF*8H4izEN7U1|Q22 z(_&vkAaaTAPPsGPB9+cOy$O{9c`?Lc)ocRda2-L2q4c!EA0hBy!f>Eg+J{zhZhpf5SlW4eYm~zvqxPHX_X@V@?N}+hRm)S)*^RLB6IjKd_)LkO7J! zO*#up{U?F^zYa=wK?;pBGqer8$pT{FkU+s#6%j!R|8ck3qT&x2-? z4BzF{&U^Jzy0N0=cJI`CA#~}a>!OLit|70wE1|rzj23)rkw+>t5*fQxrb;8VgVE#t%y6vE&^I2wj53VC$DTg7 z;eKg!1mSRs*RS^DIy15T0}|M=eT<(Tj29yQs8CMf@RlVb)Brky*eAk3SU;t2Cu0q+1@Urz2mAfm+inf@4XGEEK}d?yP{ zkcnEc0lcjRX(`n6zO4p5G6`*9C)TQaR7WRTZT4(xY}XotrIW^lCeNCoK#rL~+XJ~V z)KRsZ&8Yqr$71vtIPcYt?ZC$!Ipqh=6l~L6DQ*OHG_=7X4R+p4tX#8hxUdG7Q>j?x zO+$IssKc+n(8&7$#H>_0R<-BzK#i$B!L4mp;G#CH1rGT**Y5Vn`8eh!Y*bC(~z$<3@iT-f;zO5I%e9Z;<-M2>~1 z3|!Ro2%>+3}})4&JHJ*d{&kdR)-{2I3xICLkJI7X8=l7A)EXQ!4q-Pixq}bK7s-K$QRa#s-%n_KtJU>?0msy z-B=H0_hYhhnVxtS$tEnJIU&DY$0Y!`<^mc1ERiD2OQH zG49(sQfIYn3K_Qqb{?uV&i{IReKwu$HHZ58}%hcbtNKx+4qcxQBS_2|ua(|JAq$__V?iWm_l!5=rc_sX$Bn?Af|4T}fGi}z0| z9vx%Q?ZWD5Ml`WI^5?jitPX!;O^r2eEK!FCl_qY4D}z|1Q~jE?fysP z|Buigyb>lznOOsoINJZ=VRuAV(h{_zjZ^_XA+1)>r zQhOD^X*c^CLGexDbXzcZm#Od-tHvoo>N4PGE!G*?Eu5>61H3X$<$BDDH}ag*y{>)7 zaERYhKZ7GOmVYemgO%gyNeQNV!bXv;393Ns8E9L{oi3RWNggAsjVG3BKlx|Z40%F& zEZvJdLoZ05DF-{6a+&Y>M7sblv8mz9&qU!(jt|=S>hGX~neWESEz!fhvNs#&Uc5(8 z)PFri7eBIWAk0p6ZuBp00a3mm_eaLrx6%yoOM*ft-OuJ6veh8g>Zx~+9OT}mXSCI) z$*YQyb4f#SX28#$#jiY(pD%CWFFd+uBt7*NvCZ+=&CfG?|8Cm^+|)YJ5BGKaQD6H{ zoGia zGj?Lr1l>PpamABdAq^^@m6kgd52pM}t1o$WZ$qt(m{8($~H1>u|3cp12|5E5|>}?ihsaDP{3YRt(?$zj#d?Y(? zPNf~@kN7IfMiN8i1?)arduu{)j<30iwNdI-TBG5lTx34C`b15`TddMGLYM(A)xV0R$9Wr`bX@V0lJ#+D#eyEPO zwv?q$m%sX6A1{?`r<%QJ&iabUMKZEO(~{?6cIW7Dt$FHwyQ(u9c~yC_`toN2C|&%{ zQt4f1ymZ-8d+u$;K9rVG*a?(V-W46(P4HEL@37>!bB8(Tc}XD-KL)JzftXHPlkYTy zx8A3A2ft1kMBzwk3!1eX&;~Y0jl{u)&(G;(#_l_(#>dP^XD~tMvE4H~~p`4Dy z$sftNI0cWO2lJT-&thorBO{pJd#j21nCH+`Pa=Ab&HwQzb7+VIC2V=6elto=19}b$}Oj(=`pvY zxG}dWU=9=qJ=+U66AQ=3`#;xJ@2qlZb_DQH6szyRrSG@G zj2yOhAGaZh4IYKP`WDmja2_D>=lT-pYT`dcUT5GMJ7Jm8>((?X-0SAQ{zt4WZ2+)w z^3tw%H4rarxYGD4Z$5GRP0n!4BCji`K6Pcn8bmAM4fVmm%!{9iYzk<~XKebK%;T>_ z&`~=c_i1j1V$gnq&III9dXs7o@vtcLq$OX>b=)odJt3JpxU-Utt43P zSSXV_q1PP3S4fy|U}S{ahpUy2uk`8x{;^)D_L(KGEDSa2*2KDl_&pHp5AoLxzp9bp z|B@`uNi91u{$j!(i0*7959 z^|nG_41Z-s!wIc_xd%#Yvnw4SH)t2Tl8r1<{%Si{R2Gs`K6`3MJ2+Me#=P(Lpkhjz z>NYD|^UxVw>lxKL`flCJQNLHL=5jN+`!puM0M%EAb=kP-r;tW{^+?o_Z#`?qoz=Wh z)A0$+&oX^aZSPgGphZWaR5yeO^!)c1=yJ^4GPTmAi~kUtfv=mE6>5u%n}hMW+_RXq z`w|9soi?)!Y;Mq>g5vvO&P>h59>pzMrVq}0xan}O6_b)#^z2mWzJJm2H#|GCRjsMx z%O;VP_m?*^t~+N3KP$342=FG-2zgP%?$yB*lSxD1M($L{?_h)BES4>B0gW*xi?VoM zA_x|{<+Zv+#bXqP4{`o*1Gj#SpC~tYY-{Rkc>cFXCX?}ZLK|Pg;6rtupU&x5w!q&W z*RpDB-2EHW`yo$hV(;qxLcZbW`GU+LB5f1>B=_E4-YK`K`{ns2$+UOnp*uTo|KAu! zcF!K{2>(B&hika?C0;fldUQlde|r2?cPK_%HFnlIOb1b<)kOcvI4DIl{uc0- z`dTb4XUx*&tLFuX#)p!vE*sMvqtFSX69VSar22`91oh|Vr{bbde_-?-JLuK?)&H$| zm8#i(_D7W}psnhjmj_j?_FKiyOH0$N{L-1G)g^uiz$J~eslqAD*{tnwZla+3 z?I-d&&OvKRhXFf{*dp;yb?F;BUe+t|cLFG=Z^DyoM0t;2R(1!iDKVmbUUYjb1Tpnn zp8u<<96b>*d9p1F>g5|UtjAmEy}(=;2`BqI7+$u|f1=;fHdKRI|m*bQGWp3fK|G@4euY&TvxiTM=(;4SvW z2AF-daj+^We(l4Gx3V!}CUD8nV;Ku?Jy!Ce~Nesp!EV0Q5cJ>}?8J^!B&?;9M} z+D4YAWSs1myInxBv|fH_QdiAKqyo%-eby3VIq*ER7`yN2mS(ED^=Pf4+ibG!kWIPL zw+ta#{jzq)wE3j%`Zb;O=uBGZmjib+ruVGx)x6dovuXa#c_O4+4~P`Rm;r3$;(+Mo z=v@1*I>E4Y={5{dv?tdwnJlfo_+dAbr@ha`&{l!=T{q?B4|bbce8e@#gtxa3?XXKe z!?nj~N8?g#=(^82Qf%6cYgsUavYA;7d8l^smOq^NUAB-8S^oq-g6BeaI-P~6BL#7H z=1!l8YF6W!$<>RQ1lp-+&8|h7U2vvv?sASur6~L1Z1dCW`6e?~Dk(48CzaO|OM$r{ z3#-J6BEKia2NpB4_Wf?aTQporwTgIaN4>`{FhQEovIgsut9+6!fTu&6g=wov$~mZ& z+f=1SjzwMfSKuM%tv7WOQJdW^I5k6SVXJQ92Hv4-dMcsJ^I;bA%p|WfyO#u-Psq9d z5Izy>0rVli+o6y471g#PoA4cyrQ;}x?H|4~3GgQrc6M^^_~fho~M?YaTR=uu26gmQpp9pLT9*w6D1^s&F*Gli-?!F|4=O2<-6{X6;r@O1}FbZ}i zh7ek~g~C|%3i$9tL+XLf>)RK9{j|JfIc$H_zo>^&>F}{WRIXeu7WeL0Y8w;s6 zTC|oIGS{9#tfY{|>c7XQSe0CP_#o)LSbOb8%e4pT`TJ2evR?0m5@yy*HZ@13a_Y)a zbxnWMn=QYuJibEZmkfGMZ5QaXfFA0spR6OH{}rP;RDx?CZV_O#R+cE zl8Dtqy7epmm0IwklUc!^R$MW*K%6@e(_0vo7iDMxi*T)z{}lH2JBuj#%Y$!)3eoes zv^x}9yWku~+l?pF9hXnnSxZbeyoL@9>=su)YJChD^%Gg|IAjGJbrietogKM03u$Gl zZTBGCM@94F3rXJ#qbSu8b9)Qv$hiv_w#oC7RaYaid}-?AF++9IaH9PKr##ktimX9T z!%ivJ77u+3u8jXU-=C>GxwA)PQ^QD9)cC&BRCdgSjYXFEHJv0I9hCi#R5Yo}Rd9Qf zqoiy|B}V1LL$Ax{@;tP?sh*+wU4nYv=s?t}z+#;LIW4g1Fclkd*1(_M_$>Rc9{ zw;fTYZ_5=nHg7-e5jFO%hAp0yct_o2yw_du5N>EwtITf7)u~olamdmyWz8|;oX!6- zd2NYLaVGrzf2`A~*qT9;q}0C+Y;Ple4^T|)_s*N>G=JVwN)C#P> ze}vwA+f(C{-YUJ;nv%1gUQg*gS2fUQ=CN%(8-H2Dsrl#|O}Y1^iM9dPneOB3G0B`I z-DRwU$zAKZgyWJsZKr1dA9_$VM9Vl{sFuqbehR?SkK09SZtMefPL;_}=Fm5>8NR(y zDD>BtGVYsyY&nh3Wi87MujxS_-24z!rnBlE@9NZF?zi*Yw5o}P=~LTT92|ThL1;;N zyJx3+u~7f|$3Zv$Vq>?ATWJ;c?Ya(kmlw7VjYl*g)I15ip`fKZzMp_hsVr0yc+~U* zF#?95c)qv57?f1a*A18D&OjjXajSJNpR;6+90XYkNupYC> zns>W|y6mz-H5yd(Kv3NF$KMvVUn|^o1Vb8KrQH4r*9_h2yezU2E(*>UPXE>$2`i4&pm^ymMadtnlbqW3t@wqe#tzU+Nf^#pjANcWw4FwtOA2r<)$6H376N3#~!l!Ahk70WU^n_Ib`lqz3iyo7r@{Nv`ck+(a0!&>F(u;xk zlkR6}H`|z{{H`MBpn2d=aMb~D%88$vtyWLBKP?fTEf${Zym%aJ+4E{na`Ck83l2@r zf3`VYYO{|X4c$V>98&IsdQpj)w>pFJ0{nYQo7dhBqsXJSyXV%3pU(X7IO z=`4O3ufSbCeSzPt|5{O3Wh%PmGBfObIcCyW7YAr=1(b0`C%yatd5B3J80NH$i2QY= zX&lV23CQlYJzI0X28LU@8 z5^rxD$j;D0_owakB0We>nQ?3Gs8e`QbECtIYcA^;NltgZbbm)$dafhT%+x9NVp{l` zI_0Rj^davAU!O)3%*#J#m91iJl;z$$Y-CX&GX(w5deCkq2{g{gKZvNj)tBJ)Dm}&g zN85p{5Q>dQ%8L@kSa5ax=JWo4baP@IM+pZ3K^opEHaEJ4uNLH)zyL28bVto^@~rE5QgF~6+4AuziGX*HN) zdkS`ik4=brJ+IkZ8s|QuxCfxIFIO^jERF@FYZ+Mi8F;ld*(3MuLu_bl%BlDlfaL>f z25o)|De%QH6PY@i0Q;QC<^}5$qg{p5VW5NR_)%Na6umvI+Ru+U z&i*YdwHtTMKj5wA@>j@0g!_px9v!`*WSwFdV5^W0t2aYEDoc6R$8 z0th3SAboaGLvWlpQ(r)gy#bOdZ@z*~i1>=wcgmn4>E1Bj(dy>f)yMP^M0BR~*$%^a z6hC8DsVUw@p{Vi!JEq7~&fE_i7vFDEh8v|vMJdla<1T}29|il4-$||l1pQ@Fk2zkP z-6mf%Jc!21MQq167N#-i*w>G9J*P8)A}7u9GGZy)Gf-6}1>v-AXY6KOXi|3EC;wL6 zV3yH`2Ge4;HqCdu()<-);so3*=!{(XekXjZW2$y{oYr3q9`YZ1VFol`&{5Ow9deKnj-mOhrxk76~J;YJIM4$d+fs&(qjM``}Hlr88pwVW-}OcMQw*yqmtF zMYEg#r!^%X-osem`kyt?P(-)LQA{gzu+?gMbXab$#&40{Z*M4NL_d!N9h5g7ooAtx zw*p!f`)>UnHzD^`8`zc8Sk-YXW#6w4t4+8$S{iXLSLcv+v+^mwos2Dsw4yvsCVFU` zePjE^>}Jx6ZQ0eaDS+*Pg1AFgi5;h7;#56G7;s+J2U)+5M-m(%A)odO6H1 z)nD68v(A+bQpXfiaEZEdSGKLJikzlhcZ-bBy@uYh_)W~)ZvC>Uk219HDP7u+?C;v; zw{+DS3*P*t*;8?@b8~$@eMwcU*V6rrCw;bbuvqSXdyhEdUkk9@pIfK^crn?15cn-u z07Kj^`B{hGhRE996h{(VLequXGQOGgsHw9De6=f6P^-15ofFr2e?|UVq|~0;pRlNl z;Jv7v&a7*(FCEqJufzL60EUogK=tJnwV3xP^1KVit!$@s>{|v_$Z-|IaOn!f^dEFg zsai)a*_-n^c$yu625$U3X1B9gq3e{szJHKg`X`&}Lv{B-J?u2%w*Jh`m6VJ#3295Y zg=HP+7)bq-VA+8e&roT^ys-u-Hydr_KzUW^sTDTZa0miet$TVasy+J97J}YSQfg zKWX+Eny#_=^I1oci zgMc%AD~-;BbCRvxK=TcYwojGb|KbVNWh?c~HOq@?M!A7au=5QR+sMm|>N*0*%khY4 zl~xE;m?wn1{<9k9!4vjjhF`Eto<+Px*Iqb*f%v^nN|~`Z?>^b;vv3<-dIllvF3ijI zceuVzw72wt2LBIcUDdFDC~7#!CM#D3glWlJssHkI;sJh! z{9RXCoe?+0s^2)1qLfIF(53a+Z~JVrcfmXCLL{u1Y?j6B45Zu3vac-iynpwL&J<5S z^ZnXvqz#xpud97{Ct5#PmRN)*Jt|h}$eL_hSn~jtwH8J=kwOYYn}ji^@K%yi>gKEA z>e08Ihz`4zl0sL-0=j~}cC+|jdqO4efTybiWougTwe#;{3ZL&{m|%pKIH zZmN!CzkZ*Fyo-ey9fF{^PdCti;u-CgqTi)Z{2r&Ee7NPK$@b742vs6=ihr`MtK>I7 zbGe%s(gDbYUtseR(}f?(7f--O(5BXR@6V$vjFwgnRz5vSJ7VxsZ&J|m@HMgV$Ai97 zkNLzjv7Tii>EuRY3gz9jVQ1h7vZ<~UhF-5L9Kj5(l|Wmgll?7B7#pEwFAfz2b`3kC zr!`QhGK-t|O;?2-;5NS%(wfm!<7Nw=l{nzCvv8o-#Bs={Zv@}9s1i+R^8S-GdHzJoV^_T zNOT&pcPDp+pR4#ukX^bWWUOFPa_rbIUbLW?m?xSfP2nxM+U1$@V|Bd1NrCF`G03+BmjM=N1(9jgpXwzLi%!_CUgoC}JTdF06CYOD2C%NZmchcNjD!K@kE4by~f z4Zs%FMtaWif>($gel9rZg4Ar%s=>`F4w_O4dG#qfkPl~^GLI%r18*_}NmlQf3#Yam zEq49WthwMpe9gDH6GcU96}9j4B3{w?nQolk!gTiA`(h=`=c~2HAsec58sVsrq0JHy zd9o_CX*$%SkDQ}AcU@D?ob>WO(1STS;G=ch8BJNe?wyGW(UCDMEb4;1P0CK7nz=+h zdOc;9{rP(!oKQD5o@0rg93&&mqEkL`ZFyHguE-NUNP-L>%GCv(qkqec!VHfICI%2H zA}%Kr^5D3|dD^^@>c&2qDeXdkWh(R4d((N~u)u5WQ4ajU?Mo-K@}pL3&8LgXKSRy> z<&PmmeX@WV0@Q zwElUtQepOx(rNl%CC!hT(?{(iY=e6KuM5EWNbA|%&5f^}+Q0t^S0fCMjw!z;-COz4 zyyjLzwjWSy*W1Jcpbl%{kh+a*7MtVU1hkh0~$)$_^ zBEN&Eh~cT-2;r^#jOZ`#Ta4!!U1vN#V)jd0TL04kTi0Beygo0QK%bZ#Y4p?I*IXlw z9>O_+VnsyEI1%`{@n?r^@)&eCRidg^mw!F}^<_)ctCk%ApsD&H(-LYTN+=;Cl==I9 zon!b#W-SEJhz0<;9~Hqz4^gCLIXgvwJKpjaHNjS4SMI78lcv-6Ju_Y*yD+(4Q%e#x zV>XXVow~eH$vPhsXSLv{_b^JITG8zpbYG!a=Z;$ zup#4*QRU()$;3`|8Ap|Z~Tgw_>IF?9ka0on4EA$hJ_kd2mh=f%ys# zxwN~k6uDC{Bg&v|5++Le%M}XU5U@N+>skXOTNjCCUB)PqjJOH&(v>lrB_X+3D{6&F z+APs*1THEL9GSz)Qg>GBP?ct(!~U#XYEnl=E^6(L1p3M96wKX*m3@p#IGp=`tj_AM zHy(x}hs&y?FDOS}xaw-EbzX%4$m%gyu=ciOOI@c@!!#>-HVRnN8DX5@v(S8xni-M+79b|2 zSX9<27_u|0*n0c4oNG3I&O5sw!ZT~SSYFuSxFiej3O9}_e)s)k6!4-XTwR%!#%l!h zHJpHh+`uz_*RyB+(X_Um;BIK8>WSQi^wP!f9M3LkcY)O}hT(Q#Qb=p~X~IigRfDwa zl_3Gmm27>$+y@5(Mlr75Uv~owUp$Qp*>>gp4wBFX1dHUsu0FGzVUeA`19KH-@|^z4 zxv3ReIq|BFf4sPpz99NnW|4)Yl^CHAl!X^9nzvZ?=+pX^SXIrL0XebtMk z2>s=&$8oU}e+nR)vd)o}_f%Y4s`MY}kto7@pYZ_zgg>K?yn6hVZDZ9(2hH>^Y+NEJ zkx77%Fl z%f3UwC+zqtry$X9rw@(1Z@CChS>F7vgqO3_)b6XRFi$8=|27`FcA2O~rxet=g|U}5 zl;e|UuLCKJ-yZZSIM$@vs<;GDIl~Sdt;6=ya+7q~(j2q<9J5Zp`t_aym7Xz+xn}Q% z9R#NR*vPr5N1o*dstkPd$S9)2tK(E}P)Tfas85E_ zEBBokePH#wtNGkr+&v4V?^uqAm>##~xo`&N;xEfn^at2TKFjk0iJ# zqHHUN?KYYrg+BFV_~MbVJUS^&{MpgdFwAEv=)_3-iqmq3lUmA&VoX^1%>PnXzdWmZ z7R(Vp$yR3ZjC0RpXWedon;{=;VtR=QS^W=``5W0$jhUolDCJpe4M- zXf{imrA8wqX^;*V8Cr?&ha%|WdOTWA#*rFnYDO*NDjY(lY|Q!`iLbqQP;RcK4D6%= zFFk~u3F&Qer!QucAC(80J0|uztI*ya_beN=N8OoRh_ihb=FDHj`gF!2E_rRYTI!%^d6{3fKe3;yc$D-mq%3z4JhNuhY z=Y}FhtrepsT83i(_pyAgr}2jXxSJ1}{K*V7(wxPu_|UIU>i(m%?eEdoDbr6V&3w1| z5P8l5F6e>VqPP0g{shBDex^AM(&Mllvp;I~yx4~oXP+1?dMqi6TThd!`$XLNrlPlEgPnrkN3Kk7ZE9Il{JiA2fh8VqW{ zZ+HF7VzPT1U{BMX&v^3Gc9&neRra6e(t|?LJf>t}0#W%c zCQ8zllT4D>X0TjCk_cJe>_gzkGb1eyaImZq+}7~QZkp&vv6yQjDlX0l_YiNE3l~2O znvixrl<%^_$>e9y9WsfxpfCIeY1*NX#o<`N`20`jVm(L!fs2j;p3DY9#5bElx?!@s&NaMX(ZbAa}`O=L57X`#4s+Y09JIxSL_OnGVA@7Uwx=0}USzh@znk5C z%uuRigF+HJY>l!axPD~P61(V(TNo}b7hf-|H(U;{iAq2(h68wgdM~0tS<4n#j8mL? zmf56iCm>Xysjxa$6HpV1GHv zqi|&`vm1!i0(*4D9iij1w%PBsTBU*0g4?`w49emTT!k8(`1?bT+2`JwJKpJ4EJw!N zgrJ?og{x|=n&!^E3*gX6)_!Ng>J70Yj#XxBmq}%3R@NV>0@vQ2MAjMI1e8rdz}ZxP z6YJzzYNYn-PKtN6C-vo{AUDXX@@kK)LkX*TVMy+pUPDMhGiT9Mm!%N49vgkh7Rc_) z19`x4G1q;(?qi2vkV8hZBCB;s?79NCBhI(U5b)gHNpdU^c`1NuHUpRKPs3fqJItz% zv@4}J8ia{YQkRj<96h$u_k9sL3fMqNK#^LV+qGaqf|dADrho6sFciaqg>#FP z=8I4nf3vhhmMU#-6YZh9;mY7aBV&WJ`qbnK^Wr~|G}2{W+ANuI`oDh=z#sc)67oI3f%t1+ zo(jLUlTSaeJ0b8qxDU2t{GFW=r#F~pA1QC9o+XKQ7=?@ph^*7Idr`2UlX&@O5D_|? ze&GO;4V(EOy4CtITt`87?XArvIpo7jYZ+_(vd$c>Zb6egQ^*f6Y z=RZmu9wF-`rMBPclE79MmUz%gQz#XFnk;vYMvO3kEYl7I=f>giyy*CdL~sD^VNl zffuq&SZ&OoN{nx2QHBMm`*^=X#z)AjQoi8!VOjEwg}7`WpwCF|di(W}^-DB#b6*N! zw|l$q44`v^Be?x7OAR__8CuxXN!{$rVN=kq;N()T)V?ueXS5-1ZR>nGs6=BU(kyDg zJ<wR#mBL)iT7kwPMDwX_7ss8gLr|MHx9{Imtth&z#$);A*JvID@QJgNwK4Mx42NL9AaoXeWSeoHk__ z;>1h=vuUSXv;ZNQ?>4o%ATWP)|(jHVk zm364+o_^ICY8uOmmnMSf>mZxr)VGX9IEyDcw=N1l;GBnea{*7r9*eca3^821vdSTS zU1=Nh=Y8SGe%s)L3sy|dJo}%!;Of-PA6+;>7Zk&`lls_t{qdz*UZ2o~2;D6#_rwH6 zx8Mo#I&K_$fdDX?PM_Joud@0Ef?r#~$L^8GQdvD=5s@4@oEPhjh&{FabcrrgYB0vTiC668B#eEz%Rl4`l3RgNz-sy z3~KdDuv6pnm2op?WI6HN>8U}vq#Z5?v=#klvs2Y*Uz2OM-QmT#G(fs|;Dp_ST&@SN zg*oZ}^W|h`%?ZEh%YMr)8V=B$Rymb>xEHPHTzjf~r9Q1B2ly6I$vyv1!@{5A zl}6Jq|HNAA@)(n-XAb{>=dP*@dv&puP%{?UhnKnZ;D2d!cf0`_b@NrEx9JgB8aWP5 zCiKR5-s3Fm3KQ=YKpP-Gps!~is^Yz9gp#^dX=vECkI&ZA2Z`dvHQaB@)y_C0Py=%k zF`U3CRnxQX^0$}=TY4x|Zo%#kKhn(&5S}|gZCD3=fi?|@y}QGyC%3vPWK zXc1?r>ktWC$9Fqu)b+}TObtMZw@s4d_B|D|Uw0Ypk}3iQ3U`@-^Xol0V~cO_QK7F!=c3!cCRG>by1vg# zk26YrgG_AZ>#?A8i7`5N!zF-JH~nuz<4<`b)3(pqtugv*wG1SkcG3a_uF|lf z_GujG4r#M1Ti{Hb3O&kX^p3m3N9{*twVJ&S9TL6L@W5(Kwcsm+Cd7zRFZgc_!5aP< z

    ?8T)^rNH-zZ`kt-_Je376?jD;-EL7>! zT{pt&&|y*Df2bs}hF}<4x2#&ODsbq;Z)$<6>ylZ9UUPpu5l{fny|Bl#R z2(2g1&f`S$JaFh^5dtAD*!c<&e0BRgvi;w?f`>$=H0hn}0KBzDy!%fgfe70uf?yUd z7u&#WZsA86v!ReT@4Cf{;ll4xk1g20u@^u2eh$G=yVOKJ{PvykH$~s##pNIDMdC5A zP#-FAGBy(yF?WT!8I5L}CL*?&H%tuNS1`}`;p8A5SP?0%u-eT%_Epa>Ts9EWU~x`- z+=aqN8+F;6gZgOx3TdZYEWp*R^rG^H8@7Cz(VV>)ZUzPac|+zn_pHDy^duBlyTKzV%xyg@Q2 zvtM7W2Z)pAZE<`t3nzR%SEv|5T2{sr&vZihkQ(80KrO@ptzIP@#jTEbVy4Q)GxNJU z0S0s8aQ`)ig6a3V@4Cr$y^o3qC3F+6o|%A`6z|}Z`QKZtdF>mY@s(Iceaz&pi=-%F zx6a(zUvUrg&wV>{-{>rIUa@gYij@CzEe~}Qb+6mr)`$GVtkDKF@{b3QR=Q}(|3#lJlP_V)O|YxJ{UDpL0wHl`&{=-#*Qs!t%p zT9gTYQ~$FGEBtbx@)uC>v+-LOMgsy3*7T)Be&=I96(acRHX3mix(*`BD)BK(6K|Ut zQHA=^qyOx9)pJxyd%~0CKkh&l;Y7>NF3~a~8>=JpbBA%W(vh(~oYCa|)&f(_)g|rR zBf4gikF!tf`M-O!kKQ}#L|Hh0q7_!n-|I#NZShD?j)r|Mc71jF@T`;Ud!E-C+pR3_ zw68Y=4{V;*ah`qOZN%-XTqmyx+PZhDSF-d$C^_%#1nEA zP-#oRFOg?gyg9iA1kj9@`mzyo=O0j64V}zVj<t5Z*%R|Tld{`X6~6Nv@E|ws)pyu(3x;Hj1AGM`p#jT zgJDX-CJ!xs^<%xuK|v6ok3)H=a_q-m*}70frb22|KKfP9L`HNm)-fSqDOngRo)Xzf zEdW^1>FlpTuRZ=_!mT};o~3(f1NXeW(FSOsb*pMTA|cpaUAsno+ATO-uX zSLZ~Rnzq7umBPPtJA88Z)6$BS@u|0i2MvPJgxBug_FO-B$6ZgkM+5h}qFQSr+EVR# zl*d^|Mmdq*Inj$ZZX-xNCLVT-|Bd@Q$~7`R^PF{%gIK3?B`;et_C`W&bH~ zY2K1n$M%7>elpJkZmg*UIEoe{Pi3%ABUu0Jig#a&SuU5H_f@97VGPWP)z%gQy6@>% zpTVce)7QmP*!h*H54TSgNap9}3_>wwwF|uAQ;y}93)_USpQX+Eg=sWVv8}_b@{}z5 zh=Z6X-oTS2zF#{QN~nvF$6M~AL3}A@Bo`xIm-QEGh$e|+Ka>FNg|JY>n<5koB%)Xh z(CwJN@!*p6mm*98*X)vFL%0ntIi4l{WGtljkBGm`tPiG2T^c9Mwrm<-VkZkMz|AAm zI`ou0<@DxrF)oGk-SPedxTH?d2l~!mU-JG&)@(%HZy3;g@`VBs;?n>PFTdEuFp=%Xzp*!*~)%<9h5~q>4Qi1?I z_OPye2k}wum0EkIic(;1vMSwMUtuHj(>zA}TjONm?E$&@X%qw0PiI0%v2KpxKL-FE zR)wLAtuJFH`pne}^V4R8a!sHZkGWpo9GX}v%VoAZUBfA{2B3}(>_4yqtdgHagJWSeC&;^( zS~}r}-)O5r=TNcF9k(qgb!g77&9h$*e9V&N0sB5n9qvn+o+0slSmYHKBs>aRIqYug zR35qV@=P(U|C6)0nY~o|S?VWPcQ|#~vrrFzy!VloJNs=$m%;j`y`m&r?8dZw$a_gz z6;FivPxnEaL3)jp|7yCR2?eA5iU^wr43Nr%qfv{G-wWx}nn!9&FRRBmD_w(`?C#Eka&*VQsQ-s^=z zzsYYsW9$FGmrlugMD*tvZ)evvNSwj(d{N@x+(lnJDe79}Y0Wme$l_;{^TFxQbhQtpL*p0kGUwAY&3)<1Y4z2ETuBx`_0%r*d0%sSuTp zhm+gBKG2NG7tASyk;(P1%q-|<3pXt&L7Xb(n;vP_VQwqku|XV3N!f9{mgPWiSy1H5 z9PUD@Sm0t!Y&;e2OEe8+-xfwv0)-d5az^(0f41hk5;{j*k=|={pI`x?oL;~($O_cS zFOOv*F9XfJKT}IRHcTMktdJ6%Vp(HvOH1#9c3{HSmK-*ylEs}DgIEW^3e zaPV1kxi5?l+aZSl?P=mAS@Dlz(&jmEHR<@0Tl2DLd}h>^$H@B1i20j)caqrCWU5M< znNijfC9Ni_9~hipUkQLwg=g-eB;)OJJx(NE#ZB7I^04A<`Ni-kS=<5MX)|G8!LB>_ zzE(bQSZ7a<_1!n`JlA1%x)~gb@{U>%nyuWo4O=mLt^!=Q2R~8iBEptukVI3cx1mRS zv4z=E*#K}2(QGog6kX2nKk;P9L&5aMe97pD)0 zMUs}rw{-VjI7_vtAUC5{lkKZZBBVVu>oBsE##G*p>MWJ@8S$bd%VPL%)-%`2L~Yu8 zG@{VT^Y~*h)2u9G+0t-a?ES6J$S_w8%9ChG6y<7{yMe@I(${vy9bIrpyq<^CNwxX2 zx!RE!K`6bsUS8u!+}>1SKhPsXdH}$!haK1%wyUnOOsIyd4oTN~5UXU$L!WbPJBeU1fj=LP1kXMfia3m zE#hV-%vpv;_ROg(IH$k0AT`sebAW9;Z-d9cZMRMJ_2*adz-s`RBy`JJWq7V+Pxurv zaYDq(3VdLJ7md6}4?cPwS$EqD$-0wV;JBRhMA88Jfmm$?>CSneZ~!K}84KFw{|zy* zcukmd?|VJji}&2PSb=>nsBjh(CvgOocf`Kq6UVUZci~9>g4$zr*@!H6pG7{SvdnTF z^Oq$?87c&kK{=JS<)A-DjQR*U(Bg-VmUAZa1Qmx(uloU6&}6{|aHi7l{l!e<(&NR| z{>Py3hJqQNrO(#7Vn_Uuh+~R5!4?ftt%;PyJTPhixfmup2+PWzitMu-wxX=3b8QNJ z(g!v)B&YDtXD5Yxu=ewjwXCyx3nPdcC;)7WHc7v@yPS=DI<0#HMk+iT0=YZg(hl zJGo9Gl~Av57K1~7T3QdFv>h=cn5>k+za)%m<89vQ_ToiGW&A2pEa5A6#oL!IjU4g^ z>8#2>e=z$c7!YbMM7W4b`H$s?3dm_kPj+l)Gqj{fdQR@a>YMi`r*!`kx2OBpj-l-@8YU<&PWJ7)S0ldd zzaVG%?MO`rA`l9;aW*VmxB~0q{#s~rMB=%<0Tp;1KXH9-?Vi6R*mF5dXAU^i)0F{Q z{O0qP{TT^wJ&r^M>?vHV!7n)yhjEf~qhv=g7uJZm$~9Ablmb zdwKEGdJ%5>6FAEE^~X`ywq+4Lcq-LCXjUI%2JT#cV%;o$rL6N=;_-*&BM4$(0k~bv-^T2yxOA?=!;-X8N^-Td&NQi8XS*7S8$ zQi|HoHz9M{2jp*3bnnD0e=$OSIyd59p?d)CO5CXd0qY-=#(dL`F}2~h1!RRaV@6g= zJDLXBWYeg;%~spQ;PYFkL{;v8J+RKAD3cEkt0ruPY-~rVzXCVBPPg ztSqmswp>}7Q@jgnZMtUax_K8Mn6A9(QM@O++gp66sNk6!ney7=IcfL4*oz8a}q2|=5>@o{Dv z4R_tI!2I>@JAB~Rk+&MB`LrZn;OGmpeaA|0(_UNOa|e@59^KB>mUm-Tvpf=P@ilPm zHrpZk=->L%ClhwzOg>HhIv*G?>zepTkL+ggLG-(hy|)Q}-}YmW2c6Avg-wStp4N=X zQjWq7=UcD;wkz&|O`XWJL2$~{7D5>MZra30%ahiBSL3&~-k>@5lYiUy=+A-J)6(gD z;7PH@ED+8~zWZ?DcID8C3tt}nr2NaP{d$P;)~5*AZ&nMJv~!DuqbOL?IvsES#!7Au zY@ilz%${56I&%m}^+|urD+EAGYP8BHqFTdJn!V-LE-Mm5YUD$z65-+!<()TQ`e4&t z_yxVUd!(1R7zR$H+|m4ngvvOhCl7_Mqq-4 z%Y@!iN6|z`g1oJn7x+qTBVQ zq|ZM0(Q&x#bdA@h=3J8MW7%>@aVqjfFUc#dX|)pn5Wcc|;CVtQH#U6%H@{?_UfYR> zfI*51o_&Wce8*wV^uNHAwd;;;Rn(uO>h@oM(By}jguKseq)W-u9e@5f*Wl-AHgE^w zc)>Qv%{iuDtAqX7O8?t;s8H1YT{(s=-w)T+dgZM@9s1-S`;e05tJgVxI7RMMeN4}$ zx4Un+36}C`Auy^%$Fw%Q95q}jjvw}yCwY)iy{d(c=s?w@3_z~xj-F{qp>$$#M|r`= z@2>P%2S6k$eXR~)KL>qqo71A5X#*rxaMc8*ISwS4u?I?P{h0Ltk>(e2HmdHXM4mTE)p~9hWnA8!l+qN+&)VpUpK%f+KR&D^$Gy^j zpA_U{S?7R=H32fw%Rhn`RC<9G4jSfMOIz+Eq`-0*5Y})r3R})!jdyzJr(X1Y{}0S9 zHRO}hAWWF6)=N}@>LSX<&b}O7kIcP+N7Q<%3H!Qdx3_;Jx3V{5zbttiY3Y{y-#LTP zmX3@f#$yB_kA3UL3BUAB={m8c2fwy|pcquFvQD3xR3w<0fJ5yb4Oj!u1oFcC(!=Gb#my-UD%o;V-7)EbSdcQR+6P65UmRV224k5- zb;!oUi!0n>_VoqZF1*KjSclLsX>x|X9=e`7UEFCRRL_fh0#(Vm%7f>2JIq#HSe1Ar zX{3T<LpXwgf{x!3Egui(t#;_G8+H(h|e1IF`t5-Dl z$}5qpY1WCFeg?ajOweuwdkj6gUw z+vxgw;FUeI`Ju5EcFkRmWSjK7uWVwWb7zkI>3AXzmtr&j!E~Q4mwE-0r$Lrf+q5rF z8<&|4@&{>>FStie?}5%rV&kNwWI?YY1d#U9-76=5&t(+B=`?S%Od~|y1ONW8PLj+1 zsP$|a(QYg#hwr@jI3VGabjcrd!I9qSx3S!E_zMdm3J3OM{M^{7eU1$Kp(uLo;z}d( z=F@36!Z*d%4Pi@4+xF0i3S~)^OUv;5kxOUZ5OzeW{;G5IHnX~!Wf34Bo^+mgs+uhu zwh1fT=E>yA(Z+c7xdxt){|ftZ(oz}cIbLmA6nbskSdTmI3~Oast$6RuOE#hzHe|q8xw9K z2zL_VdmZ)+=rh&i#EM?mHgRB{x2}XS5#6``13!)da;hqnYy^ruiJZ+f$c96v-ZJdV z|E_Pg-TA4Q6&1-pc3|iSxUhqsRy+#1vE>28uJm1vU-WUDwrR1{e9L!1ZmauFj&#{2 zx_^1mZ5JHc@qPSseK-+L57gb}yHCa-hb=w0di%S5`@X8ZZE6}uK*W2zz9WQ=*2hW+ z58hrcHTa;Ha_I`O2w?+YJK0fkq+^RZ9ui-4Trar3?}y!{B)74<|Jnh=(i zD--3&>0V(dyXgL=1j)$6Np?{Nc_9fn_qP9$$K{Je!7(k_6e^34M?xdODbL$>rm5BLeP6*kAKl`7%hyql+m2H^oA{*O$MN_=v~*Wfp>7ha~-& zf9XXd5^jhzoY4DO??woJM}1iFH~#U*1{^kX^U_-= z9xAO4Ro)62v?|L=Z{mJJU6_A%NaDUTbU4j|v^11PT!F6cSJ$bG9$faEbSQ?vf8gq) zhuV+jYql_FuM_2Kmw#j;z9z)vBrlhYx&1kq#5V&+b&;fNxb)GGb#fzNs<=Uo%WpjT zDs6+L?nAJ1RgE>OT@{LnoTVMT8it;$%LuU?s7cK=Dk=jgLenTB?<-b;i`RaAm!70# z;-wJs|BpyI{wX{O9w)FIcvmD{ih4HMCtbaml>50cHr=bN7(VyiSHJ8(tN*;yp_D`7 z=U6V@ixvN7`rFp{edlt9xdr*UE|Ye^f!;i>*)+Wg5cwvI0C*s`uKw%L(6O@*$76K^ zY6md)h4wqY&#I>!a@pVcYm};R0Dwa}>!$d&?6>?F5!xcGCt!vy8TZ;frhg~C)`+^Q zyTO1K=*AZ0Bbv<{rU&%!T*c5|&<1Y*U1P_y$%A~_eSoAedFL!4UszP~sBOU9vmso2 z=*boP0p0v{l)O>cNc|ox!AGx5F%HFE)TPzod(+ri(d#k3_A%GZ)KfvhFRk;2Z+2&) zQwAhuVcR@eAC1NUc)QusK`Up1eutNA@muMKh!z5&ed5?ud|;c39-W9KU*p8QgwXv= zhOZZV@|!cY?oLwVnWwzF5M3vJvz;IFdxycj^d`ot==j(?<4AfU)8QYd)C(6Oj9sx0 zBBXZ=0kEj`k7B=QQJuB?jEpc8k=N+1~xzAL@NZ~pP?SzfuT{79GSHkb8lZq zd<(vMisB1<-&#`bHIPJLBZC_2A}>kiqsSWwWt!eB-;{UNA2K@0KOp-})jsw@NsA*yZc`leFWzf7xF+=}hkRX?Da6bEr0Aw*0}Fbtucf zd*POaA^T89Q`-9{#1!%|V;`hI;KTZ%i9 zA?tG94E5x2yR61$-r0pP-DWk9!4jGOIqv?)N_ z9^3!)yyE9n}+Ln4560D-ZQFIFI@j0fj z&g5gX!zU|OGZ)+5d-R&KAVzk!4)VJeJ)~qlWW?XAutd(RA5y(67y@ zxgP+#72Rk~QeUrK|M8$BsNBDQ)haSi$^Fh%(c7V#8ibo1zwc6j)J#lE73I7>+iu6a z-=1xXp$Dfwt%?^IqI3ptof_Fvkrb59j`zSDwO{EnQS#gTS~F3u)-d!*M&cBZ)L)v?szAU%^pfb@Vf*7aKP3u$p7(D)OM~9f+vJ5e1gza)A6UWOYpbc~PKN}W^CZ?uHtlUdSjvsK ztK->K@6;=&wB)cb%8i6qV_WQJBmXFD(pffs#+k^l{f7V)_$JV+_G^3KH zx-HjjcNPjWdXsI7E#Et}lB_yin9hH7Mtt|_r+xits`~($+J-Y|^V~C1w~_8_r+e!| zXE9^Yq`{mJf_qys^P@-UkEsJUFYZ~kCw<;2KElXxkWF;ofc8OU?C(W+9cl=v;S zy?6-ym+AfvG7*5d|JjiQ`^3{@0>Aipf}!)byh;@_C-dsh{XRQ$DKKmyv^O9(!mfJ) z05a707UtRvd^0AhEG6>&Bu1rj@uL!HYtmmba&~p0o~8$xHxfR&lA9xKHqGhgJ2H#& zR1ww+L~h<%My|n&k#2Sy16JSdQcv^2gjc5oyBL~FD-j}XGhCYiEd(U!8TVTrDr-mz ztBHRDYgJJy^1X!fYw9L_ACsx3c?aReG2dih`~iG(R#sFbyV71+T?8vZRotU88M_jm z_3{|m(@vXe8NxfKXta8os~0*E4Z3E&aj|`Yh98E9td^84gXS;Qh749Sj;0T>S=PCg zS^dz-kLd(4S9fAu56O64NnYW9KRbi#UknhNX>xc(f$t3qNPwr?iplWf_}XrRlEoDg zTC}-@mMYNjSHje*^f+Ffp_-dkjfFH9 zgm2AS(2=yQRA;0j7IfG_u)T#qsE#OJ_dEyKS(;E> zBxC-CSz!|kH(6ixx3yN0cYI&l6^XVaVWx}fAU+9x@*+7>G3BmX%2_c3ZFswJ;n zO(lejav&4E_nzXBK%0B__EjZQ?;8>wXl}={5NWG8kQUKdWa3jx&xXO9 zQw5YQI4KRKasjonuKyDN3Y$n3*r9t~Z;y6tTVd~Ub7<*K+v#S=E9y`-sjQk?W6mO& zskG;09Xz(S9$YiECEd$I*lniJY!~{mBfFCY zriG9ji?ru`(!LZ&Z=`yZ)IFaWv3n<uSiDctWe1CFrJ|Z?q&T3xlO8Ui0?rAOA=+ zZ#3utu0HPu#WTR9cGdz!bQyg1?LZ2AL#gCK!`;PajM`1ps;3D%I(t&)Mui`DDqeEw zY2&j$3qNN&4!&4ok6LC1rwnzn7LxbDCJf*ujqz+HVC&dHLn#4nSbisn8380T#)L>C z_O@mr=$8=Gf>FWy%O8J*?W|$1l0+n#*sl4|B$Ju(~yU)rMe@p2ju2%b2opnye zX8jL3<9LT9ew|Le@oT5AlJ)`(4^(*3T_N1C^LMr-V!oCLQd`xam0H5Szt5)w7p5+S zb&h#FML3a$rD1GHi{h5alge|tsbQ_uvicG02jR`t1imSbs&?`1^WG_Z<@JJ_`N=sb*;B-+~j{-NJI zZ47_G0!mr9RvdK$=~?IEwZ=2J7fgyJ3Coy!5X7Df9DmBRPt=#nJGChzZx%iAj*E1B z(?Upqc*QGF_FAvyYOfq3dKevh_{kwK7`x8A!f=Au(k1%98J> z3l9}I%85J8pZPpO`OHtQ?Dy*0jnA}QZ9?RM=dgO*3Y%*|O z?N3iZp%CY`yAa*B=k99dpBlf7CY`%35`5K~`KQi=__?xceoTS+9npU$S!-&dyMuH4 zD{~9xfujxKU!h^bt?9~`Vh-Z`IW1WzlJ)AZXK}3CoeK-k;h_z~;v{s$PM`D4m;Oh* zyfE8cK-O^nZvcs2M=k3$PLyF_e7zYCHgcvCz}8u9qWkJ`WvZazP@g^6UdVcBRp=02 zB7>i4c%;p7Ia6igY)yJAv+DntgdE{P#=aax>K5o?VkS~PU&yxNOuJ`T9co-#jAu;5 z2lJ-FayrLpZV13wIF28X;W}U|kL(AuO?)(trXN=i9tqwE2^1_;w!qMY^W^)G>J$fS zpXJBpKA7~F(`9oJND`Rk7Kko0Yd_4or|}inao@Aw-OC~_k+{=Es0iB0VAWX;i?3eN z-Ypet>7_$|3yZxzVWCUDna}q{?t%mot!e5ajCEj%78;Em| zwK&DpTbsq_wgK+Z-LwY_=UL$vO|a`%ag{gM)A3Km$-q-F|3Z=f9Tg(1bnS1K^nYt4 zepg{O*Y)AHl@Yg^Sqt|Tz=55R^Q@Pyns|!&h`(OTPHnWA1IdF+iIP!C3^Y|JOw|rX z(Qkm%MVnEU>Xb-ZbrAH#?(=3XX6a2x_PCiBf01MNKB!kKc)5dh)E?16+gT9wu}!7R zU3SeM%&0i!ak3kTvkC5> zi6^*DfoVxy>Ehja5R=A5oQyP=A2XAlEkCJ{wyt|5X%?U0(OpIuIB9Oc4mig6 z9VPG$KK$;~C-j7DoRThbnI5fTb$#~L$*)?Qxk`CjQY4ww@eyV~urT(xwX7_pD{Hjs zsZ|0X7>KOjPf@#sr-iqN&iYAi(P1zC{*{L`hWd57 zE==u$Lkh5@tSolS3+31^H0!Ud3;zqBrcj!uqc?&}QiC z2F6l6qG;EP0zW=idz<>A^jbn~8|KpbWuq`?p%S{k1BN*uxDNgD>Ed5yM_{3JhLLK4 z^I^cHe}<7ns1ifk#ZRx*m9eT5-ZPvH^GLF& zMCQJJmSgSH(Zl?no2TdEUSc*-**A@Jk^n{YKxe_@qbO0fer1!Ei z;{p>rHz#E7`oTiFM=R1EfeM^D=9O}P`u7aRudwU9_61Ck=!7jj8J`Ko!nssp=2TI` z!s6cD;RfM2v5)Y%S?O(t$*>C#`8|7J$D1*WCATquSL|N|BLY3*)#v4&n<$N6`VC)8k~G6E?dD< z3PRf(&&?amcP;DiDB!jKkY|0r=}z?D=Hq*t!~fK&HNOwRe5KPFa^2~y^sHgS`2>Z4 zTldto;rw#!S7;h7s?6aBMX2f8&>LT`0H^3VosIS^1PCzX}?F#sV8pMQ!eLq^6I##*qT@}an-;8|MB#AO`z zS2l`5X1@@iZ5@*PCxhDAy)ez)qrd7?Vc9gZ)jW;0%c~aJ`d4g}4hs>O{CyIJMy5(9ov`z~t)m|m$ zY!n~?ZnEJGoapq@xz4OUq^oi~2k2BSuKlMm>_JD|GMesVax}t@=9h$eCCWS-86;B~ps%Z0csjFjnE^qxQz& ziG?sKzlxQajMmpmZ$~|h>dV$y#-?xr^|Sq)Pn)9oS!G__VSs18?1o>uV;V zF)jV51#7Fr^T|QIwW*mnSW619BYPPAWAKVtL5I;Q=+6Thy|~{?23CH(^3EiT>QO z!-+c|vQq7AMqKtX)a1_5ndR*1DsY|Wl(&Z*<7C>TUO=DB6q^tyJ{{Fs_7od*I1<*L zSegg{HvZc(s}7HT(8%j0g{L>6_j-NX%X%U5As$BXTZVZl0)GBIHKJVvmO1OE$B8+G z+zOHAk$=~HD=wYzO!VcYxv+8dg@+EzL`D9XVc8zv9XupDk+(K+@r-=4G#t@8 zgLF34AW+QJ>l>~1uVd4if<1ebPhXu8OcKT9p(a4-(!DNrq3~U3b7_s7&?9-@zaEq0^kz=C0IxF25Yf^{QTD(;LsBv ze^}gJ$;pE5id_k3NwUmL?61(&cP~{(wd({AcaS4u&f;`U9AW-vffz4`63*O#;vncR z_qE^#ikSmC1(|wQ;{hOOK^KG9WlsR(Twql3fhQvviTQ3|!gc69%PrMbVw{ zJ=kPc{5HHuWUU~ahutEsAB=7rXvxZh&y!{@d-xE`WKYG_7*$*hDK{pE;olo2QXyX3 zuDo_;?ysnF5E&31tLoQQJdp-yxYSQy(x<>(Md8enSVr5xFpY?tlh(22Vg3uQkb}HO7Km&l5{3D^AaiW&f&s zaW#A{_A{KwIG#&}tOn8o%%!sFZmEZP;n%Ql>-lz^|hVWF~zbf6Hb*?8L0{t zr*JN(QNkJOIGz}_#8Na75|)v_XojE#C^`30`Bt~>y!3-#K8>7Pd5hhZ!}{vexrW-5 z!a4H!g?jRBFD)7NrlY+$EZSddwh89#uQ~yAdHds#y5UiQ)aK($I7sM|(fiB$mrPc+ z>8LOZ=c*MA@CcBzdJy#_*BTV{at0hUR=0H5-5P~bL$40Cmhr2s=VJVXF3jv=3rkeI zP5Xqr*kB>c4xU@zFeM__pJilq8tV5R1}d(2l?bW~=MeqQe4C!{frvh55aJ@Hi|V)D zSX3aLqU7-|ZezZED-2rzbDE}pGBb@G!WPud2*Em`Pf1p>N@n zh!moI{S!A>4RNg+G{@P)7(7d+KQ+@)uq)P`m&w71@hhl66s7BBSzRJ3r_-8T7w0d$ ziU(#8_YmtBPuuE{?5X#Pp5-aWjti?+Y&J#=7TDd9n5 zg&h#yRw~{&h@&|t63%(`VToM+ga>8dx5n-MJSiQd9c1jfrYj@fW?#K~b$i=_zIITa)+oSy&lgq0 zHo49_OTnV#4}P8KPgrJ4Mx4E(K!=0j9R2*fL;L{ep|4zgWEoxIa1UK)Hn53sp1#x;%b((7l597)&{iK~m1M|LR6 zScQ;&|LTNN)q{YxNsjYz`Ntowldm)y}~*ODVuWObi2@tRvSB} zBR7Lak*dTo*alW&A|)s0A&oj@eQEW3e6-nWp~wl5T_u}w=x)(3uRmC9a$>U$V1u8p zORSlY&V8S75QMO#Jv^f?SE>UeT)a;cXC~2bhO9D|)p&RTk!H5pHQqmBdIb~-+Sc{1 zvMfY(Xf%y0$nn?>Gm#d_ySlLm9B(DXV3LKxX~W#}{69sPdOJz$uJo-GEZ-rZSMF3? zZN7eMUp`gRu39?M@(d9>s>`W?Y;-gFiWIT9XLYqM<@v56$NH#phO|zWNR(SosC2w= z4r@;;G0nQgoBLHt(Vv$Jg)9Ft)7Q2ec{&|V`UxCHalV1m{>r$}y@oQYBXGJMoG*<5 znrexS{|TmG0}LOWXm6r*UAOtf3)!%3c41b@x{QK-{U^caN0;0o%Yx(mlUbyF@2(NX ztA-!Gyi?>=2T=g(UwCXgudV~n_v-OqHVaEnxXK{?XD6!Z`yyV59~TuHd~CBkVWVbU{MY%F>5}nP3uN#X$Oa-T~}_w~ZC8UPLKG0%Zb-<5ZFmNM3Dtbv9_~iNJfC zda)!SvhDrHZ_P~^s;9cS^JZXr2z5U~S05Z{?<`aJ?G!?FDv(wWt>xUkmK?oYolK`z zNzSj3wHIkG?zp!_lstpu>g)xr%0czhVCRtOP{MXl%6kBW=Tcf7Zdcfr0B;*`H#HZn zEUAIysdng|2aOJ4@yTGhof&RIC?0Ks)tB0@`j^9yr*JFhxx_hLt3eo%V3j(3G7s%3 zoXkFMEL#j?S@O#ohkf`0|ElSZV}dtmSLQ#5;yvWtjnnD!f|x~1Qu6Ek1!I&iZTfGd&A4=D_`9{v4@!4N zoUJB$rBx$R3Cw>LkN}nU$oO-+NHp`rN?EjR+nC%u0M@*p-&i*`a0%7a=D!TlD+zCK zC+qv1k*;5>7I%5UrLJZ;-40nekrc@HSPvPl7d2^$A5|4DP3`CAThI!P&9SrLUn0Ad zBZ+}CeNF?~+LjZKTarX;6(sjU=Wt;`EzF-!CvEefw9#)U#=o>QdQT;sF_n~mW;Akx zU`oA$PCl##8UtSsVU7EjV@u{cKYxRKV;5txGUD?u!ps#Ye_QwuVHSr61)tU*vu?yx zUOL)=-EE|FQYU^uSNh$}qbwS1kD*8bIkSMTsy(&>OrN;c7lwc{y$0AsQ0O6mKqLG043V>k5ntLYoiJ!WR|qSl-qPVY!9 zq8~3SYK&#D^xIt@uwffB?-!I)6HhH&Xg+A*MS`!#iFuz$t0_$O=lGu2TcYhf{E@Jt z&1|@_(-a|k35ReX=$2Qm3OvGub^BhlfjAM1o&C<->}&boqe&B^Qq_T^E6dfrnnbU> zwaTFd)-H(s25=jl4#-A=dmpfu40=V70sHyNddCNlv{o)oa3>))AwW33JRSE~6;!zS;cB-8P!? z^G(cTh;(6JfvyQdU`L|sg3pXxDU-=cVLV`ES;RiOT*Mhu&hq8{X+U1D#K0DX4HAP3 zg>=D<E$Fb|#?|=zLnIy5Ski~wE6xlW;8c8Y zzxv_L+#?x`>r+E_KqX4zjOg{}qZE8+S3)RVXw?pp6_?+U4nER#QTQ+8uLN4z92odk z%d*zHpZ!g4ZW$!7U4$ncwh`W#SfO8;qC*zxJKRc8C&aX+H+~rv09(^``^l58O>LRD zo?_duf<DF;wJd0aks)J7OV7NjPdXB^(K>*J!RcdbZ=M zoV!Xy!}cDb?~a&n)=(EC+m|!>0a-IQOe_&oJ{L>gUeg6ifIXTTdn(`8rz8T9Nixuf zvCMY_w!iIXXx_fBeM(=Uh6gOO^;%Q}nuJz|n^?BIh%HVdqwA(~54AV74ragnyxzxb zg{1Vg3T|B>i7#5nt_j`kfuT=)jEJ(PH_$(`6H}^NX>(Uxw~fgWWO^$~A5_X&2pQ2& zrYfLqR?I)57VkXoPWFftDDNhLFs2^W>M@^S#qi)I54@K>XX(SaG?!p&6u5@Cr_sn8 zrhV(Kk0nSf1{wIL<%<6uU~_jF9ZXgne06Eso5Jau1E2_x($#~Jf>EXoItY5XxP*hP zoFEp>UKWmdp#^vb-FpGsV?o6*>QDTrXf&vQnC?ZeX{&Y72DAxO4;XN)w8s>O9jnsY zz+T&Qr>8LWSG+x@SXlnRVa?3+8ZyKYIBx1FpQ50tNXeW}1$Yma`Sx+ zu+2mbuNytrd>)=P1lSb_4M%*ykTSD|zpGhH^0mm_hTs}(w>eg`Bx@P`YVRg9WFxGc z>iiam37RH7I%gDCuqWL^`|VVt=y9U+bQZr3yF)p6;7$P_90b{TyKtd9a_AK}^nJB= zS?Jp`0<+7Ycmoz?#%^;xo!SbG8_x)113R$7MNL#3(jQyi=89n-2Xde`2~5hZO3~fPGpMOvP62^z8{Pt~iE8NAoj#a>NdugN5%eH2;iR{tAcuf!>yZFA z5iTYivpt$Tf*w7yK)@dyn>oVCGQ@@%&N?s9RPNG^jI4hC8O8rvtIi)CKg&W*jOvFz z0pZBH?-}(q{6zo=kQxml`(#%(`gi$YKDGefo>)KqO};J|k!F1e-HN(C9YTAZl<)|$ zLqa9_-^=RJ!Z~9DKpC=^#)nKnDO38+!)ERs=k<^`0;Qu)a*_^eL@PAWlGb%!Pb+Zw^a|LRKUatcf^v_TD)*F6<KGvqq&DNTvWIg?Dr%Zfj1Vzb7ae5FTC^_|`GcI09A$Gm6T{FSL;gbGJO+ShA) zu7~k>u>H5G;)jr_5}-|I7ejg(an4b7OZ0=M_Wpdk(?Ho28N#}H{#Qp;-|B-fv_S_J z`sIu#x@LZ=p&(DTSlBxC#0y0X?eq8Oa96M*`i0MQJsrF-bC2r5Wf2)SiHgC4IuMnz z;^iW7rL33Heq-r#nT`(w)2$Vd^Y-)3(bV^|MT{+`mcF(rjgQVqo4Vkjw_XFcTAgE3 zSD^5_8J*=l-}5H0VF=@xcY;XN`KVNxwS8-KDzyazV?uo1lEYjxyl+T|wBq57OxDc0-GSu*_nxe13ZTiUo}x zU_J7FD!*UJ3B&~dh+yf3^MV&60@VO-1UkIu4!x}b(~2UwLWffW{R8R)I6K&PWU{I~ z!R+N^$Ny1#`T=(gaM1vhvFu{U4Yqi5rJ#NqlNq3)hDqxrP~5Y>RWoc-n| z*eF|JFeRd$d6c$duoTW5b%~`k4a96DHko(iCTgl{9y_K?TsCXqlq8iUqbNI(yE&xW zQ;7uESWraUW3M5OY;}S|=8Y<^S`QOkD_&j)DUbV%1kOjLR!z@P@Nj45(h7Uj{Yh3< zUbx?|$tGnTJ8mGO>IGjP-I%)h)Slp16ryp&bw2<(sAGlJBxnh(xlWgIqy%&&6KflIu zquk{l0sC7Nm0X{|_sFqplQADGb%&O^bj>>AJ}>-fIVwElj%pPF^xuXro(;BkgEiY8 z?E;bNR5x>5NT2bst=8SN-YgSRd^31p;jJw`5Zd+1MAke3l#!N;#=er1 ziXw&W+G24_A4y`|f6y!km3^N1y{S+X62v?JK@|YuaaRptUHo;2EH~}s@Y0c_X;aTy@|Nqk$Rm@R;W($XUL%o6Mw@kE!5DNZFy@0tCQ!w0DF7eCzFWXLVpTv8a-dN zXxoo>ng6{}6)~qNB0Yu%A$=mz)Z0Gze0oRD$htHimG_@KrlE#}yIPgKLr$Sd$)@q> z^M8cNn>YM_%9_Y8ugEp{-theJVa72h)0; zP^$}9gUKG;!EqBi5Q<9DaHoYTKeI5=b@}pXcuD9pJOmN%!(0@=_9VfctPs@?&&)%s zcGSuiTW(Tc?O=Uu3JgcwSf&O5^FbE1!(h~5tHbR@K7g}6X0ZZ<9b!jlf%t6{DV4qobNCClKmum(g6{D^#-5VSVM73L8C*hfc29JvK5B8CC+Gc zRvt7hkhv!=8(y(+Fc$uAII=&ru%L;K5(VDF7PWId;2rD24q@LWCRkpZjk%m=JwzBf z!#P91txe%T=lRa-zk6x>8}vt~nH%?GdGiq`C`g7~dD^@{Y-#(2-~n%(c-8>}G_g^K z!U>X9MUG%$9-uMt`Bw$ld)yrcU_u6BvhCEcv}HozOp=!{|7R|=`OnXNHgoh~0Ip2R z_h1@oZvMHe;cvv>G%inE$!O%2fk`|N;8t396#{v@V}-fzSpTF zV<~0)%umVAMcb6v-wY&)X4Sr-bupvNDrt;z@wuChjXq*I)fttpQBV>- zb%^ed3OaVw&aj?FJ5RspY-C^S=;0hTR63rA`C|EYH>Hvd5W6wDXx4_O`m#!w|I`G~J*^1+Y@zRsx$eUyUz&|g6UYUP z#H6rLyjD{E7)2g3k(r+_#ttr|d`@ZyP1ktBTx5li8k@6CtP!cEPe^jDn$?Zac9iLd ziCh(21T?c)mqOZLZ*3?!5q>^eDsJ^Mnz@2m-E%QeAxoLB`_%mM+v+N#0Z9tM2Nl8| ziaVR>R-zVUC-n=SG>YmMSLy^<3;)I4J{xsB!9%Iv8Kxk`pv}hD9aTeIs`ifhNU770 z42!MIw?a1Jw3WA{4!DUvhGDwnYx(y$^hQHgmG(S)*QS4YXmeSuSy0u}jMxNhy;b+f zm3auTjtK1It$-~z%3#NVg?|( z%i_o6gRn?i4SGdsqVtTN0NtcJM!zbM0)}6I?lIyj*>?OD+v0Jsqp{cjP5f~4@qjl) zI4_M5jK}@ybeiv!Ghbuy=}vv-U7baR&ZHpLK}pkRZvhI+t%%!X%X^(YlQCPGd{Lv! zmG&$uN7>bE=ZQOywr8);9JiQ@uXrxnYGiKWqy5!ed4`?q8kF_D@sv~TL`^7_kBD6? z^;Ou63-Qllk?mL0(M6eN!1&|`!Xl|ns;K>k{;q0;axlw+*4=(Kmyv8xtf&sj*oc9M znZ;JN%b#JW`3EtyzPg!Vm*}-WJ%SlZ=DdUPT)|l)^WnEAJXM;0$=zAjQE@bltP|3U zufO2aN~Ap;ZcC+E!B}lBf9d8f72$iX^)2?2Ai%Wfg;{#H#K1E0#O5M43_6%9mmmyi zEZ-(tY)Vl%fl+A?egzcG2A_pQvYTqz9P14f^IaBIIs!36``SvHG0q(RhO-#`I^VvA zTB!M0ecYS}4%+%r(}04!+8O6GZh;#%$TH1U9@QbZVXn1_w&=~Kp7bFtb=~ACDPeQm zVv6yOypo9}XKj2Sq-_?n**-QptIyId*PGqY&T`d8ck?^j&VxD{YL2AzC89TD}S z`Na+?V|{VO6tem5X>h?8O~8pZzM{@O*(zNokV7t{D+Bl^}U{D!aPDl=7X+ql|&X4+McQB)KXz}(VrEu$bNEUep~DuCx^#W zzBo5syty#u%2O zRIZb5r;l5%=Xr(kXFkQGeuM$r{DGhN*#(=QkncgKJhe1J#; zwJy#qah{hf#AGG$QpAEQ>0dw~dg@ph$07JfwV}AXt)s(BOffVa4xCJQW4bAu+e2d& z$V=v)D4g&k?a#|hQ~VhE+&szB1O7S#uKk#^@)IV0YwPmLA@KlmC;%yaylJ)3PJCA^ zg)L_(|6P;IPMu9p8`6Z$N=J3X;ycT{&crN^&XRGV(_lDKYl&vIp;jmVAw@HI+9nO6 z40p^fV|6hkAD1Y{MBT?Q16AG6&xP97ABJELg>Q3Xhz9Suq706PQ@+|%R(Sc-t7O$@ z2sA+EJJ!*y1>jhpDso0{H|@`Zi-RRwoM1l|d)m`!yBX6}CU2WaMH!wLy}l;pRBaqd zZk?V`4aR;a3dDw2&m4DkxOb(>Ch8-Y_|&S<{TVv7;xh9E)96}wW}t%FWN@G{^)P(yy2YN{$j*$s39#DtTg=@_Q(I4j8tWvHr6Fkg#>(zRo`mxcZ8fAc` zdJ3vBH@)+jGZh1#_R$`*=Kh~c59xOY-;v|AJ!o^QKl9Ovf|MV|LL4yicO!RHk(WkX8+3WlaJ0-<{r-UEp1aD(1Jm7bMP^M6Z7I>W|Un$$_fZ;2^pj z5?7#qW}&aG7JuTY+iAW_Tj;$Y!`#;)20^g6zqbU(HhvjxA!A)*keTiflqmYP{hp|Y_h zi6A=EtRjPoJfv(o+puBT(~g_dQ2^RwomIs+4m`Bk*V`704B=5UJ`vK2Yhc}$@QJCQ ziwgFDIdEv^m-4_O2E9xMBNgGe^TMY|wJds_kAiieyDKH!=JSOs%`J8y`N1nL3tk!B z!tFxr4!{#Fm!>bV9q0<7wU2t`7P-rmUGMm*f6(sipl4bAEIij(PQR{*V;@b}3FkFdqzS{S^Uz-; z@QsIb)Fg3>)W6(Y_A++2&+FjLg@hIZ=t>52DG_67!I^2cec;hIm{R3H3U*7ySkBu7 zuhqX~F-gy#G9zzfL%_^tj*)$=>D{%Yeh)ZkamfR|p;OU;B&!Jl(u#8u^27hCWKF^3 zmPUtJM#n@-)HrgpJSl_oX=US0FD%|c7Xz!@m?!rPfwY=x-efkJ^#z9jIUyp z7#uJ)oGr_6-V9r_R1Z?jHUvynmkWQFojPxngWqbJns(XYM~71jorHJJtqz+Z<1u5M z2(3>Xop;8!7!|nz+TTbk`T&Dm;rXg<3I4FJ5uc(bK({k9}$=+6scIXY2ic zM7?QTl6f2cUDK4)qE6$OyUt9LPFblb?!q+P<#d}mWr|ypT9V?9J1CW@nOkMK3)5JV zDWKuLfE${*P@>=p3MnaWh_Wcc!`%Pp`9Cjt$Mv~>{5X&EJdW?_LJA$8^lXS$`1fyC zFP#hzV%>pBmv01qdu}{iwLNLhgG%p9n5d$3=x*Q}yaMMzR&UETgXq5>+r5MDvTW<9 zz6cAwvuGbR3)}e+IRc1iub!xxSV$RJh$3qn%Z{$mtcj>GSG{+t28&bZr;dj*h#lc& z>Fl{8S}fO}7`OsXRBu0kz~7-4aw@jZs0Ny^1M~YR z-Y|7+qV4UKy%tAKc!z&=p?70}+j`=9dBR8;)}9zGtjUd{Qgw2OB)W?}Ed!Foo$7-) z8&UfL_~kceSguEd?y3WeO5-A?Y1V1O#(iXxduF zm1Ywyjlpd%KqxpRHe%)Zt2+T$zu&xwAJjIS}IXJPj()wVY7C!lJR9=gZwtg5-{Jfmm3nX~#dIogt; zZV3Mbj0LQPw5K0sY+bu1iu!cvBvRSb{E!N#dZaib(OQ@PB+%E-FX}O%4mt596%0M( zz6su!6%=DC)&YN{?c% zf4g3D7xoRU!g3g2>;{++7)TW^HB&w1qEP*#|=t9q-hj_mnB1j^^z z=!Bm34C0~F&JhUlBA|A=+J!8;1ZmP~>ME%ZNou1@-FviHh^ShYbkVA%-87LS-UUjX zT@|WRyZ0CIXE!*j!LY7M;9-7io7@MUv0zPa{o4|23=;KHxv`{_O#{gkw$dswB#Cxy zn__XZV|GkdE$p=R=tN=e-e+Xiz^x93Zp85nn zh&V9yZ)~MgG;9}HD8{fs`V)3vMem~i*5oTZ;qp!roottkH7{l5 zY9Muq@!~@1ujc_8pH}`Rrx%oXZH$dwFJmjL3NoLZeS8G~!;EL4WASl%qQz_849eGW zOYJBE+^;{uenYT-XF#GGCT}(v27kx1737)A9nNh&3XGI4P5GqKeNJ92lLisd+|V_L zKzeW18@M9W&{K%LzO4aU^P*Z1w4)la;&-mFOa!H8E*SLz=B+INIHK%vL}-IExw^S# zkH&mJ?8Cu<))T(8<**qN7ltl}Z8iHTJa)|14|i`lQWtK7_j%fcDQuhb-fhb^v41pS zVYU}gndZWJc4#@}v|=2<>tryk97}C%+`)o{3GYcY0PHhB(tF-fHxj6}#xaY~ZKS>s zCyeFAf#Gmfb*w`zRwKPiT?O-IGd%({m=-sc-Lp1c7y+Xig#)Vr5i=d06!e5qPF@r( zQi!N|P`9OpY2bhuTfsP2PE6Oxls7BHtmUhd`ABs1xkG3NSkB0lD*$aKTDG2NDU3h=1Uct}JJv)g%G24w@;QY_DS5o`-3^PCY zKC%j(aUc;MQ0b*B;#S_LC9|H>y6i%S|5NJ)xZnzDh^AJOYJ-m*ZCWlRE&La5nW&>+ zy)5}#_^3(&I-33JxvVMQ+bJfH_Jn)gl4%!TZi$2C^SKggK-au+UOiUBfa#TLiu^hR zPIcw^>~%kRIeg{sK3_=)`}^JJZ#2l-FN|5jG$g$ly>Q%$yfxy9EVeQ0?YU)7T{y&; zdV2}DG5#Ol1i?JkQY87jcrdGn`87q5B2hVDQh*?gf1&MkRq7EClC5EnaZ9AHd12p_ zdBVrK)+QA(7l>0?U;$*!x3FtAcn{JuCp;;~#atuw1U8XBdI!kD?$VTz+gy&qZfqb9 zwytiGKDoY^G=gn+5O9bYIU0EFzJm}vB|QtPx%sSrIfB$-%lMXTV~<3H8+f?ZO~Gmb zKI+hQuBL%(>%EYb9Z}}ja(9E?B?eUcEu9{D*D^JyaNMpxQSE0r*bgJKaSixqk$oHv zg*B)jd$)NEMvW9fiBwjHp{m#dpMq0!E~Gzpg>ld)uEcY4!px_AF?)UgCkODqP*U5G zvDaTVl4U4}vkHp4Wh^Fb3_=pb9b|#XUOH@*@ zl*8ANYx8o6lm*O_zf(=7^{|kF1@CR1`hM9kNEU(7{uU(qevPWyNV;3|3bI>HmUUS% zg4w0bi?>7pknV2OZa-?UhbpJ5&hCYwkbVe+fKiq*{52luPR7% z01HW#l)4ZilDluUKUfevUSF@9lTR4z2CA}zVXU|v_cx%{y(F4X**tRx4>b@WP6)S* z-Q)K`*tX^VF+J|*@Z2)1{GPL(x)B$2swsfbq`BYA!fFKR;x zb!YeGh>5BGlo58?Ks*nzPR^RH^0O0e*~q(>Pzg+%`7wE|sQw!uYp<9`{R|dz5pNfB zXhBG>Lw5Y5|BY;n|9YSpr$!_O?5#iz$hPBM^+EeAOAj_aXCu2;kOmTd}h9v;4uCobhBs?uh9>GgkQo8DT&cXey}%7}@tr0=S( zJcP~U_arSCrE|60EHv+!8*S4xdkEH1L$M#5s+-N3r&fCB`-el~CeUcsAUE-j!E`If z#>xsv4~`)8c`@Dct%)+UNa&|4nxmbF;EO%DPP^Y$ikHff0JQ-Ge)<#%$*=5pFb5NF zwoeZ^)%0P;HO`mgc&(M=HHkoek)2}R3V1`@E`#fAE^sMD)GJ7TA$}YaD%;B&s2aZLA!|^0Uxo+tzaErFU)Cbvq$;7?_hsN@;EiU19X%bvAGKN~{hYGEwXDfz zZCr+IaWtHn+}N)ZhrgcaF}=8$P!>WSA(V*Ccz>z*uRNm*Zs0Z_R3E<=h>V>bR6;2m z!}O69z{p2`Ced^M*!9NX3z|iT#0~ zUDLdQAFr>I2B{1WoWd{P&UJ;YC9;zmRibnRL*n4)nejTaOqz$oySr5X9AX2|p}MEU zRhx3#TI&`M8oc}+P2Zx4?a$8OL7!@_Iwp;ti?^>0xuc8cir;zhgR)J>>pJe0;Tk@~ zTmFgTZ^F+hi74?Pm^n-rh4?;h1fg;%wLQx$wsX5U5|Xl}mD}La2bC>OBit?_np>9J z)=ZF-ex%^eOLk7ggkf+Kn)1wbmuBr0Xp;~}?Il;L8m1`Pofn$C&2kFm-Y7VX8EoD^ zx}x~8`eOXwJHgPZzDZAP0x59ce@prQb@QSRF1AcaBOL`Smsl z#UTd&!j)R9Nc=eBrW^??uj-jk^cW}h;-n=^Qe?;<9iLuXq39{X3|YAb?`Dw(!&;}^ z<3Q3RuKo7fTlv_|f0d(jg_psw)}mi&E4 zHLz*uXF0Qps+s2)uC+=Dzvd7kHeDnkVUNCp<)VfU0<7jJw31Hj{>nqD97@kAl)w*= z(RnADH@!AQ>@Z9V3qoK|^;D~Ddh&k61MsIGMzE0Tc`(Vnx3}ww>ONs#f236#wtGQI z#$0q|OjX+y4xydbLJwi{aS2Hld2<&)iBZFVE&H&|g)CR+u}dqPGCM4n4p{jTO%**a zhOPRC0HYwe%dJfYy>2iQPg}a@CfZlgNDo)Z&>sKlCOG~dQgt4;es!rNK}2Wm+{E@( z(xc5X@*OA%L;gC+Ojk6c&!A-bInwHs3jq8%()(Zo1TzhV|Bp z2-}%76*Cj)_m5y7PZx- z4|jLFDurk68i{-AN}sD=@G zjXxFW7xig*a%Rj@^v8_PBV`cBLWMyspubQ4?a+wUNoGjC3v#%HdkI@4FduNWpAucm zU06vvoY9ulo9bgxjQIxeiz@lwrRRgvIQkunjNv{v409 zIo9vcysUz9MK%;#wSC%Zbj}s|NL)vJw7=dJeL+!ylM}fj(cj;YN6Dk&VbM?hCwu=F zj=g>mtrDT@M?G*)q-eRd!Y~ao<`L+k zH7NWK5uW$T0qJ~WjrQ}EoCAHtXxi;=3*$Sm@)CG62=;xv^Nr1LKlAdbcjexLpBoIn zuM?Z60L~`hkH2nS)FO4X4|kp>?s+i#Ev(Nz~V$nT49`vw{ z`aY>Dg8#&Tr>#>kER~@9&m#+_PR@@^xS?rJrlBo{06th*;x1^(s|jNvwMNv*ErPwP z(Dr`?GJ1)71#tKht(80U`PKQ5Oa^>A3dcW<-hANH=DEs`#9PdVlPwt~4Y_#Au|VqU zVNCxkG% zVO$BdBp+X≥TJlfQ{wj9)TY&?|_)&6*b_{q3L;=U{4O#n*esP0f$(ROa2D8UNX? z{nEyw)s_Ikp-kA_%;=lH3@Ou_qJ6rE7+UO}r>|Uh>()m@ly;-W+B;3k7oYIZ%#y?Q zu4hTwwwuS>f=-GPM9bmmuA$goiZ9e$eWpf|KE;%j!a7PgZfcJcCAG!5S^mJ!2OcPZ5G(&?9Hv5{z2B?E}<>h-mhl?SPP)^Ynih3XLk$>-ho9Yrlfa0F~#^otnyeDWU^Qh4BBoW@6B7G6c9UmDTiT$`W-$r0IbRkRD`+SO#Uh2>FYr8 zdP4bxi3|fZBUCRRBRX_$mjsRv)`t&UlqV^VFA-cP_V4mEW zc6q#9Ocsra6`-PJ2x$kly|!>nBqB>hr1wJeIFT@OT+!NV|BgXk zPAl!cyO^n%4?k=n7B88g=&IoTQ(8hc4xhE{+J{0i_9Z+ibMGIP)931pGKX4B|5J#Q2oU_ zY@M?tALn?ZhgQ-gMi1#1^^I!hy+j%&RkpJ8awirWk#hNh{h$BVx(U=Xrm7J;;Ipnu z3vQQ{*7e@Td$PK=ds`!B6p-Go07TNZ9%=Cak#|v6txr`xhsapy+nBE!*zY*3rVuLk zPW%ShfgL$*1i0@{iG*BpRvjM@ORpGH&z_&LDDQVElZp{(3M;{)n<6~)>*k_Hyt10l z>%#Ts1uqwbjleT%i8tkqsWT##i0x?)$$dpwPwa#gYVI?Uz=6B3dzD(K8>d(ss-V@I zhQr~Kn-yACzTh0aWTzw!)3XLnCXpL|B>;}x+ zJ?yaxNNd6Hv>U}+9ws~5g&f*mW?`3h|3HoUxxGpM^A)H30*P1rr{d92m$mth)+W(M zvTdbmo`i0>;)@M9akMYVyrK0G0L17N_!>|PYxY15xCuBq4>sm-&CG~&<+E_*|4t|f z_sPv)O-!Vh`v_wYKV(%}3F}1;%p@=I!|a2Wo+WK{ajfSm^MOri;4JmPhVu62_bEHE zwQC!t>Ig)uY^0FXaS4u8oRIaOQYapd%W`KN+gaQ+FL!TFN#G1q7p2N<<}wn^9@l9CXaa04iX%G}ToVME5(53S0~_ zR~`D|!I1{f!p6*U;p|9PW6L3}_|}naGgL53hhP?htJ0)Im^7F?v59TYsCZSPZP#yo z3)a=wGk|;7igkHYRTsXKcdp?jh|HX-lR-Ks%tldg-c9{QYbWM9PMJ=8! z=~Td%9+@|$s0{IoCA6z8{@aon+J+RMy=? z7B`URtBQx@J;LE2S12m4j4Z!|+A!N(BK!o{x@8JT;1+_U8x0U~UzT=r_ug6tBv}?z z&r%?{47<7{AZe$3Yhzdb!N9)%;kI%J3_hngW#iIqs}#Q6wqJgul=)=um(Db$2{!ZH z@tF?ih=ldvq?0_8y%kjl=>$Ye9`f}eosrUf=nYKSRsH7`hv91xXqk)r#sM9YHZ3yv z`<~=g9fehM8h7ZNRAhnfdP(z%K(vO}#<04XG~J9WNuRCQr);BScI>MMZTj%;h=xAZ zyYaW(^{9}5wl_a_Y(@+Vy_&SWDi9?q6o(au>iZyLlf7c`%M<)pgqZuO@_#iG!S~=! z9<%LajOOY-<}0Cn4f*mvhv!Rdw$^SNj0jGdgL|BEzEfNGga5Ik`+_!TaroY&IZ(32 z@!H9KKS8L+SjD2=oQk^1Z}&y0H2JpMGybmCpXi4$$H8ZnZM_m7tGZp<58Cly5*yXx zN`f*0+(`DkJ#TU#kurZ3W@euig{Ngwldm+{c7F2SD`kB8B<;0C(PKZVSs4ox9SS_f z6ufNNS|`FXy>Ok!9DCcveV#d8()Ed)LHPO5mE`dpQVUwKNwy z-6$M}2v&qPnpciXp4g!*KVJZJC*fVBnq;FI32o;Q#pm4MGgrYHf4D_3F`<-_KESh2 z>GtRk(8j=gR((o|oJ~s*e6{-E=EsWG3%slHQ|6AT?F)s&=mMV6nx~a-2P2~F!oq?3J*Qo3MyNpeh;*O$A(2Qx7E`a$p z^TqFHe@+zRjSDcTFphZRy+c`)r_z`C8M_Vg9X2~HOAY8iBlJCpgoiU3e+L{ZFw8kv z;;6f+Gd)-yWIKj!?W&Uw9t_w28gJm|0{HQ@-7%oZ+8bNDc-{*T`zEh)YoqGtKU4f7 zqbo1N$4|kOLuxjzL2NV%$t}?qCMxFwiUb!%vWp?O8LTiQ<@DOeI+5-=v#hSAGA3xr z)P!N9QCTvW>7J@UAnK`L+Wm2LV6#?HD6_R$6^zS?b#-kh6a6z%=Fh(1$no1N%{LjO zw$2ash8)wdKWh#Igje{iem!9o7C9a>jxjKRBj4S!$%vx7G_;Q8z%VrH?Tn1zGFKQ_ zw=5v7RC`E^am)~3=!sLVNwhXu)WVlFeO9;}b>0P=(J=#wy>5Gw%8nL%v>M2{58D^n zyfAmrM;74QEl^g<_uHXthw5iA06nMTc-NIHAw>z1u6sFxF3yp_WX+sH#m;o;W|%Wp zRd=MsZ8igIXfvLG(EqoC*YZtC@kEoKVoG)Ez!WYBkJ))Mtw`F~0R9YqI|XYW z{oS2bDsCb<9E-5cs_UOtgeRIx>gV2XZOt_GC9n2mU`m){+aBec@#E6ib4@XR@7#O= zpY|@vCpaue%Q|?Dz}q)`euhPK?PiTWp?Y^E&pDgtA9~ZoB;9&a8*kB3K7ijla$}#e zYt;D7?a%8o41d2KjS<8a{aUQCiL0GC zfHiL~xBP3~r97Lx5!9BIZ(q^XRPBZng%*$Av;f-ra#b0)9T1*or+9!h@0!51po2RI zkIodmEoUrYpCS&rd*sLMzwOU#__@zXl@$Lll1mFAQkA(|5qc?%Ullv3Ex)hX?-v}> z@^HPjT|v)W0PHKus_u!Y%SNpPQk|92g809gKiieVh`&prXd3GFy|_=0e=;zmr;3*d z*Y}OY<`&A|#V&i69^E8(y)}H)yy%`1^RY#PRH8RTR2bFt*C;oR(pz6<)Maj$4SSDW z^QhU%pj@l~$0mK0*Dak41zYdBC~x?AAE;Z(C~PX^R+)?>*!#FPJLPDb5`*S*2)w&P zR#|3cn++!b!S1RySVG+}41vufdvY7BqtFcQ&5D7-JR3mAO&#kZBqg5Qw+0VaVJHwg zt>{?$HO#nCI5=S&BKj}TdYI-dYo<;aNcg6J41c&@6yKjH3%O!7J*W)X8Lexw66=nfr zwic&>qrlfGO?5~CViYm@DXF^>y3;}cE=JKT7?!eV+x|pfCE)m(GM<;1t;4ZcFpemZ;rN)MqV5gB15;_F~Nynzudvl1)h=T+G(aD2DgF@0;Gm5j%RO4%EM= z3rB6k-WFo8)}m!CW|?O6d8`xcukiUVpY$Q13Trw*H3IY&r z+#yVu$97R1`rGpQ%}F&=CGnU;e42_K!vw;h{}d_{EB5JHo5a4Y>qbWT;))9mf#9+} z(OFiAyS^e&N*zxxsDkDdk^=CE?|>c2E|KQ#TS))~gqPr-*yKXoA3~u&`c1>6eap1& zBSIC5Z|Cl4R^xSO3dm}psb3**cU8UQo_{kob1<^45V3t)H4vv9>K)a1Na`SX5KHL^ z0*)&=l1$;O$^+0vpgpO$Um^PW)eZ)_`3!iwFtsqzMDpPvs8pMsserDf5LMvl#uj<^ zlVH4URTk!DZ9ww{*z=KE>+E5uXG(%()R{IE%*N(M8>4p`G&WlDl!5@_O+ugP!W=Lg zQRWvKGIOOd-hQjEp9Q0SJcxedo;`p`PpAghb?b1s0!uU4u%sbqLy@YU{!q}tnB?_Or#b+3^~O_9jBh2{wF(QSzZ1ZL@ywHSH&N+O%JOQ6j6vP{C!M5AEXt%ecr=9|9n= zT&KRT=>x7gMw7*n&M4MxOfK+1Td}H9GCY}U=TVYH4ME@}XI*R+{2H^eL$-^I20!9J zVwiK!{Bs&E`5zB}n;-3oE0Zoip(1Q(RDgT{b%iA~rgL3LeYd;Y_$fcZ5Hh7uyMFF= zH~`&MCq7SM{$*0i#QX0heTD~4UFDt1*?f@5TcvGpwX}@KSivx?Q{kRAb$Qat)m9NY zU9mgV+uVm&u{6T-mIbKGJro~ejy7TCnh3CZ5o^N|0cNTX6z)e_kdn2YB>bh2v8PZ z>dR$ZMa3=#>Ls0kH6XJ&fm=A|U#5qIxo#&fwOXvb_GhYbIdj-bWk$B1}H|;e=8dRI`!s7XV)M?E6MdV+fq0?wE_{ zQ#3*SV7L>jAX22(2n=?{&!(jkAjeZwj8Pb(L~_gyra->|0rx-S*S78&lQ34Rl9-KY zV#qh#@hP0Ab<_fl`5n!+Sj|W|(xUC-1JriO?)*^87rmyWa#WCg^h;MsSJgA_^5_&A zju>;`ebG0PRfEY4RMsMaAgY?8fHn zwu!ArnL6nO<2#d|39@r0NTg@%yNl2Az!NVu(8Fcj*UnG9rw<9dc25(w;pc)7grvqi zb}?h}*atA~-ju*BNG12P9pkp5&I9`)zjzEk;Y^iuUpq>myJGhGr<@f_g#PDSx^)*RG zEL$o6xDxk;CR&g+9DR}F{sgUW`Vxx78(G-8>WB}ig8%k=t8vB=eL3)s5pRH_BcPLI zUcQ!-*wD`aJf^j4RA?!p?c?z1o)52VkjRC;$l{7L19Q;&uo;keC1KZo?yyoDdo*07 zfKccuogq{n4=FEo4xC@8#||SK9$wdJogrF`*k@;5Ed(c~0jTx@HzURt6=@iuKq1{L zDrx+@I-3By_lUrC5+iAz$vJ|7jkoyC%xR#;iwneaOI=IeC+|vnv@pDbI?VBzQ%4>1cAYHY zomOzqWJ9`cdn;`ceL??xHm@d6a`tR)ahqh^Xr-Z@^Z$Z$f$E<&7JVA`cU*NXoBq4|0lH7bI$}Hq zpt!gIeqD^MxGI~PKShg}J|77tflBt<(#9KFN=9s72X}ZN#6HTgNWiF>HgFR+y|=VS z!Fai)Ug@B-4?Bc^)az^NxK-`Ygy0$Ej`;8_zUVeVkOy}h`!xMlGEC5$w-EX{5$mmY zGI|t36n#W&@b}7+(nrb0PRv9{NtDuOy%4*JxT0CbE-||rKh*}Ve_YOYNVH00Vb7*m z?CzhpMc0e1=69vy(+K*Zq(mi}fwKzU@N+X~vaTW=3azYuIv z8z&!iQOl4)=GR*8rF2%GkbVrKzm;tEdhJQ-Hp3nME$xvXyRvsXrb;XF8n-kDgHAdI zF63WH5(;TgN_p*T4xWKg!0`lZU23Hh!c}FtJLniPG$O*)-gn!1a8VMiH#hdeq(e1-1{Gbr+bLj{u&b*MQNLLBd!KZ^8Pz%>tu98k z7x@Z>bj!AdENts~@|yoHax|FiVU!}^{t)WT?@lX6{Nf8zJ--xM-g~vji}L}tsB9j# zmSeM~Z{@tO-kRb9<-Fd-CdX_VuYI9Z0b@QP!%P;8Fc$W>QtBv&dzdj}6;tnrm-H%t zL?xp>y08L__1dn@>qEkydSYdvWu^|CTfF+=@b&TEk9le{rs$dm{T$_Rbnq||K2KW5krrY+MA73R))CE#6qYTyHb&0cgO2HCmqtb_t9+_VQqdh zwV(1OE=;a~+VB0Z-SF^;{T{iu@M4zSk-rnN^YX30#Fy@C_f(OX|4CLdeCb|-(w>B? z(f2A&Nv`KjO;<_qRN9))?y~)Bom6?SF678W5QIcMAeekaOa2 zSxCtF-w4(lS>`Zzq-6A0mxks?lmki-=cS@{0or3?$RRvR?8=&a&y$Y@A!4{^ql|3A3SftMIJ7tk_Ok(y#9bv*u|-r*2z(Y5ur9>IWcW)m=F}s|j4m z_=mwA(-edPCEXOSnLudi3!>@E=-zx2!?P$t3$xh}&!xlZ?;3phOHsIyHC$knXw#C> zZp>618Hd{!kkL+14n8uc=W|!26)DFz1#D!$dW*F6*^J@9Eb=}%B}RDU3zbz0@Cmp4 zcdogcfX#*p1&w`K?zgl>?Nhy@Us|+#5O2DCokN%U!72F&Iw|Q5Yi6h)Zb$}-BT5oPElwDzoz0O?y?muXD$g{tW6BzBC3s@B-{bSF^mw)=KBmqb8*1d*LkS28$Y z=OBGPwVmQWemy%R7^$kmoHE;H)BJw0^AkjY!DchWSWI|DsjH-?6{O&KA-wj>hmlCy z1j38Yt6K!kwB$dK)qa5AZtWJBsUxhEZ&(A;X9Fo2v%4qi7GX)#y9Es$)|CHU zVnZ3+8v?McMaR6a4)x=tRYx5L!G(j#Evv`0Blv{&1Dj(p&0wsEPus5=rwB(RuwTa>?QfNaPeWh5kLPB{FXL0EY?%jIne0UTMafwH3 zn)%#$KB}OtubvbfE|@??bo7$xLzYp60qc&|SIF1Yzf!-LsLjq=IC7?E-`L()7@k^< zH@0vGRVo0cL#aUu_vnL#zozH6l0K~6o}CvmU&Upx{yRTMx+I1vMCaX3NcVijb>e=9 zYs824o2)AklHFfW?R_Cn6 z=}&af*NOd@sgfeCZs68uwt4gJp7V144mL8q0(VAu5E^Vg_{x^v9)5VJ<+V9Fe@6jX!27Q^m- z^DLs}MLZ&YP1QF?Bq>G;h>qMZ&i!AwI_U(i0vOhn@c9vRs^+*pK_c+SKdZ#T*YaXJ zy=2za{Du5eyKsO>6kCZBnXEU-6k&e)BD`08V1a^_o-@1ncGqHvPUxQwZ8CA;L=*slE6z> zvE?(>v$%xw;r2U5FMu2$7QucNIJNQ&SUR3Aw_)U`X95(auner?ORCjO^nKyWi6T)Q zeJwtj)skz|H=%2gz54X8@2>u=8KCqr^#{A&b77CC1|t9XuiC-QGSHRsXX&7yZa7vv zyYlkFGU(-vj|T4Jvd_Oc|MAB+nSTU*UvmAQL%%g1y=teg|AX4I@BjQ+wXg10ZCzd- zv;z+#i)3p&5`AiSLX@n(MkI}TGuGz?dq{&6GHh1FB(IZBlnbZEni_k7cWKZVge7FS z5QFS2t~ImC(U|T{iysCHd$jFN@gHZZPETObHtopg@vCrg6(SgNA#$KItK>oAy^UJk z;>(dHK|jw((*~zmG9hncu`BNKH-foc?S#;n+p}Fl;qN+@*^Q}DMRyQCksI_z8dAe4 z`bUgye~Ul{G_5{4^5c1%Y_d}Z^@;l0$!3ndyYFAV;t3%emG;fz)wCnlI&DYo1Xl^{ zlTz#UGr!*sWN!~=S+OQlxr1>Xjl5E_k4t-IOMqzsjt#2`IK&&&FqsIe)3U#nz;~H$ zP=ENNV{=Dpj^Wgo*MHUR9wqeJD`)ZL^HNBFOaGdS9Q1*Q8Nge4;gOYR>^tC z$x+L2^dp)j_^g`Su~mbK*JheKe9QF`Qjxb(Aottz5Fa8gHzl##{Ve4;xrjZrQ<&TE z-Y62?9h5qJ+8hkmjmHc~RD4DZ;~J+4eL;L#ukTdhG5aUSwqDoCShC+jeXUKj5?*M% z{XJNcL^XGfEWI4TJwgZ-{TskGFjD=14I*Wzlp!OX-L2tg;wvrt$N4 zUza~5a>1`$E|HjmGD2++t7_->HnAi=SMWh3B*9`11_ zIbLFGLzS2YG8$F5ugjS3G4YTY8=t}!*V6}BvQ<_JE`E*m5%%fiJ>MfN{fjqDUKAc{ z|6Z#ImAu^v+9F#2bKtAn!SBk-#gnX~z4J|fWT=o8X^x%|vu3vpmfqv|uWK8aGF#sa z!u@@OU*1aJeSMNbF;QU=j>hIH?GnERoQS@o;YhmXS9VnX*-wBtcwjN|^zAv=!SC57 z3{&Yx;94>5$+2wL)8N|B&@8>~be{kl8MX>w4hkz6WCt@-=2N3qV}e|u2=>h1Pk=DG-g!Tf<$L zGo$kv+vX7ch2kH{mds$I(5S8rxT^+r=41d&9!%XmF1+cYSUUeDcoHn^bI%=&kUTcA zlLAjseoaTtP5XUT%?))WNlob~qXk$Y2KeAEEK`2ZF}FP%lfRi*&a1%JoyIj;Klecd zrsk=gh_LxPZ_pmZO%f21T)~h!erw}1#eWtr`nqwTv8=KwBhdIf@^y#*dDc1tg+Ina zz^>a5JEJzB@mWVfgqX~RFavraL1=!yrOS2H+y!jEVb0~!WA1vfmW*P)IN>La3l|%P zi36&n<2#{Eq(NIfu1MjT<7-j%!|(dBxb|ysQC0hg2b$hl(EytqFQXzY+WS5*>;k_~ zcAbju;%rB`doAU5Q(ftx(yEGZa?Xunx7d3}-_cfb>%l(V z;)&QVLPXNS3l!quFBD~#?ND0wtx}RJyhyBwKoefi+!G*- zw|ebjw+^=#MP4(@L*eCR47BV8gB~?P%wlEXLc2$ea|bY}=%)s7lI2Kkb0m-GH8YE) z+_RtWu}o=hX1ijGBsYi5xhsv>$?ueUJOU-}bM3``L!s@fMF$?*4Y;IC8ZG#P2kk^4A_rPo>%kbe^iy$3oq`1$ofk$4i(0)M&XgcO80r7bTE<%XJLf?MCH4 zNc>CMSxi-48MS@(04)R@ZEXCDa#f(%OQ!y(``gO7rC3*8`ElEm>TVCp&O8lZ=uR42 z`xy33XP|dCV%LEQz=R7kEf3!Zct2Usy&NGew)rhLTKtR)Y!!dXXW{j1KmRGesq%JK zNe%BgqF&};u#z0c`E~UroZmUWT7_H~5gXsYAI&2zBE{-J2h1Op9p5z8)Y=`zo#~u1 z{;jmp;iZBbK<(X{+h#E1%`FwzS5RdA6S*!V68L0Ew}8-(eRa+?$!>WVW%Q31e&vkc zNl|i=cY-Y<@q){ftGi_cYJUdZ5gG5$wR%P=9vFNcv_9p=ZvGHh{J_Xy@a_4Fw=QfP z8SF%cbj5yEspszBvY#Uz3e(nGiAXp28n+K$bU#;iTBfq1SHM;6-ssb92LFEjc=jLr zDEp`X$nenxEA948T#t+m<66__?a;w+h13;uKs5o0%T?}*RnI` zn{(GcI9K{zR(bMGy}HfZFWhMFmB}&iRxgCKMC6^pviTnic{94Py~;fVEfcgDoad1Y zIsAbd9fI#Z9`$tPX8q46WXY5K5?7qyUET8BuTq;d^E?oyaK>Oa{whq>h%USe#8Sgk z9)s-YE!z*3l~OAv*kR?9+a(gJ2ZR1@+XNLlXAGV+^Ny&Xm)uW&rV5MEVa&A zhzEAMJjfSBTZoQ9rczf5>~>aLgdYo7dwx>Ni7S|h*9qCq)4Y|Pk-RmdlhSjpYE5An z&M{*`rdWJC`}{l{=FgPwV61V*kI~ViF zeTz^6KXN5F;&KYM7Qjdu37e)BP``_M9$n#Jw)4n&BGU&NPA*l1F)GF0c`F0469Hp+ zC>^(=nx&qCipxz7QQI~@Fg!9$GI#3$K}3g7wp>DjH!VhlpRG<{&-Ui+^_0t&+px23#NCJL9|Vb!tZ%&-pJZUBkkAkmc3UY75_ z1)lSvMUz8%Kb6~kyS}d@6}mS`^1tPyuGjf=SHFPpb+?1qrK%)ex!L*s zWnP`P?Lq3Qa~72!6m!~GWk^|HrNNQ$+u4-5;r+J=1M2ixojf2g?c1$0Ns8rs2mizV zNu2Zz-S$0+I83nih9+>nJp|Sal>D{UC@?*ooR~(r3f9Gy2jqknmqzt$jnyE}IjR-n z@J{novqR5WlV|g`B>>KX>a`@}!JQGzX83=6-73j_3uWcPmvK>@KLoB2gL7|*r9RhT zn6Q>~;}rW^$=*xHgMwdo7XUts^$Vw`cYY<)nNt`u==6ANT7*e`0nRGLjc@42D~G#Co|{mI!PEgsCkTBVg(H2p?a&bK{IAIVZaAn`zvO(m-1JWYjg3qD;BX2Vmy#4@%X#(P2kja)%QKM~XTQ_uD?Wb#r?;K~oKsXd5X zwe#)5p;DR0lL4^}^NsaJShet$4eo2s>h6zKX`lvl%_7#fya%W6$25pKIQ6#=jvb6`;iBKRP5#Ox0}G{MGQuaTue$k zGYmfahhtmf-VwElkreTpKj$B7(9Z7b96WK$>}%Hk+@VxS?49Hc^ZtH6y%pE8xVpz} z?W6Pkryn?pZ%8R`O;}Og9-ejQzWc z-kmt)g8(6LJwutzUzW#G{@Y}H7SYV^c~xW%@|R4I9vW?AS~^5|tXS;5akCdX@iOB# zd%6F0^qWD=FNE$KH0r0|^77|!PqDHKQAlHGKuOACmi2eLQ}GLK$K5Z$r;2#K5y{OR zzNIJGTnMlpgv}f7p*qy(l&X-{4tur=m`}l$xBs;HGk&wwBx4in9CV)XZ2Zu)U$b{L zI}?hBS++3@LCofmCQyNF+UBpu|`z(Dks0_670(&a`WTWg0JsiE(JJcRW zP?WJ?%2`l!vTR?|@ujhH--bho&>}N-}ZZer09bNQ-FY!ZbBGX-ekgE==1vO(j!iSfIF6 z*0>;=D+rmHsiiemYA&=hWkO~OEa?8mumrZO8q!7r7W59Q<9^Y=&6To}@O?6Sr>mwNsW#uPR64!FcH zb;$Db6P6D^v&jd0LFAdP^vPs%0u44k6@D-9>XyEZaW_0-l>)H5U^mSo|k5-p9*B*^-K?e!JcSL?~Y#Q6Av}VRgAc?k^nb>(DOR55-69a%D8w zsCq`O*4tqtT;`JluE(stCj15Boau-erywun%T0?_d+poe-}ui2c=wu(d4J9 zJGKQz9#s4-YpLQ_*NQv5weJUG?ouq<;MRVAH@6B*k5B4l9xrNOLPE5#D$Orz#2_ikiwVJ1&F@8)MFR3ib6=c+L zwhbts-v6h#scB!b2Ryo98Q!rJ09_I{&1&}FQQH4wSVHHwmpfIp>8=YEX0wH_3Cn{ZpqHdN#+ZFBCRIO<;NKI{7LAm9qXTGn@Jn7O7S{2B;@R6*U3iIs_ z1y|;8LLQWC?YIHBPt?>9J!T&?{VjM}bHr$QHg!>MD7CI{4m@L2WZol{8O>j~v^n0) zbhIFG*=_zl!E9566nNY-P75G-9(W01=BFvjo*TROv}6VHRR0z{eZC5wzTN3c+lI@w zn@4TFgymi$%$6Vjxr(*NWv0RJ+ZkzEV*TFNt5&*8Sb@RKhZKHiPGC4c-LsFMYBuJq z9biwpSB7*7&y7XDr5%oHd}L!eP?!|rQY6}5{Zo;F`%@v2N)~8gvSK{?sj<^L$LVb3+%Z7&z~Dr_~!@(6kH1s{R|QbfCD7m-auuG9#jR`3_z0 za6*X!J7-_bUgS3i*H;z9ORGm(s2E9tMS^+;>zCL|3$OXo!eb0BR6_fH&$(z+i`WNN z^2eC;j{%(P?0}>qS5*Ux662+a4aJN`%wNq%B-xsT1`b^Np~Dw>fXm66!zLXdnjlu2 z2fCGfSRM*Higazedjbmj*si^M1=~$!nsMRvqS7eGtCa)Gs{nmZs*U-Et3awbToGLm z@-L{DhU~;h;O6pH6KoXyC1lJCxWa+G{^9V}4(yGt6mt^v| z^4dtlf4V*i>7|j*guO&H=1sb9x3Ln3C-g&G`+tRObPt-ekiGT7-0SL>hhPFG4wvkcmgsG&5BL_*VavhT(yo0m=U1NAob%*b7VgIFne3$X7A|=~Co9f0Jd!_*#0^{< zwmFZ0kWNc8^8&H81sIO=@-;A`$iL==pT#-tjl`2s`>{H~)#1_LQK1Vgy4%v3gO`O*i&I*D z7#GPxIDps|VHatk3!-nlmT~z^kyqmO<+&}gnY}CleqGVr*d{_v;6ce*_NP}X=3Z6* zWKOuUR-V<}3IF3&3jB+oGBUV8T7GWkyD~78r3jRDjUJb#o8DNS zi#8u_W}V?b#iPpA!|)9W%O)={>KAI zv@UqWl1?wz=1tmq)|E+e^c-Whv9hi-Yy8div2**y5>o=K0z2IIWFUW`!>|C4%hZQ!j< z*Q1A@G1F%0M`c;7Z4b14H=)`6^%+p-^^LS`Bv`foX211Yn9igZxTvgcTNG@t(C>E` zXu1`9uTF4tM@G`woUwEO(EM{D%jo?id$-rZH(!_Z1B>^@^Q)8aokI@krf^rGc%Vc+ z(YVk06Nv+PqYRlHu19gb79NCDT&eswxFkR|V1xSwHJcwN7MB2SSutJ}5vOOS%4jn0 zmBu`kuZMLptBVu!?w2Ew>HIWYv0J~{wM+T;eBCzs4&i>pMMN3?H5P1LahW}URUT0B zXRXuY>&>~op57IW)3!hk1{jXQ38d)9irAR+D&)5p__h=hN7a!~I#b<&k8U+Iw_N zUrd#>2mB^*1L22X1BR;!oOzX321S`g@!!yc|F6n+H&PiNu0sOKkGKrMtojD~3jQ%v zT5gXBzf93D^}$k#VkcEkd4X=)v1+Fhh z%Mdy7^NJ;x)u>Y2fA15DfhL_H1@`gkV>Z0IVXgjnahW4{x%YOilv6S3!Ha%n0lF*S zpX2!ZM4MMTuZlWYR>D+u=X=%HOHw6i5g1xyS55B?6BAcF*6)^)FOsb09_o+bXo3c> zWaKXVWH5ma>Q5E+EW)udO#)^(mg&c-vIOhYnSKuXR`x4S-HBOM&89&OZR&pj}4A5!>`jihb^71(WAi z_H`E9-1!-+Y(K=EoC)CH`KcVdu=vvBC8@x0+|V^%z&yT5E~j{{_u==zHOdvSaDKqtkaUJZzzC-rY>L>_H-qp4Ul-{jO&?X;YT#%kCe5 z=Jw^QFEmFleZ`gTNL)w~fF&o_8g?>nmp-Mc1->#>H5~bu z zyJn|p>7wehn1YPx$@A+9Q30%v_8wd8I;LQ1Kk63>6zsgFC%_A8~# zMlEE{@5C0Tap#~(q1b6+KRJ~7Rn#=rbi8lmVNXo{y%t2}hSQRCTPH5-*nY^PY?S5&> zOdDmle1#6y=SvMz<&rFHE~9w9={%F7vwSo77pK8LA?tN|KIJrvEQ-A-lD5-CWIhDWWBL zYti(`_RnCQ032w}X?U>sH=#EyrI$}4s7VyJodmsxNAMDd`v5=h^T(=bQH-f`+@29# zJ5*&u-5@aKE_ZUYolCxBBbpI~LwYnph}PbFwRx@_oig3{bBha8)s@l5)Gckap=Zr0 zC9WqjHkd@%>S|#)*@`3l-kBc4GeXi?%Lufv!1!fsRQ2Qo6TiNZ7b);tax0$HDz~@o zc8OO8+eGD2(5rY0^=s@lgy~f@!MFbgMp_2D?{^t_rk4MOATn+~$g;Z2+CmEVXJ~JS z?lpXMOCS~!Ox);(-Tj!U8#=Q zy#7vjByyn?RX*^he^Qze82`1)9IjTq1;ezz{2scebN`Ko(}DsmpDO)asErO85S%vz zKO7LfPV6}8M&S^^!a(ehST%&Jjy89J#0T%}sUBHRln75=;TEXgY$MxshxGY$T$8JA zwvOyGCi@bjXbaB(VZ&ROwFhC16gqdFi<|{6N6dcXak02LMq#z8s>{z)1#=-HxUs~Q zlHS#5Ot$y4Gu73qcAI#$rQL3>=WKai%9)khyb8#!^lDE#L4zXXP+DJJb#oq?M#Xmf zR?oV{Qc=F&fIx6Q&Mi*!Cu8jDDZcD79ea$9REw5TyFrJn)eGPKz z_~Y13dy%!;0rkyoKE;XA@6X&tc;<+e9#YqT?2SD{(M3JDUr3ls9$3}`CJQujs3e3q z4Z&3L*>#S^m|HbS!%=_eVD!El*hD*bf)93={*MX%amm-vx2`940o~x&FGI97s~lX< zR^-dAA!Ky&vI|o7H*uSL*1x6|`~FpbD$}#vluO&jOyf#yzcvj?IjPE%W&$?l;tN& z$x|f(YfuXNWRH9sXWQf+hF$4GRvti5TlUmszNV4s%FkIKgY>K>54_?50oK8^lxoQJ zkIU+!`)Z(rUVhEAan9wL2bY6%MSNb4xmqy>gM4UnX#U@}L2x<9`XQP=@i}nIowTLc z44w%W6ZambDDG@pFdX*G%$2tP z{JZW7IiPqW&+qV&HGe5(De5zwG0(j=dhZ1=*C9hvBx1Ye-c5|90l`;lUuVGliw5B9 z>8g97knGSwlZ3gB-VvRcTAx*Z^Bu0yCHBL;%ToQn9H!jQ&1;+|x9*}W9PWci|JF;} zX;pnu_CYggS;@-hr|%Yx`!4sM?PObfb(ajXuV1uLMxKfgp)>)Ebv{R6fvN`*0Q{@k zkZggj#|gl&$++}z)aUejPN(`>42GN8gb|PTq7QMUiZesQOcmIrJhvvsN!Vth>pkBs zX4o04p7QXek@)Rk>)4o7jz7bK+1*qT;9#)I38@GyHv|U$q9-e5&xU9_Z3ZS~^MfrI zE9zO@%nr=`@|1elTAS+lMG;E(STlBKcmc_=s?f@w=jU?0;^=USM75a`SD?pdzNl~f z=s+H}i0HOt2g_NOIlA)#X3P9o%+#Xow>0$s_X60F3ZEN=k909c@xmm^sQEB!( z_EWtdYRS)-a%?4ce^hvVi1YFQ8P$M{mAwt%^$dA7ltU1uQ{wgLL14o*yX&#(OGiau z{Q?*qiNd^f?cT=stm^Koo?_0Y%p*2P@O*PTUt-5xu*(2xDnK4MWfKU3`_Ij8;aAu6 zXbIjmrO9ICy=TfCCTVN&k&3#kVcRQ3>5>t)foD9 zZ)MkO-TvxA={GSf-A&a$)Df>4whT;W<#ojbLc4q+zvhzYVf*KLo3_`5bij*OpU=sH z2YwaWj6@>{eT7RU`0&7y=+h(Dc$TORg0nKOQV?slsrQLYvS9esW00cEu{+Rfo{#MHPk&T+?X zRqUUXL$M-Z%4&1M1Z`>@rdAJcJfI!mqN8UTZ#8{^WO9Z&Q5xrxh||!1Q|NjQ+&>oR zaQkE*u-g?ng>X9S(>KzERV%aCS6lLaL|=>+eL0k4(R@0*D4}9rRQkKhR!7$k9e^s0 zO=9Q1GA@n2x}SP-<4GYX`knpvIOY&h29EC6n9=zW%;pLMj@TNt0iY10 z978u0n*KRVnmBoO&Q@)U@d8%-^qw96mmy@lis?5clR9^&XRy^vzaE`1XV26t|WJ{P8tFyU|;|#qUM_TG~QUX&M2t zci`>(6TQh|B}nf~RS&#E1Og9594E`bfd#m$WV{;Kc5d-NTr+9KDijNV`v(yYZ&+)& z#iHF@##+7m3f?qB)w^%_5nt@J-;yW0*77WpvHcn%JHu~XDpRmicZ%y$N-h_C359y$ zIU%OBUaRw^gJtEs{jV!bdZIM{){i<+w`FJ^`sXI)W<_owb}*%rXxD<$J$*Z=@E~QQ zx8}Gfu=V|{-(6{!o!53qoA7-9oURQ9#|vKZN=Lo1#sJ1_sU3O|;nGjnbh!pM4PGZo z`uvf0Q(`MLW9P8?Gktf4wxDJc3OS4t+}s@yHk22Be0b=Elzo}FGUU^Vk4+QQQn6t; zg(wDJv|Kcj`eQ@+2;=P^x>E6)TeC3(7kjR4n*Yk|FemU|f9hN+RSAybo897otFy94 z$>m_@+5zrhYGx(2r(g+a`rEdQFq;uGnSyHE+@Lkem-}C|vf| zPk;iT$9*&)Z5rXdX_qS5H$4Wb%7fOJwk!`bH3v=IycF&U)NxAKS)kvhCilINdWVTgulE19d_5FCAhc$z7pK7GwFo_IaYVK z8IO*eF!`2^vM;I3G}(o}&OP|ecnV|23}@}{G4W(%hHW%FaGzb@6`wos7rrx$V@Z%% zT**q(vENiTX85x9irKl&4PLKkvW+=)w?p?|2HCE#Kv(4=`U88@*m}fv5IFH>7j=WU= z4uIvRFHzt=;?O|rv(sQki=?F$AHUbcg;>9?=tN@D?ioq8UL3u3HsqqW0Y^cXV@_d* z35%o>K`ICE&WWJL6@;oD6MEeP6OXj|!$$wf)gD0U_EIqJ!?CXQa%J#(9`z?O9s6Xt z>HAK}M#ad*vGD$xrmVuQNrJ9a_}u$z_&tr~K`;fC)O=NV>u1&-X4OIsB5G6=IhQ-r z>A5}al{W~EfA{!&QA?2ER*`4$II!iS*d?X<56o3_1a@V2xY{Ix<6hkt6pj8FLc}^f z#u3xoL@zZ!IHSggIGE8D=907EsQC8PgjUnC+nY9x4mt@x)#F_emOEz}i`fZAwY2q` z%v~dLWW2TRV6ncapV{`a5%JHpS5{gso2yT7h*=!-l2#|9l&aUH2ay{Y@}JJ1dQ~uq z{KEZaxPtZi_C%-XGxYM3dSC{h3437XsmYr6Nl6JWxUlFP>cs5#Fk@|5_C;{aI?e8q3=3s{4-i2Gi+QD4R1k-fe z-H>&(ecB)U(v7{}^_#9N&da(y(e?cC^@ZG&?FTWysshT>rxk(^X23U}^NW-nnV3k- zC=p+{d!WRzYV822&-b-Fi+B+n3*`9A=u08nz&_Z)3i}{V62-6aBY}+e2;%Aa3~w+2u9}nm9&uSNAH}l;X%WIsF zPZm%!-r0j{7589w@n>gxT5HyB3nQLFb;S`#p{I~D^8>%0*Rc-Eqo+GX%ROy3DDI~{ z2O?oO`Wu^uq{gCEu6;!H8{);mtRdY9PlIhGEI>Zin>H@z2*d8$G;Ue**M!7k67LC7}`5-(o0P@R?_9lrYF-srp!O}M6NN%{e|01Q4Ny2%!TeT zzP~wUZ#LZ*A)S=|ObGhBvV@tvAU(ZF``7lrzN|q_%&>eoxZ|eUUtTW?9q__v;O@?! zZqcgolL!|X7Qa1X-xc_@1#dBPRvEo-VKWNfW{2AvmQou?V|~NNEFDb9|IA%@?Ic_6 zJEVFzx0tF|JGroO(sF2BF5RV$6xVy#w#fx0j(j`ny(YhZA9{uEVa&i#=bP0&x#-0# zzq*vIW+w0?jzVw{cDKhoHr)4==(;%k`^A*cQ4SDf;0Bt+6e~@SwTC7KsmxIL*MQJ& z-EJ=Dl_YWCZ{pYdSeYRg*C~m;5~(eFWXua@+Ywd#!29};Xpy(72NpIsEDmbte%W&> za=Y$B5mDKhZx&=RSJX8}7i{1vzkMkcs&4@TX)5C;h+8@*EWU}n z(`8}tUqjpY2PbPWX3 zdDDr7#=`#C_~L#9MbfSPHY^V-#?BgKdYNpy;qF_ zvif&&*B#}*m<5+hke|`z-@cgOFg{i_Y>s>*$ab(m_sI<5 z_9-2UMpLhZeC);LVJ*vdFXL{r&4qhBB9Ihr_ei!8nuVE}Vbvd`42iCr2`ik0COlek za@FmD(Xs-}YcOf>7{dDX!xypJ6KftAtzuW3FUMSRfXvFhWhxZ2nBNqjllk85%G|Qo zM~iJFjSccb17AzFn2^HVCUpZ-4K`wRy?~ToRkr;H6SU)N7mB#Q3VJ8JSnU+$c8#ub z>@$nE!x@aSz58+DQ1%<=dR(Q;aeTj7_>h61-(&(IrI{0oTZE3KIm$XH<{`1yB4DmelskpbNR+ZTG%p`p++#4DP-}_K+%nvh ztUK%nRMr_C?8#8CD1Q6MQ=u{r|0sPcj!zW+X6VC2S=?Hwth- zrK?9248>E)ESE9MH7QM2oVCY$_(|gT#n%n2&s~^|0?Xk~Mvkeph^zwlO}xK7?#3s2 z0eY)E+Dv%Q)Wxy1*&nfHC7)p3Gc7KF4q!#0cJ=7oV`L*MJtE@o zUiNs2Kkmx!ED<{Ao^&+AGn&HDOG-#KcyY4faxQ=cLMN@u<_@{(7ODe`YFCe&;(+#& zFSz)u^)1;FYSJ=2#6-BBk(K%3G{NF%MNeqBoI3q|Xso{_HN(nzc1Ps%lFm4Do% zs=!4_Tb}3WdH;vZ#n%)WWFs-M@4dKx?|{o|I?$COH{q%rXVDmC{gv*ZyIpZ#TH0MI z8Y2{2Jz9F$t%aNxB%d?8O}>tIku*8(Ue~?DCOTNc(UjHuoC@P+o4>@6UFc425}=wu%AW*B}U zVeW9PXuAvFyLh^-%R|o>nYD0yau@UwE|0^REAdUMAF(1>12g1cmv%H34GM0W=+t)qxMS>1mXb3&C zDM?-vmfsIq;4eY9>S#UNbSbluA?1)5<-}pk5a3r=a-PD55V<`40Du<|JhbfuMee3= zvsaGay*F1*hy~IcW(EKX*MR)zA0vO;FkotD%_J`j&o3)6sPf}yxV$y+==emzp;nul z%O!aluv&#}o7Y$n#z*4_D!%5ZOWU;sr=@>pN^5gLk1OYOM106ITd@4o9K>}9It~|M zh}GG1Qsx40*~SGmTkH9QL>?w0MUEYj{L}d<4}5~+;Pb!snN*r;@=jsGxB89Kgg!s& zfuj)SR#@V(KT=i%nR%U>$0X1Kw;X0fX0M+(UEj1IC@ij$W|9Q!(>GundAGo-+Iv_t zZ;3s=mUSasy*jV+`ReoyGr#zbVs)K+@1#jZdp}o zY?tw@@0yJ)q>nW7GTATPE*31!Hi1@2J8`4e=C5)J*uE9*4e_&+)q%61(DEHP`4vXv zYqoP?`1_o9p-yuxb!IMvf&2Mm$-_C2$XB$nZ0KT`r$pteo9?NK>Mis7T3j$-;3$P6 zqG5#L_Y(`U{%(|{(s7 z>u`r1Y9YLcbU=bTW%wDcZn`~@-?LDYf%pPo$P&QBU6u9;@)2$Q`q&!A#17?awE8sy zqWs%8JsP^)15Ht85|x>vd>|SxD*_PyV;@A<(2OH~yNXfO>W!^o#1y&2M%PSq)!ybw zEKJbJJw<^B*F@vn1<|ii66NLASN*fSXQ5wnkjx>IN=pbQE!lYBWtyxiA@(ZW{!+ik z-PQ_t;%B}gM#SWs#~$8b8Jb~~Qhh=f|9)gwQat?I@(9rG9By<9h~A{GlmvmR-bePzjOdI6zKY#$t6OQHdq6C>!-<9a>FpsG2ot?B`mUnYwl;PK5R(Absk{1AwS|ITG5?2eppb!EhOI@o)IE}%P-TnIhUzW!zlyjLjxl9mK z{g!R4vu7-=uAoM|A!`U?jaFhLKWObwk@q)TAOr1-&7Ql3a zXZIv+h^hN+Vam9F+bCLJ4XrSc%PZ?QLE{a~L`C+**3bV5I)@;i9|MestnQ)#%r1|5 z^5$SOubE(BY>BakWb&zU<9swTg00En?a=eOX(}k|)x%h*9>#Z|ft1BLXYi_U4_Qs-5XPZN~s+}l} zc}qe4>{}KZIF3Mi$QO3-0nu5X?@psEPhs%JvdHE zN|R?dF&E&vj;>Gam!kLdCauXZ)p-0!YOMbcJ^U)^pw*@6gcVT0_uxE)yLY#5#+YPV z5dlH4p&V6gST7f><5oT2n1a8vTNjaG|9gt_M3f`<$QoBXX(Je%<<#%Uw0CX*f$7>t zBW9Of1O|pkygSeD;&)T(Hf$c<2Kkg%m=?m}&1#cufqvb2HDkcl2!Mq@^1$VQ+;N}KGY?N` zLP$rIMF2H0V8;t1xIt8Pi(e@pD|DT`1`4Yk6m9asv-2R=yx6Bx{OcEfYv%22cnAuW zuP=mAk={{C_KD;k%KHFm0${txqZbm{6L4%qHM_kA=cBcwFXOVT=JLuix?CblZL13k zTIm5MNK`j06F<~9DcXsGA%#p*nC}M!d*EBWa^C^YvLaX%6FB;;|7urWBW_-Su*6hXWp4x$80Z6O~%b@DunaD zgFa}%^D1J(j`2VxXk;k?pznj~3A|L* z`<|qq#`Kpr&N0Xfr}fgC%iVgdK1K=Fm(5=o#KTmdK&VELf*+zes^^4$`Ob`W-4D}V zJUyI@qUB;|s`UJ8osSxOD1@%BXBXu;2_vat3Sq0hAWN@ot_yfT1ke?(amG0Zp@dlr z%0}LZbg8wG>>C5@@xW{J!N-fUhGzs#U;5yn*kk10@Y%bHblF52<=v7glf+zhcTl{e zxG#-SFWU#LubbDS;U%$<{WZ0Nc44t`i+RY|w5nJ^NkB~ZTo$b!dc{zr9ka)`cK=tT zAEqX?&27e6f65pLw0h{#8UJs25WnY9c|eAw8A!vM;O-!T%X

    T@`|81O1W0!2t%7CiA4Ng|>Xi>%!Ke$B7m`d?Z;l%VM z(YoJrsH|TX!*ojn{Tnv%Uo44_PDgo;Tn1TsnmPWW>KAhFub{^KFvpL%cOx;|a(iky94lGKLSt1|rb^CO zHrFp^2S7Sgw&Zk$I6f{_PF953f1EXjGd{Wq#7AYAbDvthRXp~!>Ca)iK_B_+$#ZMs z5#EuGKuy*NA_pxB^e?p8Ui6NG2I0K}9268tD<%l#9HN=JjUGUbm#jPB&7?IpI|F|M2 zSC)Sd+)BzB{&R}`uQ2}+c-&?x?)}ip5f$dPS; zZ#f~Yi;C<2S;e|C{nxJMELswnET}IWKv$&to-umX8koMj#lo&HD?q>*d7I+Y?)9D59 zatr7p8^A-z1SN^u7E#tEtU-V*zpB^>3a*xp?t}WeFnR}&p$3gQ8*0_(`wuUoi8bB9 zi8-*`3M8i-XnkES&y5qFRP>FN>e5@!ly-zOy6Hpya!f-{S72zwHre9qPp6<_WX;oY z#+1`UaSrZ^%@C5=4#-C@%8z-*_}51VimsOBrPP-PU?jUy?}*MLk=pW#r0Q&JQC%XN zvxvz}PtDLj_Y~DGwP(dyGvgIIB^ITXwX` zo;-oVJ;5yO4eA;z#b0h4lO8fk4{7i31v2TD`b6=rbp491ArNrEb4Bf@p^h7#ey6TA z_KdV4489A;eCA_`#cV2xo+9Gv%bWCyVLh84hQ}=fYKf_NRQfX29tBr(A=CD3_+4F( z(Dwl&$`mnVJ6d!0?9i zn+O_-F;vDwZdjEn?!yGuRAHq%H1>uIGJA+i(Dzv}qzTI-;W_22d=@-_t;G+@!K9}P zd(2o#m+f(xqLX09#0DeU-Wsonnmn2dCO5uP>~jyLUVKE-T& zm5mnviNbe8YSK`yVdN`5)ae(JXK}ZRLL8t2@~u2G?=Z9PA6tN$Sw6{X_JZ<5$u_&X znLJ;A*RO-`5srS-`&*CYTr7*b6hQ5Ln%FhV@~o-cXlF<@G0H%mdZTQndHv9%8+Rf9 zJR-EV)la$v^5WQG@4Q}|7JQEahv#$bM=bX_I1Zp@E8^rWOxJ)Dc)|Bl#V+=oZwF_A2U2g*jET zslO>y^M-CQ=rDIZ?53}!qE*)qLYTY_yhDMOFef^5*}g^{&9b@>&0NsS085l0U; z#2ag@UZO+iMv)KJ3|POTyqLyXntZVPr9j55Hj1X1%yzXpR#?3)5`uXHKW^it)=IVK z{DUmJt5)4GSrQf=U-2Gz@7LVQV%F@F(<;nw0FXJNGU!HEX=z^Iao~IDheO9Pupv>n zfbAV$8C)DFh55|lKAp&l6JIY$+KiY{IcA65t;p`P^8jXyMr?`-M~jIj4Zc{bnq{BX z`U1KS%d;vd2$N9ab`?-S3CNi|pLnTsd)cTF$kD~^UOq^d9N9<5efok1Oor!Ukr+y^ z^L5YvO^0p==as<}(m}Q1-dMzT34wHQ(sNIKnMpi{XTJ(uSuzDQ1{XYd0acxL|<* z&m8&Jv8WUSf8w^hcsD$wtEJP!GeGx}M95JfqdxF=<|6O=s*quQo=p#3AQK6>P5fr~ zsRNR7Gl*kRKIRiXj)B;WpzR=RWG43rzD5jo*n&OO0OT7vEpFa@HNTLtNJwq zQ)g<$LwX952D7h`ZN88DF822EeMam-8oeNt8Iw33hW~!{rE6at zAX`T6ci9E09=0+4lP-tW*14Wq+YS7k|0eOzXW#u+_HJXtafaY^RoQO_g>qCD%Ln+) zi}#%R0>l)CGbbbl`nTr4)1L+~uT&d%fp;h7E17bglxX|YQP*XyL4aH`{t}n|AS3oC zs%Xiw`p{pC)xMqMC-7N^Q-G6k6;@0R&}v~?HTGu&&NJVx7-z3f264k;&t*T{n)Zd_%f-)u&C0;{0%xUd811yWcgTGsLnVJ?i>0hTz6SYQE}$ zPhzOmr|@lwh6ohDdV?aP%wgtyBOLc5UWr>zfkC_7P!+%RC;4?AybM6(NQ%bm{MVyQ z*MHhvQB;(^3RmcwV!3|A{&js;0^eim{v_Ky=7n8+Jq{U@ym%Q5r`gdyRgBHti7Uak zf?n55YMQUg0Dx3P)^9nCo~~|ptPF1#_Fvc?no1eFBo(c8W^8ak?=M&V;;ju11AAW| z#ImN{72n6cpMUu^`5N`L2*`WbGJ80D@9UcU4=nGbJ@PC~SVe3ci0tGt?V|XIj93F+ zmVcnt*Al48g{b1velgxTup$?OoIM18Eq>c)jIYAtV zdRe_w#;USFzcpAikP?|8}gE{`S57O?PTmSbev{ z5566{3E;XZDkDtlx_?ih{Minl^pKI9{ivtg%2-yB!q${>O}Y1vaIW!d8ac&JcNV42 z^&}leehTh4o=M?rJ2s|taL~G~hTt@k~eFbQ8C=hHW=v@HYc(^wWbP z_)O#GybFf?ExJvoHw`OCc3HL^hDl8!i_M6Zo78LR&Nw!>kT9|`CAn@L5$20 zSXnAx21@IJ1h@h9mFic&k~KeSgrj52p|9e>TNYf0ptJ zQ?qI|yoHWRbgoXa1}Yn;@7C0j`?Og2GS@_2zkRY-c*s|>^{|a*@R;i;BS)gJ-@opT zW~$7>8=%G04vZSqlqbGX(I_T`mqx=gS+?*- zPk>7eB_l;koV%5;D-JVyez4`=CVvh=Mx=+sd7FyzJ|#z`WQB9WXkZw&Gi0hhCj7D3 ziaYo%=LIanETTLtMn zdMPDXauj8YQGLD<%_8{fztw2pO!%kS^p~pDe_s-kxe3~Fm8QQiW;51X*g$8$57;c* zi|}v5%Q)(TqMo<(5L&R?y1ArFGy6DYodA%A-&+RvFM{=bj*uXmb9yB&>j~&za&$H< z!JzD_g$s|TOs1ko48TFy5j)6AUV6`FbJO&CF?HvDrKzK)EL{!l$_dx*xpFrFzIz(? zYX^EX}Y;VUvhpX6Oxpur$2EMqfn!K6vZt}ls8?)2CY)n<{(2tJivBR!WZ1I!T(d?f* zT&1EP01=~CpWdH-k&&|=()uA+ZKv_wFZtmVlL?O6%KLlwtXCBio!Sv+8NT^9%h~ff zZ-ZE+lB?EK_+bqw|A{QB;MyU&uFlGn(J*yWQ*!;F)})Ya`hf3z18;Zva#lDCh|A=S zz?^4IPezh3W~7%&R-js>A_R zF6E%Ni#Pv|sJtXDd5ZNbf z9hInMu&neEn=A6k9H11sv&2#|5_g;}ph3dTI?6%Dac5RzExS|S?cMtIlm73O#Z(iz zdWC`aik8{aEcJGS*)*rp<-eWtdEHK5msd$yOKwZ=$t$WI(TVSH!AEvmwrXjY^3F_- zZPdEDMA!eai*7jO_}Iu^bK9+K!+B}i?9Mq9eu>tVC3`NhKs1Nu4EIcLwQ#c<^4o3v zoo(^5*M3oAUvGA){Ei+bIn0rGFw&6(G695fdrQI*hMEGH=?HVJ^ zq9*;@!)AUsaL)jLZ?I@7c@ZUzKM57ge6jZfZ-je9(V=qB@yWAAUb1wDU7m48GQTdKDTI@q^?ycN%lFVrTjQei~le$t`QHRIqg)cju(H{0Al!cB5wB5-1;_h@PkFQK!>~t(FxdkCn6V-0F3WlBu;LhL~M(1@q45 zA<>JXoc8gMaGrvch&HO-iy0eijFH9u4t!AsquE(*!G9j^9?t3{lN=ig-q->+ZxXF# zCJxOv2e@N*CZ8vPx=!R>g$5PWL*(L)O`)R`IVt;8D3RwXekZ zrZ_7R&R6z!1ZRWto_-K4>wua^mD@Iy%%h^J}`Ir|2*ER*X-*w0WInVB{7V zg+GJ5Adh zZ9U`$cKa#hD56;aNF;C`#mfnk2b^AB%5|PU-|r%_}cVf+E&^2n(5lTHc*W2 z1KBq)`d4h-G1l@abwbjRk*4anxOd)1Fqc<4v#hBD=?$ht)a+sgA+Jce!JT?3LNR7S z7)_hD-W*rjG&z1Ab!VJ7D z5qDW344otaL9_2_|2e9hn6IPhojCJTyQ$;edK#0cr{Mt#9Qx_DMVGma;b?Aw%oLg? zo4s8V)F#$jxx?wSs|bqkqj&2xwM=kDW_rOCExefl<@DeO-nzQs*bOT-1Ym>sLf*!I_fAYTi~USn}J5`_-^YHkQ&YLP$6Ak9nt798yaTD zp^9TMpL(x=^A!tL3W%_3YouB#z6S#>aI4G?jF!~Xi)Qb-P;M8ll^ucf&(p31ar=;o zM{$RD*2O~@g$^2<&%&e?T2}frN(|9f6xZ~QA!qY&YvEBpzEbmR`cz9s%q-Ri&XVf$ zz_#se9$l@^L1WmAzxodA%R_r&+D706A?)g_V^Pa{Wot7G`l7ZMF?8T0CLeW-Z?Yu< zcYkx93nvawE5~tQ&Aovxhz(?h^zh-4hV6%OderXQO#nt|W3$)Jjp9t;mZH+YQps3V zLE-J!*P2F28{dy_ncf5L%Tuq3}h+3BgCL3C8tJw4>X~5a{&o) zHIQS=cD5GliXq9KphId#+R*Teq(*n(8jF%i2;I-V4+}Iw$9MbG`2yb9VytgQxERrr z0kea+idDRRXLh#P&6u}Y(1`wzCrp{h!sYG=5t9$Eq$v>u>o-Bo9 z1JiC?O`n6dj7~8aoLdtEMx=*F)ay!)iQ)Y#H5yy1#V$MC)F$=CCf^Y3VDo~19O)Th zwSb(el@xQILGW%505&g@2S9*%9Y#TCofCV=-gltb&t)3fo6z%!-b2<6w^&VW$q4S> zwJ+)*6j1%jVMR7CT}%63M%HQvR4%qN7j?|iNlWAAkn|~vR};KTm%hoLKbNwfKK0cG zl=S^V#y5OLB6&v=qLw;UaBb=sm$$6F)}$PqN4B zL&G4j%_rr83QDIuG%u|u61Q%)(zBEW2fHa4#m|k!;v;BTFx0yp#&#o)qyJqCQPISW~8HZ zz-GkRCY)q(ui})pyy{dH_IaeV=~JRJ3pj6!o+t~>>0(|PP)Wit!z@aafuzQb*P7H^ z=^^U+Wk&_TJ8yg{X|3TUAueXpVIvOFKFc-rJ}4fhSEw5mUY>jVju*tzB1+hP0<=Xc zITuOX2p{3KY|gXsyK8cSKT#Yg5qXQzn1hI}#|os&l}~>z_n8a?yTd#P7+J&SZ@5_8 z!q1`HwF*3I+4MLw7R~Uh%0X>XU6oK>O;mWM6|3!c+s~cCsoxTESn61pQ+^$_InMxW zt^s;F;Rp1jx@3Gz`|ii%TK7(AuXatcQc7_>2Jfu0cT;)q$qDXC*2ftJGjwutqv#5} zL8tAQ$CH)hkS8hEfQ*Rq3SHZHS1+~JE0bF5w2t&33X5l+g{8c&ec7VAd)&Xd2+$ev z20O&Mz5iMO;YNBP;u&>K@*~;!wCGOZDy(sIXx(jEEAu0!G35&tLW9!sVS-XlBQ_y2 z&W@imxWlgiOhcJomW3(tMz0;%g<*PeG3FH>+uU_VoXne>s#TP**7YsDwgA$8^f*Fr zk%pt#YSl`g$+*86Q3-%)_9ahwCE-Bf z0+&PmW0Mm!;#ZoCPp*7CoAar2px-&QSV4>}uSdX<(+#c`7^DaAT$&YPTUbFGFq?|S z0C5rQN)le%S5HU>oac4;K51VMW=T%R#34F^`0kFEi2r>sfa6>>8r-xnmo$YUQ=Kwy zw^!S)Qg_5gki54ffnK}d|3Z1^l2IbpE~j0F+7YrUSK#x)wI&V3YSw5Rz3Ih5B!E*u zx>wWR*&!O7+phq`&cP7;{=uceQ7RzHY55ExYsDvr!^pamoHMT3ZqfK$l0`W%bpuBxw@bj+J(l#@u(_%-PlKAQ`b>FJ|I~+5RXE7jkx#wuJqBNV#%c-R3^z>z9 z@6u8mIJ;w!R8dR0r$Ytr`rZDd@uW}q&*;kd?skffi@sFoncoD~48uH1p3vkiY|+p$ ziUk6u`{bWrCK@n}PN7OYS|{6ZlBN%n#Ccd+2|{XG>3A(hZ$7Z4H6$=F=6mN%kBvam z?^_`-qm57WWGm^s07d(ry9y*JN5jeekP0wYAv|2|DTCpn(*)GXXGRLo;rKZ2Tla^@ zZOwfd+cJ#n>|N0v*Z>jBbKaiyY3?zbE0gRkzK|NhpM*tp)TiB#N|`e@WXgV-V7$a9+)Q`SQ9-YLShRHpB=)y3ztJ{try&~ISfn0m z^dqHssmD$r@MeDB&Pq-0$A-t~f@b^Q63gawD(~Z5QODTU65Xhkfc)Vo$E}Dwo1w-c zp%YMiH!O7VJ{_}m4XK8+N4K0D!80RT*jC24)j$mE?Wb?)uVu(lq!FcDE+gOf*@3F1 z{U2JZ_lz^nE>;rle36 z6Cmy9>Bc!6Y_jRs=S;1MD?c~#Vf;v`KyRnOST(Av!c6^@5pK6@81}XCyy7V6N_?s1 zRFiCjy>hHbXuC5kz$emvG6%%cARG8mxqrMvobw`bQGc`>GC|#UYT(i#cQoQ=+j5&< z>3S_T{8)Y1MN5O2#jK4zH=c8~dYTQ`JVnEoO;-g*4DP zJC56b*mQ#D>IOUBtn%xEC%mo|rR{O`9^W-%v6y>W^p9M7=3ln8{@bkB?YNh}yu7zN zhpVdI82kN9+R7^7p0Ym3twH_I$PLD&o?qBs7Nz%O(q9lVq|DxCmf-Q}wKkmKi8hVoP@+bR2lj>6sthcyg zQ@j?W`n(wcOzIkJ#wFON%oQhGOG`0m$+fXBFK^b0+1kF0UJWF}VfYU*-hCG5ZI)RS z3-9oMg5`X-?UoGj4+N%zAA)sXbMp&nBEz<*2a5hMoLY8_gIvyc!gLQke_MZNis-kR zZ2Mu0f$p&554c{aoQMmayM_4kmZ-kr6rFPk*NE_DP(XOydfxM>@a6yEob+RsCq;tJ zrV=lCQMkRp*3wy$Sd)=|0~f~*R(M~ei59a1Nv*bMctnO`X!jn(hUYKzLpFhv@Qz4J zL=c7g|FZzv0tH6%pVs|i4B4+7Hup|$c(hiHY>`$C*TmXIG2^1w^p`Axv!Ui+2Wp1BvUxKHItPW3y2x^d^pUq)o2!+wI!H@0;R zfoDmZDFKByU0Hlc$_*uljC!peMTzCR=!M%mAVo}U60j%hA-FAgcQMHEtyP00_A%97 zDQ_Cy@Sfa4-;u4OPAM=gfr?fdgHckt!^BAAa#pr*igQ1q0zbeCGm?&p`pEBJHubZn zMSrm?XM~c!oZn3tmUw+>A;VOrv>ZAfx1ALxa5q9+C*%B`#~&oDrhlJ-r~R$&f<%0n z)-v1!%idN0!!P>hW1Q|PNUT447`;84;oP%IDcdiGN7DQcS^=6P9NOEuYRfGWe=_p+ zv>8Zoo&N)UP+8qa*y$e=4)xeqUqc5<23F#+MihWZ-D7pI%Mdu4^$?397+1DrE}Y=*2q3;^e}8A zp!`|LxA20P8D^h7O2$5GZ;a4RuCud|{EP_IGb6Pm#uhJ+?DX(8B)9kL?d;SiZFY{V zK!c}8ot!++?aRp%d#D^=_~qjtN@pLs?jL(xMtcXngC(DT$G;qQ=8s>_d~^0Ts@TdI z8anVZ`Hx?AH~5#nr#*e|e6~UR*PowtAL=~Z`R~v_H7EY=IG!_i9Lrmm@;9eO<(1O` zX_#omMAwEZYaoFzyf`;_d-(KbTl?sCU?6x4ET%keix!9inZnF|#SJy><%{@mwslFM zz1U%%e5a;3_8WaH^cvpwMZ0nHJoLvuEf*+We#|9B%gWR!<|;}_e`19F5g>XES_NT- z%QuXv|0YP=;6o=?W3y1#!wn{h!|d=4*4ieGF<`L=fvlvhHKZ_YwVj)rBo#!;?I@%z zm#4_E1evv04L547SrW}gde-`nxNQmqQ61=VLZ;(Y$B)69St2^>>rltWny-yn?vc|K zyS}n^h*S~HDI5x~+>|uFv>yXZY|j2Bz-N%r$flQ8?tjgb@@y;cU6+#+aDVfPi`?yj zA`&JLy_Kj=#pjJJcWIFu&+#Z420PJ)PcV5ChuZjB57)nboSiIGne=JZO1OHBJ(L67 zFe!yUs`8A=9v&ZzWAV<*4W%OPK171j8M31|JP`7~)UNI%0$y=W)s#nYD#?w975s1TcHp^f$K zNggDrwrEm?ig0S?d4_9NS*gtDNp7BR2629sCTUUYuD)P>^Xrg*Dk5!1fm7!4U=??{)gKNost;>(C!j-z+~89!(9G z>o6=k7EdLXqPSbhLAR*8Z>(u+-wRFwRn0T1sUfnoD;r))Tgbzol1lt22NUR*SG9Fe`!FOw(^$QApgskZ{19H$-ukGMkr!UK{wY_|}G*tDd%em7@`Ryz+ zd-D$d+E{JLoQJyx*KGDhIKZg7vq`0r0LM6vy-r|n^fS?8WO!4d4?Im-cZ351{1heN z$B?hi18KPmF1N_G&P;S6#G_?JHW>6E7~V3Ww4EX^5uC)^-W)5Vwphg%$M#|Jkj`G# zjtf0W2E#Tg=_9lM+JW+7$~_Q0NEdyBMD92pn7j?>uWVZ6z0Bx#1jY36W_P|hc@`PC zM=;#THre-2^(Fv6t9x{wWi&}^QRz`;h*$X7svOoATh9E1kMsY2xSB`02=%6E)YH-M9OGya9 zd}TLKM*JJ2!x;%_LR=uK$t@*=!}uTWf*V`!)896_Dije)&W@ZxFU%+G@A{E(ygq&s zQSvp;LSc-^+&oy;`f|Cz(si{!Ie4m{M(s zUbQmX6q4>t&H*D*Zs|ugMrczjFzE`3P<8}o`hV&6-Gy}hi+|Xv`Q{mnn0cqiu)daP z@c5nGrO-HDQr%~$gZx{$9)fM*A+>|HV$kb$hN}7wsM+OVlR;xnh4h13fe!MF7aq&@|OB87T%Pn z_l(^*MpA8pBOXdjC{JOFy>aQfiCl2EjZO4gg|dYK;)#$@IFiKsF(*M*$+m!P3>L?4 z3Z1sE0S!{MBEEw%&9E}#^o^0B!8DSqpt0=>9n=bc3QW`XOZ=FO z#4P*1-A`<$?S;>Ol&!EN;g#w0rj>BU_d}a&>zCz7H3bZ&wt-Wmb8-kvrJBs6KfA=B zB3HFRS^EfoYM-kt3#unDS-iw?UOiFLSMjSMD}JhlAXqTcPwM zcFo+e9YqC>8i9fp^kaM5f|S*hELUnN@qV2UVr<6austO&U$Ix1bS>8_rRxh};e87g zW}jRCS~{DDI|QsW`^E8m{OP~O?ZIBC;WV9z%%JCf`L#2kLk1ly!4a^?_={11D0l7t?42tiooiUN zXc@z6wONNj1>RjeG17QiFw&U4Kad#Xu9D~f$z%#eylzq3<=H)F7iI&IsYVAaneBep zaRyyuycN=?;SxK$H*oy=G2mwG{UUFs8O8jJQ3h)!&*b0<$JBy^Hw@qi@EVWpEsrB`z)_#kn`IcK#IwVF9jPWkR4 zFGH%v3t@F}1M|YelXHlkZ-;yB-GZ$9B?%7-@w0XIn@jJ}HG=*FxF1KKZlogY16JU= zNy|bX#FyacW7yFYeSvqI7>X={&6dj_^rB>^Rwihw1HKhNW~{PTJ~rNaAOX#tI&U7h z@o9WPR_7uJ8KB2K%HXKro#3+pwSJg34pD!;MvFb*Fxgg-SKuv}Wd$}a9$*X23qx;5 z?&1?Ch1<7`+hm{{Jq8b8Sbz_u3o8sMjEK8VdrlM|z7#=mRj3%4V#LOel?M#_~{mjIB|NP{s(E`iD&n+qMYzjN7;WdjK_H zDgA-!3XB${!L>y^xdNC9MgN>Zy|T7(3@AJ{`qMlowk)=?P{R`{_2O&fJ}&yvV0p{J$)kh4xu!0@5=1f(+=NX>)O^Z&Wyt-FPY?BIgUW^TpqdyXH$d{05G-@bUtTkC(0 z>TVxa*%zFmsTn*L9dpYQ?ns=e_rrZ`UJdAjcAgj2yH2Q8gv?GOO)QC$_&!pK zt}}S40X6c0Y{YXM|4R4JW&b$+kr13?q3&EeP8o-G1yRm%HMfK|^J@%Pa}7{H-DTxX zdMtF@4)BX=?t_XLi9Iz%K$j&DX5m9+o1}$??FH%N=pqCAvB?2g9Xz1IM!)^ZVjxlY zZ;_gBJ4RW8uapViP}SRo7H!cJi<KR2=WRb z{jeY=XX#L*Ypq*Yc^%crPx7L8z;=lFrcLpC(MCxg$-R9amLX+W?ffad#ry{e^4eqi z&e`vr^1CyhHkwXX1~zEMh^VWZ8|HIZ+W-^_HQHL^78&@kcy;zL;RH-Ema=KQ2L*#47<;3eu zXx470I)*P^J|BYfibjnvko;9k65*GmPS7-G!~E^NgrTV$PElVtQs>r`B0?D_cKcJz zUqMXK*8~~==*TDNLOtcpu5(@XJ>Y0Jgo;Q_F zmCl@fQ=}`eFBU?h7O!F?*|rY?Rqd9@b?p`l=lt?R_lFx&O%C|9WIk4OByv;axpppa z7Nq4L1;vXA9?=FRUR&?AKk2XVlaC##^lZIeo`KLF&Ucd^#sKud$)1A4AO>yvJ##vb zSE|`M&cy_Mz5#VoqLQ4_{pH^6zMzrK*y9Mmt+s{Bx=$_3Id_)#o(zvTNJ_Rq1oM;R z0od~W?CZ;saf}ixPKI>tW38l1MZhRu>K!u4C9EUtf7N#B%=SRY&Yy1dMSmQTWF8}? z#Iba6?9vF4Dyo1M8q+xuh+q2HxTI(iA$+Hv`6OQ=DE_9Khi_>&il&$mlnO^B%6a@4 z=ViUh@VF)@1sdi=_b@ln*6%;;%)R9lMq_Bbb^Ua}5c-R&IxiyLzRM!q1I%kiYLuME zDcT~j()6A?6mi9BAh^PZNQwC>&`4<4-@N`_QnV2lk7|HI_kg#*`&prmC<>#PWbu0i zXywg9%{hfwi4>OdgyAwRp^?+3;)u8PUpg*c!KZ$U8e-fHZdG*G%#>(8%1;{t-F1d6 zzY=~-I?`sWIUhQaoj}=dNyK*0e&6k{!&*9zaMaK1YyF9Op;gusSzuYF^PM-azr>re z3%hE~IQlS|)*$g4Qa0CPj1`pkZ}tQt1e!ddmCoQ`g`cn-3UtDgHsM|?V{G32=s6Wa zH`6tTyGmnfE8t@0?Lrjdv+fn%Kbsg`3UZl;7~fygz(am(M(_ zYn3v_JyfC~T!<}u9c-LnoW_a^+qqgu9T_vOly(Zp*VKp9JX3N$?!qydJRIi_UzZRH zG*uAx4Mwo~_-qorC~9rTb*x;cG4v@}|7U-x8kXO*OW_teV90 zYt7io$tx9KhslRbhf>z+if>C~bf$UbY#|T#lCW?ybL6lz$G9-##W6)bWGYro>Z4{a zhwQc~!?j>4Z`azDf>ODcW1ZBo~Aif#Q0U0Oa{m)7mPE;kMo_tEmy5Q zr2G?l@Cj8k%>$62K0qx;$Jhj~?0f^v=|m|1vQ>YM?AWW&7==C;=bBJCCEg^S@$4#Q^7$xZ`ne%!Q(@q1k?#C7Si?l%$?uWkR>7!R+M zDut1PoQ;O5j=-Za7=J%lP8QTRL{7!5jpr9t zauO2MS68Q*u5=c(+-Q*H-Q*N0ourp;;Myql#)}DW`{rGZ76+TZ6hq`ooX!fnd+l*F zW0|D6xBls&w%iq#frZ7cJi*& z1MH=0TZa|+K4{_!U0JlaDQaq)gF1DQeMtgB+-_s&$XpwJJpV=yv#eZt2e%y-J}bG! zY>mN`xm6^Py1rrqgLq4WE*REc#c8E;oV3lY!p-jYYtCr1r{yB&aRh-_BJ3X;B+=SK`xFoLGud_wWVCbr=Yiz37w7_U z1N=nr9R8K!N9!2Ie|xs{7D+oTJ?wNCETUjqAsSw)UC~Msu#a=xjpUc37ES(o6nKfC zli{Dp%EN#+*0G;Vsy7bp0qo3}?z{BRXL_r95z{}^P4oZGW*5aM94eec9=hp*F5m7K z8vXN%>H1`%#bj*9TYYUG*W{3ekM=3X#u3nKT)T{Mktr#3qTg6=jW~904zv2EbbWks z;^T5X=))1qgT8@s5#Fv@Xgxnrf7OuOIs13e$7j8^{PnUiA+~+cD%gH(vkxp=ZJry^ z-*?*$6Vg}Y^aN7-|z{hDLF_^ru2?PD(VY zK<)L{%pq~aJ2n3BwQIsOuH7;%Mq&#RrpYC?35|)yVDPwu$Cog&m;QKN)PP6YVF zMU#%L3V%H>Xxb%iP%Di+9*||+{1&8usG&7DedfRdiR<>MkzBWIE5$8<T!cN0=%=65zo2{S>rrg%&ueZ* z$@HuFLAe}D#b$Xvej`u4-YwlVCkrFFe7VV(U)7*Mm*(;;8KVevEn&VtF43EHK=H;k zOr~kPvOd34__9e^L}Wu|pM)Lwj_XdCe(4~V9(rg{ps2?0^fo+P3bF}QNCoQX8&@ta zjt3Q-F-n~iyKKg7gdq2BIss=6y;UGkYqnp!%2?)|<=mYllpxE&Ix1a3i_Tc2!r-Rb$xi@AC8Lk)#raK73b>MZRUIzQ z7c4p7-HWAdxMD^`F;&nQQ0wY$ky$vM*!myCH~>fB=w_34es=|nLX zCg+lQ;LTQ!C(QPU;zcYg@Y>1h?I4yiWyTL!X)NTn;riN zC>i$?XQpYJ->+%#Uijj4@7GjzDM4(bgR3)-D$|v*(Y|AbCGUI}0E5 z0n{~+e${UziP$XW-*An)o72u|4JtJ=3VA!VRR1-l9~fq%mo_NUGTd3xqr(H(RxAx0 z%Mtj~tZ0diz!vYw1?UFSry~afA_@`qtIQg}OJ@PbSGGXrpt?P-8!66!Lcd%2Cul_{Zxr zmsU51K4|Iym&-F%pD&xbRym%}9Bb*q=6#qF$*C*xFG3ZO@HzLle}89=R+G+VQlBlk zxku^20GEG6Ex#{GS5AVD0bWE@#aV)CV-t+fi?)hHqA&Nov2c>Zw#Q&qznK=gK+7?k zozILLd(-1EQ*j?^N}Tm4%svH! zeNpig5`M?|w1lQ?rTjZ%r4iD91vDHU*~sZ*wj6D20oG@%CCO=n88^YNUpD=JPnmB| zxos#s3X|7+we?euo6@wnU^+Q@(F>CL8F75>l6d3sr!5LsvG>I78|a7WOk2KRd6M@u z=--v#r^9%$(hgks+5B~%x6aIXk|6#22Yr~bhUJK-yV_cfmHYlzKfDNisFm(>`JYEW zOu4u9Cxu#0C2;N0_~0d&)zacnjtL^hJjI!YY5g?saJ~ggcvc4WcUHY%da#s7@u@48 zVKs5-Is2^)iM`#co`!=bxZ()hOSSOi=)P<_d-5B-q?V8*t}PQ>wO|H>4o!7)jg8L9 z(prLPYLFsCKNKKfg!dSXfYUB4-MO6(g<2UR5=i1besTuB3@^Oj;NMOBR%bZ!zp2m? zX#9s9jD)SY*8ju)`q1uQ}l*! z8_jU$Q-Sspj@9&DDVXwshFcl$@$hC<_nKvD%_~UHSLPLn`#AWNYWwgsF1Uv-*>@^Y z`d`3I4u?o5$j0z4V|@~jK;r%1(ax$6)h=7@(jl#`K5!t0qzwKMls9$L-f|n_y@#8j z-zs-!YQ0bsw;}I{g8K-^lRVpU`mMrMm0M+R)Bd{F#X5;?4t`3WJ%24B$=5mY1TdP#m7_FxB=9Bk`x!NS&QIRn#!=shDOd%H{isJ=+HP{w zY<6#s1u6Yt5dgSa(llGy*r1YMtKLi8t-f4iWEMIJ zaq8S#>@B!o9w}S2L+hrCme}_umnGfgsf3#+ zymjc})JD=y=dxd>eC~kHi(mlr${NZ@>#2#a%av;7V%fhQh#QmGkDC@-|CZ*{6`8Wg zJtw~wKLfMEGhjXU+)(ri+Nc}NdP31s9O&7WYM3Nm8c4u7zQ2d{FbY(AZu->uiuj66 z@lhOP>-{jT8>zAxd|Ivh8c3_Jv$DS2z!6ZJk!Kb``X8qM+4}5gyH}nk?vn}dq{8Xk z=LVdY56JJZHVEqx7_-)LA{JgTklgD;3vH!O;UiF+P)DWdwF}c9t$Q<%$sE%}{Erd2_Z1!f#2imyS0Nf^5t;Dv)foo5*^= z31CHrGv$k{+Y;08*w|jsl%pBgp58GD^+C)-?^^KwAdcv?7d4*5iIkvddNdsE$m6eX zzyr}TZX04kFy7d!Www<|1hUNf9O_JxRc^#mcJ*bf~@!ox|Ch@3EF`_f5kEtr3~P0o0@ z=)IWzsVP3Mcgr=65!99R=e6N5tm6lFKLj{&;DuAnVEtV)1Hqxk+##6zWGpXvTvRfrrNN%Ytc@w~?S~EyQtg||)7)}++JG${Jge6DX&7u?+H7o9r zST%KYqwR=Al&!w42e)9a>|?LJr8_K%bJx%a>EtR84I#JMolXycM%6|u6ya_W&H(^# zPnb*2Xjz|e9maWQgqX+2*$$pW97S<5cwUY~TO9leCjR4+H}eili0JLeF;>zdv~BZY zKsea}POiZ=!;@W`{S^l?C{(8EH=3JP^=vKrL!xTn0j;7#;Pgw``Yx8VP@E4u&`yT7 zZ-Sp#totdV3WdSAqL)7U={l%wF8lo7({~tSmZ~Ymwh6^jitRlw!Bo2Hvgy~eNu#56 zTy`+%vCp+#X@f1r^cwHHFW`o~~{Bilqrnn8k6h*oCsTrHFv4~5aC4QtQ?`hl_5+8Pm*63118*WeH zGZo(eLlv3)(RHSP$pmD#33wvTVpOSz&WUEiJ+>rF3uA2YQ4-#o<_KrSjXM<0v|LkBW zy3Rcn<;oRVpX4tXsvaD2ph z}*;#Jc^?&A-gXix9bJtBMM?u$zTJk)Y zX({)@@mxT;W>Q78@XI(x&{_8k#(+P!vWf=xU;$WU&8z9k$gpi>(CKK8J@B?g9z@|( z9)%B%>cjVc51fjXQNH~gM^Bp){U5nfeI;#L+SB(Yt+zu3l_>#s(&f_IXIFMloM=9J z&ywTth{qzXB}w?Rt^M-w&vlwK{qxx>fpQ-m&RVPaydHpW0{2g?jg~Yd?VYA5l2p@$ z4$nSKz|oR51$Pn{zD4&hK$oaUa4`O>1*kuUux3*kk_GF+5Gpyh_kSKr+y z>w7m6f3kEQ`Dlv>pMAd36vtNr<8uqzw*&SWP75i`ZQ&XlYyFx#;`%0#ER^~`mQIHA zS;dMBTrMs;^;$a^OTEX)JrL6qN$_6Grg@HfyP_&WIpr2i&qk&PTiE5a=2 zgr@D#M=Ij0_ise?T0!W}G|i3K`@f{g`ko zt2J@-9gS*Z+ndSo_~KBTDXxMN?;n$-9^mrrgJ&LUD!**7Ige0hC&3`>a;!225-b~K z5oL<9l8Y!Pki1I=-(oU;k5z0>$|bB>9TrDl@9>$lq(m7No`*!hPuW@{JG{piO~4~f zl$(pE3NJlf-%HE~NLSCN$ zUa8gwR_XxZvMl-i?l+Gj0!y2&uwEtiPK`Et!-?Ew(^F}56~lJRqpr_Dt}#>C1wIL~ zlUK~-uy?2aJ#{D?ZR-6*iE`Dgfd`;~LQ?aX$Bf!Bp2`7pLsXhhY-a>=(OawU%DF`I zn^NMm4DvZ>&O{l&sz%QSTp3MDj`bU2tcilWW3u_Zk-}PIR`ujFvlbYhOX3Tjr%inW z{<-wcL>iqeN%1>h%lT{P5LAMq&*rp$%26!qHfuziwMO2?mf0{PS_C8_`1I<&8#CY{ zouu2Ops$c%1VN1G^@?(hE~|{cN=mA`K$pe#BFuk|?wnGV6VDf!xiaR0E~-=~puW%N zX{|m~iPAoLd0%MyUvhz+YsPbof4V3JnV_LY_E0{qwd*`UdC$^;mdX|gbaYlV?OIK9 zjYXYNT-`Ha)jwIEzfm*1sn2zWXuA)^B@c?uX8ulcKn0KeAvgQIH>o8c-uqB|N3W;* z&qCiJA7TUNgZ}e*oyps0mxR{}x5W%w(y$G9<96{VK*yS&plq8KBO=3X!dIRgR@1w0 zB@#h#wkP&L4>##d&(GUdOF|VQgeOJTj;nP|I+~NTgUgQP#jkE4=03Dptq3Mw@k9w9 z+veoo{~t}~;gtjh}g}F$r%-katWwgu`)5=nF zKqo8exHZKIF5H4;?u`PPq5_JbjNiNSy?%cK*Y)Okp8Ivb?$Qm7TC=nMb2A+7mf?`H z>#lm934UHH`TXa`5{tDJwq*JSP3Xpo{%n^$80v*$=(^X9qYlsxiF5_mNq{Dqu>5poFeP zi6Z#&=M3!`pO9;l)_A(VdjTaTm8nq(_3%5xczz!3YCjia*7UlrWl+~GtjHqsw=(HUWxH+%3d}Ji z!I=uz|NT1kMc%sUam61@fkUbK7Xdi*U9Fo-q={|FntOS5o#RZ!F;d!GFsIe{)?3i{ zvB8LVm?g4nWzc%q#W}XE0&=;A zlQXS@<>~LlT;g=&zm?Q9no6|z1G7&HO3K%KTxnZ8_9tY@pEqf(Jo`pf*DJm9TKDhV z*qF?gqj;aQHuRfG@KAfwHa{Mc+Sq+SQg0rn;`-fHeSJ~ldjh6wdKjDK(Kq8S?5)7u zTd%ogiNjq?^6$cEDZ9x6^7srhfB9=8v#ey+36Y__LDBd0p=VnO^mYW<|$eFS? zzR7Cg-7zVuB&v0PLd_V2N+$_4X5__15~a>3t0oCD+HQ(yb&mRP|c3k>*ZTCQ6?)+j_d z3ezCglO1>qZ^=`B`#6rX%K<&IX3`QqllHi>ydO%7@f)t>Q3LwvjDD7(hgmyT(gybn zZWg*t?VDtOw29U5vn=PtCcl0Rd~msHj!-i`@s;QiwCET6{A*|ovRgcHd}$8O?Llaj z&<9c~6A_w*D<8@bF_VSWkh|8%?pR@g{;S(49eDPC;L6Hh6<^?%2`Blz1_k+nbQDLk zmPa=;Noe|y-SbTisi;OaU47v*ymWTGTy? z?=htJl-q@{m|n|OhH}Y7vd~KDBgGa~%hX`oeI2Tgx-lV^@tRLiupzw~9#(=l%DFeL<6)nz3yp@Fz zfy#1cks31f6SSJHoqXur_G-J)DUaXje>q%{-lww{vATK_M4g4(Xc<`Ht z(Df3iwR8}^?i0jrbQ&9zEkuHW=(b;%$}*yQq5sTkpp46^@vDNB&x?4)de9{FvBPF! z{MI8+f=z1QK-PmrFMbjP_5cB?)fqQ$lU2z472OOz=>iXiy8Z=xebPcrywt zj-&RiVQKcYh4mKc$P}kc`W(3xbA)73uA~{B7ZV`78=w*xqKO#M+!H`*|ABMdE@?}; ze+ps4@kz>$KcqL;e2gOoMxYXF4a@&gb7+yiZqE0K%tX3*vY23@z&DQxN@rZA7H86W z;=G<_dTm8C^${;t_Z`N^1QM6LIiA_0OnvwSMLPSVq<`#fPRcE2lwi7AB5I3{@eNEU zmh!NUrRsMU-dOS@VC9y#8(rqYhsnP9Vd$Nxn5>}Sp5`Wg=}0>CtO@^Hg}4-SxRz7q z-`KdcFWwdA+3uG5hmg^cTO2XBpbPiezpl8y{Hwqv0J4#^Bu$1>Ffz@wzqtutqQASK+6Fax zMXj2j4PIS27>SyV4tGyEgNKUzoCvcJ6uk)%x{&A$l~)G?pWODpft{eE z(YZ_i17^~lGjBmwr-L=mW*_OS`q+nz%&@s;9EDPC3CT*jV8#l^RD1@0@ zYqek0T{PZd)6&7D{u?gh2eB*U-6@OiKI1V`<-SUriF<{jgn`465ll_nRE5*Jh2)L` zGOQ6ZJ8u2o!(RCF>=VZQDd4A!1K`a%AdUZ~e;}ma{Fk&qF)>PisBALaJHUEA zdk`XfPQPfMlqfu)DDm}pnf55S>agL!()=0Uu-i60>Fw_fJ( z#F}hXX_Ua6G2fTb;+?$32H^U@^IylM-HT!Og2+Ke9bIGPC^2Qk)5wDDB;pJbNlsRt z%X~r(=HfU9{WSad6bPJj6z7#Ek#O?>Nl>SRM4Ku)qrBBaW6qdXR4iUWEj&d3P+3MsI z*!gJWnIH)3SF7(|3>7k^_ z--l`GUqA>1jYF+Htg5w1X%jsv##GMVw2 z^bJB5j&M{Ycy~>DjsFM+iT0%E&YQK6rNZ9v>?G7c^1RPA1_Q!SP}Dt)gJ?f)ki zG;iiJTlx`E@&zi#ct3XchP?oK(rbbF?_V=ER#%As3cGiqa}u%4_Ebrd#wIKG`7tim zu1Ozd^1t@k$am9LSY61lC85w4VxV6k{z3d^@QKr|wh+3DS$*JsigFcJb-1p_YBoPY!h|xGq}3u5IGlQKAw(&#nI@d#Fia zs=W32EiMGde?FjbW$WBb`GTl4qpPTlGnKuLBChKOF*U{o?j31fV~tv%YIe@AF`e(M zU4GmKT~{)&zV?)45+Hnyb=xwY85CHKImBBu0N*T!ee&Z0$u96A{Pjg-Uf67Sr6HnV z=3o8%`fNk1(zAxq!$yqg_Wr#EhH)hEM?6L>0Z8IT1dYv$Q_MNH3i}HLe*P5+lnwvAp6 z*>&F1iGm~TUeJq02Fa~k5YlS=tuon6M=~$7Up`h{(W)wIoY=goy0AA%kQ7Ce=8j7Dw-3MB%!g?q*Nvi5^W zk9U+$ybW$}D&{z{{;6E_eoS`~)TWq6juh2~k2ha_d6`<^52!$$N3MTDnYU)XR}EVl z-tfFA!fTJ;WWSzL|)GzMYMNJ&l+DG8F}pg(NDL$vW?Qa z?KBk3yB6!jAx~2*#f;vjnZ`GuDRBlVIwg!%&fSIZ`S^anjjLli<0k)mwokXk!+nCY z;)|>!_F>pHwV3v}+q`r{m9QoQA!7LlluKU=8Ovr0iREf0PI4D+1rAA5SdNbvK)B-9 zwc$S00YlOn+kr;14jYy3in_9Wg-mM>VLUR=Vo?dhdONa)WzQlPDl3l`U@j0DrIohX zmFBMCMhbwdO&;=GnG8{fIfDBJhZiw37j6tLpl1RA|3d~`qDMwYV*@2Vyr1Bfe!F)P z)0LZTVCbomcpmtS?B`*GN(b@$#i# z+h`6CGxcc_{ah~#6k$!uh2sq#tY;KTGT8CKY8M2Mhw>8B34_P724Hh!alJrweE-6! z(c9&!OVyCC?x`AL-XTd<3AOrCwow_gr1^LX2*TVINpzJPKy+2Gy~7(%BM zjNBY{60$w;EXlhia@eaq75UwMwBhxD`IKtiYsSY{qeBUn27S9xenXhgOPwQQxXIVx zlF=Lsm$*r9v^flvc0$TdU?zmH2u4Yw-xVXK5z`0pqWO#i^OO5x_!-Ii)y+L=l|)35 z_%J@qLc`9AKAaWtlb7E(-QpI+sFGAO{#fU!jq8Jw4CuSJuXTS)jI5v$P4f> z7>&<_%>dE0m{CVWc+O~{!ZfSmeTO8w>s}Uwzasm0Z&S5zx~G=l zmtBjFL$q@Z;1lKIN*cejye6Zux(hKdOL0q0vqcF!i8)aa-mhb-&A{%ZARoatV0ct5 zV*i3VR>7ECRdLlstY;v0%(dMeW_^ux)3mGMbp^Q|!Sx=X2{PL8%}&7r4LDz;V}y)r zChO_RHIsF5f<@G5uG<0lUwi(LT*&?ZE`V4ca9gf)z8`n@y>#LR{0Gj`HQwtqDt~Q8 z2oo(y!I2LF&q9_T+l3{Cnuhe;N5osh=FOG^l%E9R9(e+g=RF~WU+)c^1TR>RvPtp4X1WlGv!vb^mX-(#QkT*tTbi?;ry`tj8-%I{n(=BQwZub@Pt3)4x5oV6IpK=;T3O{=TtldX@y%xA`~G9 z!wqNROPSQBt#{}vsIXi&A?PQdA=l1}WOM}xjUw6Z z;~v^Aq##c%`GIDu>JVrZa(#PBtFPOTNQrpCV@m1^rPQ$MO_`>(p&3WRq>Z#LrXF!j zBviFOOvz#m3q9?|Ya(IOpJjJZQ(J3CxlZUe$zEGcE%SXGlB8SHEghAWZ6))a1UOvG zeN!v22;i>!&r%`@qf_GjYT-b__FKVkzH)D|5;UrQ(vK$cQ!B)&v(_P|}5k`8bq1Wjth;DrCQw4W?9&EVqI3=n*tl6FrWn zH~F{?ZeenD`AMV^#!9!c>3wSer#%Z<`Gf03MbuQ@IFky-_y=%I;hmE$!-VW7ft9|F z)hmZ|`kHpcZP-`yA*?FtdbQ~oCuFc1V?^SCbQ2}{ILVViS98*{OdP3F)LtXsC3+_D zClp)s_mZh`ThcC`GuerYNKG(to6@Wt3WQIWX}f}$Kejan!JIo3UJnObQ@pahcnVT{+9OJJ*123rn>SOu~_vQL~`ca@4|l z-o@(qY1MViq2UT0*Ky9h-U>8n;EtqHA4r0~qWqs>=ZrA?Zq@P1zs!|9+|G>sF)#8t zp_|FxofLpue|y^&+!!JEgaXjPukT&!gAK3Hi>1g5VQoJ~cQ0B0L4(g2++*H)IRuy3 zE2aaL^>#cNpPg@8Ct_GjsxA6920LO{1Nh-_nQ}~#58aJsZ^8DT8)p8i-ze)`$BAAK zJvPN-9u071pMc?J;rKdVS)Zf4_d8ROm8NG?`@Cjq4{$C^!mltI^|9ae_u#q;D`g-N zz~fuMYI$L_CCcdAw@gLnfu^3h)*Ko(fU<~0Fn|~)G?^8QP_w}W@KUDQu&z}d;m!+Th53;x$IX%{l(XQOgFgJJEZk1c@nQ~%{e!{wm}+o zc}!`wW(0Q4x%*HIXd=c=$=_WF4tS|@tnV8yasVCERO{Q+4qBpY~K~?mHThjQf2?ZW~Lzc#vD#Gx^!ve?};utjqQGU&SMtXe+qr$SlK-UH={^PFFk{1*9V8e%Pk+* zw?)ds<%SFCl1_czw1?MlcJKox9QG+j}+!i zCZsq|X^_(RZxo`t#K)K$m*-+gF}m5=tce5a>#SAvl&toL_oLZDT&XWnpgvU1=|w^is)E-?QPMrG+^cOw1hmkSy{dfQYIJa1Q9W;Tls*uXh@E|g{u zU7ox1TKF>{*?UVB$+R`{N~=~Y-FEk_w%u@?dTSvo3SM(v*FX_jI|&x8#O%KITz(=A zSH)A7GmI7(n-x+^D9VC7tx+IAFJyhs9`Hvn20_*ySs?T~(;^wTKsI_`tM<)1uI(Z^ z?$_R&4TizRFeaI@N;t*l_l1>0ylfH(h6rgabF5u3ehs0R{rH=sNy>%_f{kn|2JrV= zNP)8)Q|-R0`*Bi%~&cK4tfZxTbUQ z*JDj+pWExk+faO!ap9%O=29J?C{Pa&abz$`j_jU1PAd!lQy6>o(ya?Yu2(NjK1O3S z-O~#%y?aD}$U?3;9}||kxQ%xiUD_+qEHnYva@+<~)F#eV_WGDA2t76?OgMEm&8WQR z#>+WtnaYvxL574C`RJ`0k`|hWKD#WwPh> z>8|xYd$T`kWM;%F$yG+y$NcFWi<$5I<0MB1|yYO7onpE)e zoqZi+)F%6uXbHV`1>9`YW*;HQk)ZE19Q|JlCE<6WUCh5TaoG1O?fq*!jdjVkLTJCV z$PC+R4#+sRZS40+q1N6pwGMA>I?lb>ck5x5-e?8C7BA&@_C@i zgrEEQlPuII2&5Lto48DWrY^ z6yAkP3qHw+>;t4IX;l|g@@PCXpDa{uDck}SiSUA@5N1r<9HBG z>bI$hbv!=^Qo%tCmX9Ihek$z2>xVK@ju(T4YVcrjf?~NNja(U|@XkzcQDs3V*dXIa`gN zJzieW!Pksf4?k;nxp0k1y;m>$24AyRBk^~WLI|a{mBjg6>ts8Z4kVUnrO{Hj6)O;< z-DoPywi&x^o`4FGuCIoG#ek?NAMl#?0i`hc zVxzfi&|HtY3|z1JdgR-EsAdt#qHg5K_fo}FX~c6bcO6GxkyH@J=iD@FC($!@E%+yH z)2CA(Ca2O0Z3O!WOQ$fu{@%cXt>TWW_-E^liZ!**H=0-tXsxL5Eu*@5l`5{gc3gFy zct4~RIS_D0*f28mr9+TUEZ*?M=}L%8tP&T}DM{*3!l#KeH%^a#of43mw&WhUyQZg~ zwQ}P~5qpf|RA6|#9kx|n%@+qWSV~{~4zX5&H<{m`-%XlN0gX@2wl!WKXfY)mxtRbM z0dzEPg$Jfe4#2mR0w8vn=80}|j%uLUYPbN+3hCSrnin`j%6(o0D(NjJ++O=cf*J2= zpoMYOo&t?iLw)YSJm(bu6MGrw#p{M@_<;W%g~3hL2CqNKfn`%yR$@ zN0Z>@?ZcfhxbHKV@IJ23!Dh6h^#M++n=51WnoY3l{J$aVGAj}?TKat5OmOE(KQomD zB{o~E=LLifab5H%mLDj#)i-K6)OK2}#UbfpY2Kk8GGO+qjvCsCi+{-|8}+8uf;MhZ z+vuuig^1zTC8*;o$5^a6Q}0?h(7u6zt&zt(c$O%EDgJt4f23LPVeGe*gXDY)d(aVRNZJ7=(;Cf^-w6B+tr?0&%~ z@S4!4DI&mn%xg7adyl%qx&_u>_8qnZDDR;)mZa5lqFlA|E$B&AV1eLgq5LO%&gNI~ zdFnrTgo8gu|lI9wYP zI5e+&0UUzNy0Xxr?v8yGXa{tQdT#uC(_2gULAfzCi|Z=V+L&n$b0MKy50eCy4SW15 z^zr2(MNpy&ccRmGi=#KS-v__;EJz1GVycL$d6qi5i87GU@igmon(xMl(5ZR@Topfy z6-0IGrnfB~g&Kp~eCAJ~nc_LQZPq6BXWPPj?yY%LSpmYKS?R-z(oTVZo9 zXUzl0nsQzD3|M=qhZANSMGdf?c5vo>P{Yj6(vPl;d@gOE@z!|y{9p%#0fR)rc1q}@ zRXAz3Q#wX74p*Kym9j48Y1Z~l=9l{AW16a?!!kjzm%}1joR4 z_(}4UV+9YKy`}YVx6v?}F2MISDd#3AtFem#cfEA~0NAeMD3H|M=M9Zixy!g?sV zGTs-8ZDQL#rMg=@T6NIt>G!XER>egL5Op7Dq48Lt2X8Q+RqKe%^7Hgfey5u$DQ3EC*P9l|Pjx&gHk_Q$Gh?iDPr*lD5KWb+aRuFLe4n|0KUjPRfHu7@)S}-=9 z&}|*-v*>ntq`ipRcwe3>T;^0p&%zsbFZrAiyp*nXG+KuELVwNNSf>7-*6AK-FSMU#nKonlDO5MF&CrTf!e9EMl|gHJo9a1V8a4&V!uDX-dXQ(B>!s7 ziOQdp!|*dn;)ev|5R_y`vvlY9H~WE#uNbbAy#CM{2Ux6~XK`uQFYu6(=k~+9;|Bjn zjo+5+EO5VkZapk*Y}r@MDAnJX+khUJ2cc4Jtr#Fsr@(6MtcdYrde-e)(*eB@gXt`Z z+hAbA%0M8v=L7{1N42jvEqyd?Lt+8wr_I1o7^)ozT=F}zN20pke-MN`u9=WJ(> zy;5Vf+DELEWpAu-YANQSn3*WOL!Mz7VGFV2>03Ycd@QF(Xf;Si`5JZV)d0lsRgq9<|6U22+Jy;|5rq zIMK+!9-wNZ(tY6~oXFA5B%YwRC+yf6owMMxFkzgmk``igbyNdKu(V!gdvl(7RKPky z>NR7+O6d+m)+0S|zh?f^IHNo12A-f{`*m^=o%;NCsQVgocB`0Pqfj5|FFj(Gp=1y6 z#&5EcCF#kQ|F@|7OFg+gVQ6fXy1EB|o~Z1l>~TqU2l72`6lioGFMKeEkjBG9q6RxW z$}x8L*3v|n&136ht9rNzW!wZ%vZ-M&f#eM+iz8R1rg#q1*e+NI(&FAA*5+n2_ov$_ zK|WQ#SUj%7I7CrhLVw9eq*b&&Ra1`nxfAmyd_nNJ-|E(B2;$uOA@w0S7JuJY z@hSU$1Qrx@JRm)@(lA0O75CuL7@CMB>_ANRzkM^|1+gcg~KI70pP!FVz{>(;_U6NN-`3!O{_q;U3ZS zUol-EC(C1~c})9Lw05zhZ>O0JP2#1;oc>=>CREJwY0M3Y8M4^!?V|*;nQZjH6p;lb z_h2h~h^PwYEcNHLLKiuNx9IbKxRf$ z&Z!~4WuE%i$u4NoMk`3Big+IV=cyKrAFT!pDqjKnPA#9WS%^_;oy5zA@c{-`tgkhd zKPVWRG2m#7e`zpHC!QsXdAov8pmi3|^WB*ekX6wNsK!^@F4q95DvEt|gd}P}rTq+4 z)i*!0z&vk~U1E9Pb9l8HlOD}%!^K{B^s6elZw!>XG)uiQsWGhb&Qa6pTGppyb8hN` zFPR!st*42^2;@Gbh^hHCxhM8RUM0xJl%?_(6%BOMj zxvJB7Db=jFHf3B^Lu2bKR0WmbQ4i(?SvKAyFN)JT6!jpjmMh0hSU188RrZK6hI;f}Vc^3M zT$tE19!I5bYqX~x z^3zR$wKSE9d87j_CWkYzj_f@PLvh};(yt{WHORdm-*I>OtiK2t;$6PLzN{AmqIN!qD6v&3aslNnYf>RehX^|Dcxc|EZs zxpuk1H0(98(#tH3N#E({T99grUEX~i1rtCq>$^UM8G@rewfuW;T*V&aZ2;7gY2fr5_k0ja@&<*DQml{DFkQ1o9{dF0 zxxoX)*95Lucjdz$e_LQf@si1506x&Cf25^6)F)v=-(uxLfQopzp<+=XUqq=)%ukWl zobdP1Rdq=}OC#EESRR7>Lk{;`yy8;-RCUS8PDkjqhe{A67$06N$CH2s1%|sfbth;= zQ)TJ=%fDZdAUsXymk5BDb}z#HrSj|%Tlqp za`}<$5t%^Q@Km#m8WXAx+plF2Rp{l63`#J%d6Pe>&vrpUW;86MrV3YMH`tv1Z2WyK zTCt_@_4ra>O=lTuEyhbPb*w3T`*(GRtT5o`NDsmf^{|*N;m=paT|MTKONk&I>hm`v$=n9M9`{soQKdeyV=EKd-hUg5aL01Z{2)f zE>WxS`c{wZ@puB#bwSLo`R1v%YKEHhY+UqY#nsc5*DoY49+(HsS?W|IP_V6sZFMs%fAvpyZpU?^-q%EB<*7?wTk zf=lNgUU`3vrz_G|Hp{LaGOwM>SDoG!Dbn=KHcZTY_7iCvD%@2ShT}LzuNp+>p5yI; z79>plLQHRYQL>%sD1iPPeUpA% zjj(a|kGv5$Mz@*`b)z+Gz-DRw79P*M9(!u`HlA=_T}?w@kw{6KTh3Fr0)#KOBaN0U zUfDAGz9yLNq^pt7=$>&6OeEAcJldUn|2ypJGn3W7N$&KWE+cD%@f?c{cy!%cz4eYc z?b7#gZC#MsDE4iy8MY2%cN+h7U%UDk3)onU`7JrKZl_4}gqfmNpt}@FEGrro>QZVh zUf?FVW9g1kL$xBW|KWc|B;5p!<)(X{vzoh$HBTeH6`U|x;ny*2=zBhuqn#`N{reMk z5>=9wePD5Cf3Z8JN?7sM?+1IcQ8U7``uwA8ndMoHSdUhb7m8e&j&xaV+JdW47 z4hHLCP3kAOrjY0r(7HW%4c!P$m)4&z$Fc4lu)L<5Ma*jJv2kGUYU$Lx5G|dcSqxSC zHymX7f-~lPlKE0}Zn6;>_zrixVRPJ*tn_(p%Ueoh58XXlY%EQu ztM>8@8M_xl&n8=W5o6A)sd+k$`E?|xcdg#&OL!X;gKpA~nDbyOFN(67y5tRb_UZDj zar=R{e8+6!l271Zin>EZxTFB;;T*#@-bJhZs(hZmL?NMBiE8D#uYIc4cqeq0*jQFl zH}SWK$|i$ZW^K9cLYhFH^z*h27dFrW4A0s@t)!E1X`C8d5Pj<97LwOF2@E5d{FXMu;f4- zk}7x;2MjajE29t-xwm-PVRppMax9+tn>z?Kh%xy={*jqTpz9l%Q5;z+HZo8su0_+%-TpT}E@ zt2ceBXs4NAsQ`}WpW)L#k~&@?p1o+o$$Y|cJ!zEdghmw zu~B9Hk+7+{*%S9eme6a*?|X8&>I+%65j8ib5q=}Yi-9shCgOSHw=Q$-DV56Ut&0)h z@AO_=dZVfL7NqL94Rx%w;fK=m_7kwl5{{KFC$g>()ZTd1nW1Juz z;)bE$0=5>n;CtwUL!J0Ar?>#9-?G#=T`7_?bup;$wtlrsVg9p^rPuSCO$)*~Zg&1& z({paBg#l_Ifp2ZVX~rgPTY{Uo0Q3iWAng@7_a6q{CqA&(NH7gornLrBf126wGiKbQ_Ae8JrQiH+TEiCk`uh((eU{@s;q@a|{4 zU+HfPuc7oivXPXrXv4wE_XRgv@3(shHERtbIu@M}-8o^%Bv$YBs3=3yFZtM2@%L&!>7Uy?0%mKvi7C;ml=4n`|(-+4E_o+G)5UsQC2PrAsih$FHS z9JqjJ3ANd6M7&Q>JHB3vKwl8v|4Y-bqG&ZE_F(@z@0QzTCpK_dF26UG|5hIj8eVfx zoznn~x|fu??|a-K>N+IOKlLIQ(ueSLFK~^Ey}YTeFEtIop~uqwvF<)Wpzrb7&qLMk zln-*%294yG4n80(-Ph7i?|?_JOp$?-1X@jZ{kN)$S*@PVbz`?Lun)(>76}4-!GDnyYb^ur1QkVc`8~L($}7Qy?=;e(ke~pPhWH?ht0=wP(teQQxa4C@URtQL+NE|9QuPHJ6B4VTVs>vloi} zaL@W>FeU5qXV>@~vG5B)k}*IbJv!L*>S30n@y{h<;)eE54B_8)5DN{6zhG zZ1((@#R)XOFvtAiLLySki4Rg4`{k_!<60KQd*YyS7Sux?urCgBgSz5MyN_1MVivk7 zezl=Y7dzu#7L)a}q{*jsPyj}NT@Vacbu2fFc_{SWs8ODr*okqD zGx8~Y_Yq4;=I!=3=`l#Nc&>RcKq5RzJ%?+Y5;TzX5~&uYbjPT<59$0a`Sim~-R3?5 zxzu{;7F)L&w0WIO;4(!-j6bd7|up?4Ze+{5N-dZL#R-p^u4$sEnsIQT*`kvClCqY7=D$5GA zsOz>hB2Tu^+64YB?o9fK$^W z&vHoW>dDiaw@N4H+@06&FI9vmIL!lb<`09?z1TOP%B;p2oz` zOOP&he^bV=7BC!2j15&Vq{OG8Po67X5Ab zRSxlgq8V>4Va%}ReR#p=*;y^vwe;B4t0w(lyO?2*)QHJc)-vSFi#do#MOX;JT5^ml zS_=jY>m?V>*LnS4Sk-M`5RNxV7zK0;67BYlJl@g!Wxu6pS8cY-`0=S2E4xTnS%=uy z=O%3Wro5Kzt6_Uu1sEjrIr0*e0T?EHd1>BU+Glf^MmB--n`inPja&Jci zpUk}Kfy3_K3juWo`@QVniH1B?S+>KHU##@so0Gfx*wJ>oFz+H&2w5~u741dM&u^jj zDkroUIkaWblHj_DogJvz!Z7{I#BcV+Q=?PfB&t`e%HDP(x#;(#*D(R>B?WnvH<^Dj zR4$5Z;>azVvjv-4?nKaEfiZc`j~u1+OmL~D6S`YKc%GkRB4=!n3yv4FTdioHp7qp# z;Im~E4Wl4`dJLnm4Nl0J5!_D34>e#7S3`_2`-xR_^N^m#uFamwOw8TnwXCCa2gpUo zK9p|CMS0|{40@`rX1Uq}!0N#k?7~bCv&<4$tFxPbdYji}laS409G8jITu8f;VHm28 zh6y)~|0OJSPZX?yf6`-Gxuvrj2IdeW7{M16!LQI~DY$}Xz4$7+Qa_s2^3^=7pOQXH6nEe}a16Bg8ZxXP&DVs(4O8tC_mGRY-1*6J7@Sr|dpG zCz+RKxccu|54{X)oak2TXJ)ZT+-+w6IO%oWu3i}&5{kd}a1R8^H{Sghc=sIJdUff# zXXZJ>z~Pwo;X%a*{>cv~`NQC~)A*UC(Q(;m2QrH~=e}qq<-_HQ($(_%a`)S{+^w5V zHJm@#_VB@`gIhMMKY99Y!@hT`4FTPEt9Gc&mw!EyPqkZ7w)>iQ1wNv7&)2ow`N2&7 zD}ko($Ax^t*`xIGs%r%aDt{b-Ds6hA%2B$y^N;hu`MuehR;!4~GU2*dXB}M6>LlOG zx8!Ooxc60hmg&oV^M5qk6g9TSk^5ov6vDYebcsJI=}>ujKh#qMOO? zehp>b;$z_O&g`&F!&uwl3xs5g`oA}PgLtof-;6u%7!Y*^`99G*=YZL~Dnso6*6o0M zn}+y>^G0>MSE|Avt(eN|C8DIu%5D4lp12jO5lss64@6Jwc52mlwb@=;>@iV)@VsM< zeNjz`$BVWdyeYW%tt5;ow7D+bRFu9N{R8~!^6~5Qdo8*5vyTM**tEVjG31;jq~02_ zI(BDIS6uXpA0tb&Zt4A7qpk(kBo6LTu+iS*fXQ{va$F!MhvVD!R=*xZP833dUo*0;3Qlrw8u9~XN(i?xi6>gqRHe)iBg(fWQ9`ht!3+B;Bp2g$P2$8E5`uP4f4CphBS z=}BE9ym-J3DTC$?2{VL z!d~*SOxxAQm0r$2<$WA6@yu!}x{5EZ-Z4}cC^WKEGcK8SH8xB812b^re9%?Ty*sMP zU)mkAJ^GS%@H6j!m{;ky&zWulWXOpDgs`Vky>Q-n8ey>$<{pR;*@X_DxU ztvNjVEsnXYdf0n|w@35#PoTcKth%Yk&aK6w?`K=IyL~PeZy^r-uxU>h^?0m#kI8#M z<;VM)bSP(b7vw*r?qo`ci#y@v`;w>sBv8odP8|P7hwL$~QyGf0AkxjH7MDLCJBRkF zX%~xcyXp34`6M-;dH&4v;7w;E?+q9YAMBQNt(vKYrni52;CGBuf5Osb_F$9tphN#X7yrmADo{!pP+xo|CJhaS^M!c%hH*n z-ET&XGHF(?Pxh-r_km@3PR;uKoEw99%Bt?5NHe!N4i(z2U3_wD=kA<{UQ<-TbK5Yl ztzK#&AuuFzK8f?ppnqdx?JeFnIvmQ?KAosx{fE8xWJZ|B+W({J-2a*W|NmdkOae8S)kW`8>|J-6j^6u@kYZyN zGB=Vkna8|~VDU==4@>LT_69^buH=Iz%)(9tcFt9H#EAEL<1(R9xI-Tq2L0Xn+)~XE z_a3D^F5XLj|MQo9WK7xczKNZYcfR`J`O}7-4|Tnnb*cVh z#mYIu!qxTc%lxJi2F1J2pds4EH^w}IKXcLZZt8+ozS?9D#(csZJtmW*=V}LX zHZngN`sPH3pOao}E0}`szvi}{Wyn>(?WC9htE-lEe<^9YnfOhzy$r+qqJ88YZ3I@I zxgy^C)pqW9ihRwBc@Vt}53SqiKDy_nxAgMi`I-lF{dUi+iMw^jWAK+@Uk-lghgN2Z zXgz^9?({iIeSWt8QvH~2<@A*8!RD6TMgJ0LZyG~eP<_FtxFNRK$NzoZX8Q2gp5}6d zZ6up*yWIZc$;#M``10j!t3kIE8RzsAY+6K$KGfbnZ@Bdw>dC<$DaSq{35a)DnwRV1 z0S?|ZrG20a91LI2I8eBgy2T~PW!l63{jN6AFVbZAt|`Cf_zVP0vG_6p4jI0w51Dm4b$_4u<|`Vf}=O z1;Nyy&EGwd>*MW1AJggs`F7}x!B&sSy_5!AmGo!9g^-IIMoZfBnz1W^snP2fQyLG3 zdc}f+T1=AL#GpB!UI)eXxiF_LkoFDAir< zMMW*EF*H0CxL376@ClrqNNabD2eeDnw#$6nmBS4%kQDV5SBJ8ZqZgP7y6L^3=Qu`) zjK#sZO0Bnu(vx$wn2xDfm-s(1pTQ4zs>*j&4EOhW;riR!>YfXSrkIC!;(toMeoF4h zSl{F=w5FpN+7L;l+HF(wUEqS-ve*;Cqhb4;_M`6GcVlAaMyHnD4?yZ>xp(BRI!A8` zpl+SUEp)gXkJ(Q;*kL!>U>|jy*~(pJWw(R?w9veN#C{|83lN{xaiVOe^pBo&%Pkdj zHR0$ze`PyXzErH(70pDFtKK{VU5MRFinU+pSZ^hl{nM03U$MgLmst#(QJH%5A6}~O zBgA>c)!6$yW&iecl11kZ*C~fckpki^FCd_K-)Ot$z}w%6Km|zr(43|A;PpnpGS0Vb@=w^@D&~?&UiM zg+BygKYFa!UVT@kDSaoG{#~FTPD=bq+-m(xYt+PYH5F1+1rF(7vtWZVYt%pQn;_c^ z`^|{uJnPY@!3?zx{KolT&)pFAnO1`sO(4h=D8y5f1UZ;;@&M(@seM+@Or1IhJlm6yOp`;NUo@5kNhsplB^pj?R?hxCV?lP1vB^ z+2cnj(nP!__V$*FY_O6I{fFU42rkd%dXmmNq8u}i3GB{ze(}lTOa7?SuBh+P>c2sI z?+?cs#8xZ=6~}UPv|@McrGjgSf#<`C4t6{C>QJ>y=oomTPWHC_D+!*IHRGNnkf}>DTN|C)s!s>xTv6$WYIPHDgF8yVyLCsuE*WI1!O#E| zxc;3Vn=wC40I$5T^;tN(r42=abkq{8kg9mtxm&}YNJz&jg^u6mwAnQPW2dca;ZanM ze+W~MmG5An6CXks9J9oc$mKg!texv^OaJsH5NcU;Vy<_mBALo67G7DxR z$R=a^s9J%8G>+w?ctm3!kFy&vhEaZgc|A6=Kj%yF&&kQIAN>Kq>uZ`X#NSi$SgEOZ zZ7&esM87zy8_b#Pe1fkU^*p)!DE&Y0OO^+}jyz?z^Hao6L4woP3<`(9dE1re@g2$2 z4{YK6yU(@?d-Q+|sWK9f{AFfnMrW}dI~xJ+m?1i@zHv-;Y{N0-jyC0X4;7ixYIL|Zxe>>wYgK+gf7dUx?-G}9Pi>070mAy2meoA_WcX>tweiGMLA{J zsKuSXoF*ccd~WiNRMe@EbHeWq3Jw#cqzaD72?eFgfXOdpH@$$Y&C!Ms_HTi#GSOxjg<}<5oX>fU9pZCN;!9>2BThK#MP!YO z5}P?>Lm&7_h$=rJ{)ZUEu~2UxKbYrreezPsaj<)H-ZtXxtIEdFgQvwE~=c>#qNr z83hJCQ5R-6xRLE~QDHkc)=lhYc4!X`mJ94Z4e&6F48A1=p)9^>Ozmy2{A@U6b-^Pw z`t#`)0-{n-i!Ke;hKO}Dc!DJx)?&@QKGJSg-*YHzdHrmNcdayRw@Hf zs)Nr4Gf9$ELi z)$0Wk+y4A)l;2JP{&JzK|G6nwrX)dXY~t{5Cor^rJe6Xo&8BnB+ z<^3ZFJ-GeCWc_Pmkv6wMx{i(X_HLhg>p>4$K`^3!XNm-D0#5S>ODSK1(`d-Bjtb)D zAHmpW;NP|1*cjT*KN4_s6i%y8DM!csAp=bA#5C5{ZWwiK8;!E!!`1OeeFLH+hliA- z&g@p%&1BF$qw-!U>hpHmKZEvP%s|BGZZvlHo}#qQ0@&j;&l6*nKpW-m+j=R%KJW?2_NUKdyrt*0@u=jg|TJx^v#Wrh0g?22(toZ-#E`u4B*YW=m&=k0|h3T*+)7 z^=uBUD}w3{MJaMGBa6_c zmejne4NfX}pdZO`8fTu(#{%Pv;WfuyOw&;U(Q$eL9>N#{#STUl8JoQ=zUG2;)L0OE)X~=UlQo{#Q2Pc^CIk`Faw0IM2pWB-?wlnz#7=N~F zWob!Xw_>5VB=E<;c_lGAy8@8>4EY@Rjdi+q zT6XOVr(;3*5UBIdkBtO`l2{3-%5U~f$Yv+;Fvwcjuz=SPX~gW6Y!UEqQ1Zht$9I7( znn|$vUbT=vF$(jG`Ls7H}K$g6AYxGf+>l(EC!nS574-9vl# zFsvMIUosQv^zUABc=x#8>w)~EC-tDM=A+kc;Ib~-5tF!65*F>2<*$@=($iQNvRq*_ z8@fGNftBU{ef`(97$7)GEE^}nRZcKG?A|W7J!N_$dvs%X_K|rkSkoohFLTlSxih?< zYZ@l^OO6=`v3uH0?g;3iOb~VkM?WzQgE9IXEN(o*0b{E13fNVj?szb_WcQ38oXnPtN)8R#kSV%$T6b9CfrW^EvcJ@Z+wZB| z3q56FBD^edxOjN`r>efx21=>qq{k_a6oblZ3dwAHxd(~^ed1(v%=&Kz|IiqPT^zIx}ywG$~9=Y*3b32!Q& zAvwe?#E@2IdRj7j%9ML@Dls_zY=LbaWgaGU?O{rr`(33c6m{6Q3A^VL=v)pS@T(aS zODj2o2a|mF_jh^JnJo%uHV+`px+eRja)y^JA>JStr8a``%u4?MSpZQt(ZwydqP81= zONvgr*D|qh-rk`IFWbtE{er{EKZ)+K{3$0UcU+yDvdz>p)e11G;mrJuaE4$J`C&?i zRN)xtxe(xG4ZMgn@oYi%QMwW{-Fgedt=s*s)Y|`N7cgk}tyTcGg!{zzh8N+W^-2K1 zWk9D7L46lnWAZh{nUj&7bMDWVom0E7Y5k?y)w&I(0ZZaW4d3CvkRUqS&8oX8h&_AK zp>_iWz({GG6D@WT^br0=mt+re@HtQ%>rzW^ET9FrPRNa&GCGUAEg^Opye%mT@?$j-SWsB!kYvQi9fvxS=GTMQ*&-w@-G1(gz}lf6jp_Jq^>68 zpRR+mhF8ipsFo`ad^{P_ZNl0_>vA)dG|{N8|J5ZzF!^t?^NKUXg%D{rz52Pbf;F;EuL)@N{i5v~#md#2n42g3MW*eIAfC``;ht$pN86il%ZSIP zP4Nb#qPvJ=+%5cU(jPALLs(K>tReF|E46)xrSLT!*tbAhiEqJWBx8~R6?}*F2ke5) zpEptOI@nsybqpwwAAr&75LLD&@qD3c4S1WbLvk08$y(s!t&+$bKpa@AL^69#4$NSN z6F$K0g5^Jg6!@HB>c$e^7Ro3&(0G%7UP`A#EqkvKTrV_-Y+G-ANVbKo-2GchFS~KZ z*agTQc)?^+UL1aoO_Pu9Tk?M%0wTPG{s;2MG=L)gTn5}No&prKp_eqzDIr1gLUKUv zg026)rIX%)0TCT3Vwiow?54|Q57{;Y>U`L62cPuzmiE@1MQVY1 zq*EJuFJ~yncHLA=B$q>R{W!|d0@<5RD=x)$i;68u`corPhOt7cKf;g~b(N$3GM@oF zkxbVEFTN?c#fzWQn@>OOKzx?*lJckznBOAg?E49x`F0nPV>n|=E3=g=l|>n>@b`me zJfVoFrMG(E>rP`YV}=>y`uZZgh|`2}weUX}tbUYs%I&2;gVa*WV z4GT+{#q4(+bRHHnK2?3M?pU|;gUFW-k0+LYQm|UoASwW-cLI;vldbMx-7Ozl*-G^` zX+=g({HKSo$+h(vO$4oOd0kF^s?;{Ge}lNP@L_k!459T#_ow4O@z6lf^tXs*YUI`j zqYoJ+0=atN5$YxKi9gut&|)PnkHMnG-yBxcwJjIrYV??sKIO9vb$oE^O`eM6ySu}( z3h}?AZvPVHXz-lXKvXt6C*QNr3UAnW+6?$Mt~*OG;@iCOYr4^%5@5rwH64d#gQqsK z{~0X*I$r+ss)!NAPfLckRxxv?Mldw-o+C+xCqSWit?GCq$zQh2#x256#&zERRe1zq8wN`g~$B{$Qf@LJm#3O|1s~CBt|{t_y7?LzgUJ z-iyyoCwurghYjjI=4GKnc?+w^JUAf9ETQlClakt88qRbuZG>L#7EHbmi_f zuSZ=6tbUzo!`(AkqCU6oW7I`Y6I`mAT{|#4hz_E-?k_{Zih)f3;X1IeYPp|Bjp{-h~t&ZY)-LcB}!N1#QFY-1KJ;F42|3&Bo$ z`(Zn?4=ae1JpO8j+89H8yUB7P$OJ7x9IasQNU&_!x9}SoK1f$AN{ZK$n6(XWLq@o^ zsIJ_RBY$_2Q3og4yAlT^dd_IQ#2Qx$H}4FGsF#XI4H4<*Sy7@i@4YXwhgZVS@f6x9 z#T^T=n|p|L@UL+{kT%t%`M-u&LXY_Chm< z4F@-$x1&V|WOv?iTl*kjp(?PB`P0?;$maQ%%U=gCRVeH7xMhCZ`;OghCjnl}();LR z&AS~G*rU7k0EZDJFN4($9fWkm%9JQ?Tz<7H?3*b3*5+6|`Cq`_mly@S4CkMe5`*Cg(lxBrkANr>2`Sw>l|P-TL|MKT2X0Tb2gFmz0<%^A5o$Tm z^CW%eq$xaE=N=YJ{H2Fa-i#{^wV($oF(|n2aPP$0orU~;D%7gho%biORcTy@cUT4$Sn=7PA0|44rOz=f6V1>||pZEe3aE z@LffI;U~-9nTfu+6RK$n^+I zE=NT59S!)ZG|2t!DjAoj5T%%#U-LRD{RAo35pG;@EA17ej^s&mk9@|#rTa-m?_H01cz|D> zrNHtPEe`z3jZ<@yK~9DK@8k25vt!<(4~#S|W+X}o1a5W%wzw;G34xz-<*vPy9V7zx zg_QQc4rB@RK9Xq))BbPa&#FI>9CyP@>63+m*lwC#}&Ua_o71-<9l{8TQJo_ZDyw)GzF zX>pmaTKJ?kwuX0h#<55Bf6CKp8hW8$la}w;=GP*w)N#cLs`K$E*gB1~>+D{yNvM9d z_Ev(3iB2Dv`ihP0gfRDm!Xblfwq?WAwO9HGsGDMnU3}_(>k{5!8EXOA+ zg{dF7lnQCaEf>e7P4+lJ9V=9H40_W0VRX797~IDy8-Oy1=7c7((u6ek<9;wb29XA(JK)cL6t0m5B1LAEuFU@bj4<7L%xoP~5PvH2fd@Y+DZza+7} zRq*x|ys`agr!MN7c)0(wl1nHYe)gb)l;3oSN&|*&!H-v5obLr13|r3kh(`tLQ5#qe zw~$jlfaZ}sR0-x4VTiMG8}8_X6pp6pK(wpWsxF51TuRY=*=I)=bmqBy7V!BTDgBp0 z5Myb(^c23)Ub!wV{zuv3-edmkMbNqk#r&EcWD`Tnx(JhfqqKOLC*a?3LS^Y{Le$^$ zt_}l%EVbrH>mMTj#WsYoj5u7rantAl@RhtDVA<04%h@Y4^BLLE)1<1VnbzI;3R{KV zi0Ie|(+ge(@WVu`V9Z85bvej5A|1@(A#Dp-=oiIFC;hkkhGl&^EhzAOYC-NO+~&Nw z^n%d#og&XvW4YaC%-@}PgM{vk*n@@B>=A+n1xIS>^!7DKV1;^eV(k4G)s2SQ1NClJ zY(O>gFnvp^@Q1%s-PMZFVpHKndRq%3TyKssOsh{qOkI$3pPox< z{*e=vTFUjis>fexq?`_H0=27ZUoeNIS^D=Fwd^0UlC~E2UVEOCq zR9=$^iV&#p&EcpRopqtuM8)y@2d5)7ldX*i!Bcdij&8Gg(!pk5i|#5n(g0E6LstN< zxx03`2`%8x;OXe3eVv^cisxNN946+joMUoxXhSRnI1-NqK7j*mW5<%i*J4dvYpQMi zxCI9Gb%Qr#Cg(OuS(+|TkDDWfW=J*9F^--~c$81Z2PQCVRVah}jnv%IqUsc{U_zp| zONiM@J2`fx{Ro5xGO>wSgAeBLv&_Y9fnKWvK-DHPwKHf8wx45e7}c_t&6=rQv` z$bA1>ZH#y;E?wNojz}jolsHpUIUR9AuF2%ztS=Twax^+KaeU=*XqeOlQ9r0I$4|37xRjxJMWBA~~Fhb^N^W zqI5(Au)N|*`@{?RzEO#uVrU=j?bX-Me@%poI&kcwEM?U0qu_>pemwHk%uakpImI}B$W>(!hpv{Hn>MFvmU z%?*tLdv$tPM8NLJW{F}c`q+orTegbFNYwi=KQeeYeMJ=$uAR~CQAtQ$L0S_VA^CN+ zF{}2Ib81{(>HocR_Rkh;C4wuLjJO!8wxL_QN`C4wS+ttnuyUKo7 zj@9O_SebTnd+xoKSFDZGkC15FOy!XYzn)=FM73(G|2#fB@kn|SH_@JK?L_>Wj}jrr z6?Z&*0o&N&HBi~Ven)`5mmB_ky0hF~ZCk@;11C1CRGT>TIWIyxT|IcD;gORjOLP+r zk9yf|-l#Glq!+T;it#{bl7CnKb+P766n`U&X>M`P5bLb!fUZRGBZ=K_A?rqDRdbn0 zwgFaK5|m0Mz^`y|9WQtF^wnM~-dht+Z(riB84|X-ffQaKns%wg4F-8M;n%eNm53_2 znnWVq##r11_1#`hG}PQ9U_~Vnr~rgb>bdO<8Z3;xz@A~3ALFeG&Gh@ zNw9lXHLD7u9@bZg&e%2#OPz2z4BAgf=tLb~+x4ca)lj#c7#MWSr4#Kxy#q+iyvaLz=@?)QUm<;@+e{Ue_ibvhrMelC4YUqf0YCy2>HnLddu z%Ez7E3&*$3{C%W?F2(@bg*`}|>j&G|S*0P%2T)nl=@()y&7j=Dh><=6o5N@p*9qOf zFrN~RI65HhCTB?Ps{K7SkY4_mIcCco+j(7X{D}UCl{|QN9_;>m$Q%0b>y?1TI`@&X zMo)jQwi8%$gUIB<*?I9n>k$8Bl77=|q)Px&-F=h$YP^0*99WN9JI-7CD#&amI#o=D zq>qvPz&F6|Lgs1a0n>egifj{&QMm*i)Lv zxtOff0w++5s4cMIWqy`0&Z%;e=M{1r_3|0H7ADicFFQ*hDB)g9W*P-)r9Z<98rUnr z?&xg?O0G?}iwNfKW!GdG7ESUpoAIb@FSYrQjv+Y{j}2@zMSEczu3=lVG&XS>Llf5g zgfS4GUjvUw&r35Em%l&s<#s-J7Egs_ zNtJ~S{nS1I-G*^j&x{DhK+(W(JM}jvK$dB!;=)LOex_TyGCONYw^;D(JqT+6c$P{1W%}e<1}r7jl|6;x4!sG=cP+%8GW6 zY<&FRG_Zj=aVKxZ9Y`JP>SZieH5d+_w$Q@sW{bFTw(@RTW@L7;igC7b!!^1{(i-HN zmPQ+qQ{Z>5m@;r{_Qg%_%E@311sU{mnl@fCF`1h|(_eopN*KhRn-%Snr_c7OdRLDB z6?AEAo7>tcn0OPedKGM^g-^6MEDoa@FIyL8!sYK@9B@D37u`qH)si|C_Lx%59tc(G zkU;K)1 z1xG)-o~%Nu|C4AWj!{0Pk?5wUN22x?83ZFACxQJ-+FrNO`ZAKal_pW0Rpu*?Y>%@v1=o#o#TU?zZceyc*9o|thLkJ- z@OV@Iz*(SzfU*hus2-I5#~6<>*g~mm3h{af)>kmY@L3TFE_Ik+jm?RMwSiwV`g_{P zJ>f;m7nQfIZWcl)^J{%hyusUWHVD9=DX~bM${liTud;?j{pmf?*655clKSJU{$%WU z?XC#LkSI>r5rY+HG;@c3iMekJJf3E3gKS*}@6`4A`pYc>&|jfR?|xH=5%g zm(7WpPQlJ*68}vfZywPCj~=AN?s{FoWU!(;x71>TpI-?7k}S0o0Iw4UHhtzl<9?UL z8W~876Q`3gWm2S};gnj9E-xu*O$&iz*s^F5HgU81f@`NS`S^@Yt=z^)e_Yam^4zRcWs0MtYH9N+J#=N<-m0&^l7x0pS%52h^-rkGnqd0x zST~`NRCUcE%it0eALRZInhHwypUS=^HDM z;arQJ*tLDN>Q-G?9uyzsT)FWluGxVOTUAHJ&%V&Y*dYBG+C6=Z`LKpI-m7WV0@L2=Fp38huRprp zc))x)3L$@>v&ZF&#yw9{u_Sbzcu)80`0RIM6K<1qf6 zK0PAt_kA10$)X7Qhp*oM<7hlM<+$b1j8&fCm8oxVVjX(?D}bO5d61E*X!EA!f!Fifr3wvjwHuvH7~v zq<(#%f^nwOrdw3wVWDA3VbF6tS7>&4Cis=?q9E>=f~w_-J(UNOG3b;c_$N|%&N-GijL`JBTHoVz11W0%Wb9}3s#Qp(lK9h( zyo1V9B+WI^@}MU}hdu(15^|jb#|IE5L@=xcAIxltO%-(BkdPxG^}s{}0;<3WglFT! z7c|lOa-cKY$fQ>3yCPWqmdCU3GYD#ZE)`n8SQ1-1{$OkzdyL!{y>jJcsq7xnVnk%m ziF*?n2S6;KR65Z$wdWHxm7zT|-oO(c2kxYMZ+Z_jTVm%Q;@)9n*U6T-Pa~z6AtD62 zK^MIqGbPs#fnD5Uc?URjT%!4xdu3@g^oCEFU&9q8ZG9Iwv(l<{SipGB;OoQXfOKdA z->ymf?suKe*Txv^fXLQ{VA{k^3Up{wr_=?ycp`A;qHXU2*;&1L?)maRH+;Em%vR19 zN_4`pz4~%M6;5z6y6UX!$M1J{ggVzx9T{(~^~xP??BrvN%YYXHeYkfOehf#Zs4^=DVt93j~8winIY$#g5&fvS{%~lYEmU;dDJY~Op_aPyxtAZsOe5;>IG&j zB6N4K#_B}z-cVa{1@mZ4%q#jcTfp{G#@K7I_p$y5#m)Wlc=0Vs#-Y<+fia}I-PZ~?0&lx(9V5rFxk5C28|BBb4w4*e_x6k1T$ zh4$}tyYXgSQ$eP!S9jnehTg95&Q5sEME(**tDbRW4avcS_B2uIP!VGy9^}LeErC0X zCi-o<2@be;_>Y8|QM%%_u|vEZL78&-sl{M)itSNR#Vm%Rqg$)=V0MIsUuc6yMoL~i z;lC7x2tr3*o$YjUaood3(t@6CYj=xpU?Nuj2s-;+nQGx$#x|QDj<0P_=(tvY)+8m< zehkkH?m5dn-Ah#sOV|X8p{bsSgzMep4Di#>RbJZYQ}dfHVK<$@fhK7lSGc7-T68b1 zY21vb?5hQOd(O~PaF}+c5qoxai+>QxouYYQqCnnZ2Nj?m&l8w z44VfDgl2`{2# zRLdbB^gLdb=Uq(nJ8Tm|C7(9_3aQ%YdS&fzS?2h)AY>w>SYW}#A~2!;`?6~sH6^h8 zNU~Z2d77XGw3L5RI!Ug~g8F~L0xY@P^>qh-FD5#%>Z6g*T^BYsXr`D@?b^y*8x1w? z`BTQyqd@L&37kV{i{-@eFs6lqHeY5bKSQFP$*MoNW0CzX@m-8kE$2fH4(1E zs;kI&()@r4WTrUeTXlpx@2oUA9Ho%hEJW|onalr>pZJ)99(of5wFy+PY_2)8bC#Ah zqbsgVWhov;z|&FrVBFwTa$i36yhGmvxVM!0w>85m6+{t6Leq+{rS_h6N0D-9OB=K; zQX)*H6oTWIW7VcHy+Go)Sg*ahhcq@ivT&?l&CXq$Sy3&3N6oi3m+e8PyJ=K6KH*v9 zLpet=>Zg*%Zi9eMr_u$Vfg(tV!Aev0E8QM>x+0e{dEKe-x2sb1_TyP!wQWl$+)v~D zUAXT0*yfb%;#=xRRVx#9CnZatrEnV=Wy<`hQ(YGTZ=y+W@$2**4S5P1XFV9P=ED=| z5JobP6eH1z(KaGo70&)yAlE3!a+9mkN{JvC;^fO~M5BCiNxAOY71hX9Yd{h%PAgWP zEm}V+{kel*Sav*~>?Aa&ZyIW*pHz?50a4Y1Fc7tVwF z(=v4Nzd6G0BCI15(n56yMJe6}o-J9(PEWLDYKII&JT`W^T`n3Udrc}mxs`Q_j3Qns zjliEi5t{BlIj`)0;dkEDF|Mr4%ovot5ivXU=jBPpe@C5g`O&fv8>1q}un~ri$A#-% zsd3lE=6fw7{fQF`X+7c&f}TABZ0^0%%LA+9EfW>zPjE7Xx_-GJIjQRlL&iUP6T#Q` zQuKft#Cydr;0v&!%F~`OK6;+pyfMiCk@21gf~GxUj15n_3qz7{xduUF?z>UI5Ez7FlJCoUdL&g{5)Y4L^r4B21>B<)Qb>Vrt}(w^f2Ep5Yb)>rYyFT zkQecU_B(XVYEm$Ewis95al7X}jx}b;ujr=hl&d{JV{e(5D3otD6tn>) z{KcN1aS2oCh%;Ih9Shw=aMY?~zqM_he6z!4>F-!Pl(L}v9DZo)ZPP}yiAFLT^?SbB zw2k@VLoh_AO4?GKT1m;1w&1Ju@({Bc%PdX=P#hCZXEgSCd} z)kAZpHB)%}%SbH*Wyu}g6W=rs6ajy}q{&9(RZ9cr**E9)#bNs9%y3B?WL${NB>F~?l8k@+)&BcZPOnJPmeL`LWgm6=Lnaa- zvNUF4bxh9X9L?;C#ncx2ZnfJg)LKM@p_*_)q}4kS-U8J%O`sa?7~9P*m!+kQXoo0E zBd3bd$p@HCr$CJUb2H@+Pk>1(Bb_lc$w;(P3BM7HpWa*v&i1W&k<1xy;kFokP3Svt zS9T3*S%|irg1PJ+10SqVX{eyZnHqB^PqAfB>o)${YW$>am)n{r9_Cdcw;;JW6H-O} zEBTO{(dxSvOAD`p<0GERQ2)?XRq>xQbUc^IiTsm4~YZ=VhBPRW{fvZbQ8S@(}n*QnUM|sv=n^Ikx zOqA^bYD5v{P5YN+H)JwJ?Y{a26{4N1_zCGSbJDPBpK5Wf(DROry`ZO*vI|MGmRV z7-}HEf;K^rmNQEGziR8N;UL%(NJg+ttbfJD@DHv>x~)h1t0Cwtoge8TAyr?+8hV7C zQPl5@(FP4M4O}u51s>7H#!NA+I_d#mtS4~lR2?}Y$m3$SqEdVQ&Y-xdK!+TOPUwlU z4MIMiEs|(LILV>A_ak!11yF-aKuovdh&y5rB>rT-ZImvT_^JQgzuP0r`Rl})?XR@? z*lvFG634~&aR*O(*hglU2Oc9@w(HcWpC2bFUHOo2!r5k5@mt$9o3y5=P^T>kkC!-z-L@Z*_HLLfXY<56C8t z1NQ(W1Ff?5MKS|U_qnxxUqY$*uW#M|AP%^26>=q#%#hNOILsN+u~KDxo6Rkf)tSC; z5ec0Y@aU*vh{ZBB)hW9ez?Rd!=SNQ+EyEtAzAby^)AhVTf7!7g)+1!+#tK8PL; ztGe=_n;)WEC!EY1fW(ve3*t06z8$wuk+b@c4Ckwblv-sEiCDh7UTgbSXfV7>AuwRC zuhBkT!&%Jgo3_W$l%h+wxYz<_fJ?oh= zTU$RYJ|ErYM=coSHmidU8rl`(RZxbnx=_zK2shZk2KvAz zG#f+NjR~Z)5ctF8pBXVbJFKX4_tt~0!CiFYi)FiV;NfNl!e#H9m`&4ksG~|d^Zx$s z7Qf!qEre`X-)Xyh-zJ-4$V~y-VIN2;CwB_TOs+<$sg^(M`6s?Cgm>*@^|Nhn1haQy zbiJbRva}reEOBh#S78JPJ8ApBGj09y<=Ka9D5%)&KXZ+3<7_Ei zZTXl#-O3p|YzZJ=1`TNkK3VYq2MnVgcVP@OIv3CPOP1103v~ve#8hiG5nz_`R&th* zX|)tPbrqt^%MkaMM3DbODiIkkX%D%gBWdx-J$gAX6G zs6k1W%#Yh9K00zXHi)EzV?|#z{K^Qgu1$cUBrDALQcFB9!7dhhM}ce=D;mbV=TmzO z4n%FyfN$E~H%lW{9oUI$ysQ*Vmx zWk;ffLC$$ApJl_Ls;(o@;)XmRLlerSwD}W`&gdwyeS$8F#osY8fCEy}Qf&*R?aac2S)J{U6gIP~Bxj9e1eyA8T>Gri1N}e}&q$nRugY;?3SpfzeUK`Q?)o zx%M?Bon+yw=Laqj+LqI?Hm(lIc9V-{FLpxQ6=2RA&ypbVHcG3~SBdCJ6K$E;ADoVm z=l?w0z7-`5&39_kw?=&c39E$hPgy)Vrh~$F9l`@B?luh7c~; zW{W3C>i#7?Rn}{EKqMUV`~hra42SI~oROdVNqQSzkn`@{w6ejw9C6m?^GKk5Uqdes zG~aNwUCA^nLP9Tm;a;BS#LZx~YbDC&&&E|rJX91=Kfx*T7Xrmq(_*v7hK&x)3qlzx zrypkB{HCuPAK_klz6F z$_0l!Wh;kKC)-+-%bI_`XLgPj7iB<}+)%4)#h}2}VF-$n)Y+HP-EK`-wOUkwv$USv z_DIkc2=ydjH9hEQs!Q2Si2tbQ&e`o95ptz<@D6Yt_5>iZYVR82AjQB?w+fN7%b&|b z=Ze^XeKVJtfC#sQ+0Hm}qHWjMZf7?8vEZatc8fS~h`V%b1oj8}SZ_NsRnwND^t4uC zrlCZ4rWEDH)b32}35uQ{eA|p1l9Fqz=ZELTq;N~@dyX_gKqI#%vSw7{!R+n;}uaBeQ=@7~LMeXO25{hxhihh-i88eKW< zgjngv=f$w{W4^e$(^;OLu;;7jBO+|9RNzo>&>#vFQp6vitcM5q)H#Ygm--V>rdL&_ zmXgLF{!}Qv&Q9}b(x0rpd{Y;_Go1NU21*;38Kl8$0llDxzpV=kRbHuKlyKXe3;MUW8W+Jdo ztuy{_U*vl$>4LgW33iN*>7_-m^XC&JXvC*@+2;UDGsq$;Y;_r=C>f!dE?&4RwScyf z<_SuHj(~}4K_e1)VB;!;JRmNPik`e*!Oxne)`5N@b-ZRCJ}x!MW+(dz2Bit?N2c2l z-*k@3iZhaecJd;XAoB?6ubt5&1xhrOj}2;`#37FRL3a~8BB-80j>?8}0&0@?8(1?E zavXoOJgvUfjTL=>IvIL7qmuv(BmU5FSzxtpq))LT6{cBi}<4Hhst1FY5y^A6h3ELC)|HsyQMm4o| z@7@YXQRym33Eij&5!iGR6p(-j*?qCI>LBj^x1;x;)Q1*#w*E2FRK;*WL`?@C*#h{v#`arMS~wy@S)7zvE$AVEhTk$1Y?fI$+v%7M?|WqhZ6!Bhm9_%d-OR~O+pS2wQFlQh5)c50$vPT4fN zu%ocBvAnO=n!`!`OgZ{c(0qiiGW8rKd?sFWE)I$7zfjW&w=YjK3}EsM72I;kZm1iT zQ4Pz|`H1(I#gcV+#SNR-P1d)2cByN zP_MEn!jXU37axz^m5s=VDKEZMEkyQ4Ul@nv>1oc%1Kanqf@4nwUWKVjo_b|g)_f}t z7{@#x4>DZu7GjC6_Ms1{jMoKFZ)d%hC~aRpaxWawKH!4(B|bUg--|9D8pc9h{9Q8I zXH=+L??mK!DBgUV0|5cxUy~YLGTRb&e45=w^gH!4{KF(zoE^^Y#->!&jum(bdd87o zKeAbJc4P$4yT@3H*z~E*fi!tdih*n^e1S!;H~Ee&L;|_g-@=YrXot^rw4VatQry)<6(nSFG>g1yKAIT(co-g<>3Fzs0?7@GG55+;0{G4b|quE#W&gkOz1GgPk$Za&o zt=6+7up)8lG=5xMl@|-P;}|WDO+EXwltjg_pS|w(JF?wuJqj(4$jcIG|I;|XFz-() zzzMZ7o1>nY#vS1XABdFVR64HNf!YhwefAzqyth3NRU%%(z(TdB0WXBauLu3%pkk7| z0wu;2Bk09Zkrnj!^D?JXT}6T}q4k-3hiYYvprfz@@EG0Uw{Fm9-53WG-kq%lI-qBBdxg@YTg=6Pucpn9 zRF2dVBH#~Akk7?Uc>|NX_+L3-!4Rfr6;rB^%>1`oyHo@2F z9yrVpFDQ3fsD&BaC4;hpo?dP&9?h-9J7)Af2YPO>DWfo}GsVTXY#&l}n@Bbq?-y(P z?_;3zrKAyElei;dDf<`B0@(2P`F}uV-)72Xt@jxFD0Ugx)+Ni0D5)Dd<0~x3td%~c zY)=z;nZ6Dw`?eQEo{|HiT8u!e6#lsf1s`joOkoIi)RvXtQ1eJ95in#QEBwQ~gm$o# z_Y$1&%W?pytI3IJs!Av3jd$2^jET9inOpAyn8^W4sP`bRiXhDEXs^8=?dY0R(IbhI6SyvDCqLlY*?Iz%@So4eM$*MVI)xq=wC@}2d(Qh)4>f$?N)WmTk$~3zA}QRrMLc z98G9y&%4sbMtzeKdsIC^*2NPdE^Q7&g^{J4YGn5D&jyXB4wg$XF?#YC!8(3VeB(ko zB#6ueSXml>g%tRpdOfExH(%~sBjqY*dj)f3&*FgnxVn$%We-W_HL)W*=hvsl-e&#g zR^!HJ3X#j@R_jThe$4s-D$?cDi>*A%AU$%fvV{JVvZn+l;k{n{=XB+x>WZM!AQlH5 z(b*d)G75w$6GwMd!KfEqela4#CtjQ75NmGYkea!I%NJUS4gmvR=(abDQQC~!_y;1cZzJ*-q!W<6WTP;BSZ3e>Zy zR^see?OXgQl(@Zh{R5G`6c_DnsA-LX)qD6)cFd?i$@s%g<@sAvppk{ZJk9`Qk*|o` z7^44w7a~t*wysvM^z|F@L1s?A%%QCn#LF?PW)c&>yVU$7rAuNi5Aj}9h;pg5Hv0*C z>I_EE&9eUj;t>D7v(v>b0xi!%k=YEc8cu-+L(0lTulZ^A^@%MXjG?R2TBaAT)mP@e ziMyN1Gs7O*@aM}a9fx`oAEqM|0Qwi&@0Ptb>ggRdL|VtI^1P*rYWT(W2~DZ}vk8IS zRh{lgA``xj7dJ66cND5q1IKv0ycB;4Z4GvOo21?Bqf`JAA2&Otd+l}~NY>Z|Ek6#7 zFWd?p2|h5AXYrCiU>sap>;ikFXSTwuByH!^P@b_pFX(iz*}&(xXS@^@*+RZui@Kgk zp`O7S$>oJH{WXMw`n6!Eb8FJX6~awhca6CAwvh?& zuMEX<&g%w=!*`jzc z57BxVbOWK*$b@f}hz3l+Cdn#8G9`%DDi{-;Iey>^z$HqMO7Kir^B&}n6;6(Xz;hA1 zQqFVqW@(7*SHr&JwDe2nU7#TbVb2{lgJt{EWaqbv4$f%B90>E)d-I}XPa@>Ns-(zN zdsEY}%PDr`akKUnd4O#nWLrph%j+Ji@B<*1&&|5(&Zn~3c5V0}zY_L|{3#W;Sa!hMTbOfm8&4K zCGG91i52r>0k$B02aLhJHSgcrC7JscMg2Dswk_E39|ftlsn0hVn@IVbaj8v>C2`hX z>A4cJ^X?eiInPY_Gt2{%AF#w$U4*Gu8FP>6^W*C{w2#-MIWWAm062nuMXGf)Q$Nto zeMuV9-FI>gjNSKFk?nd~hjD<+sR(EjkVWmQ7NEHYq`BpFcS9jsfy>zO=A6Q{Krw23 ziWJTt@z**9_1`&XdIA3NA_o2D`f-t64flZT`odw6lIfBic2kRCylt9i;L&O8!Lulb zr!L?kdNKF!D|qgzd6b)z2t9?5kuEtHs+zFebeduIYqG!!Zo9q-`*K` zR}xbaK^iZn`}9Pt_9=H^L`@_U4*$J}BDCAC!)^hSa{85`_*M4cX|7qEcVE`Fx%BS* zSlau=Tu@){bHRnLXOPS>; zcyf1$X&Wx3sgv0V@kdBq6L!*ujHKu1QJ@`ZFZ<<+W?-q^!-qYiGc7?C){Ka~Fr6mh_l4ziQ{ds$q%ly_@ zfFle+tt5_hXjXWkj{V)o2z<7y9#h5Kw&)5^moYlVyXNh^$Q0;y$|aR0FJUy_7kKO- zi>RxiVppVeT|#9K4RY%7drxZ-^YT_?wlT?hopEsJBvY8B zvRBO)KOi(eWtI^pJo_WT|22|hI?fi6`A_4yoDc_aOYwh-pMNa~|C22L=d<@LH0#;< zS2R7||6u`~8L0&}|KX<7W6ttAxuajRJN)}XCp`}R7hE*xLrWcr-A$msd7QYkQu~9=i#`zoj>qM6cC*)}K7M zgsX?_3@H7wKMWI@z}~x2q}g~p>5}c5>tZ8bNhgoq`Eu_o)5vaL<~Kq5#yPa+_EGYy zcsdrRgx=uB`_`YlR3$S(T(WkfV3qZGhOnCtH7JL6EuI+?7ehL3*m0b=y96a(-D;gM zTm<o9^@Dmu_*<#a8~dlPLs(AR^<0?tD&b4^+peh zsmiqr9nY*Rk8XNZZ%4cqXCak1z_~Pz*q@LrS?0cd8_AdRjWRDDk9M&M(CZ>68tfxI zbhbW|_HJm=_AY^A`=?k+pJjnSa-w)>p()VCCcW|NbUm>QR($-bJ8#qUNEJCY_1RvJ zdYee1KOqG*Q4wDJZjO!n^mOqS$vtJ1|N0hk>~@E?1(GyzBQ8ed0ob zo39s|b*2l57hF=L(&h!!gQmD}bacyg)LJQr1vKDSg_vv!$;>OG3s%OpZK6$1IejR! zLPRFgD__!_-hJrIPso$E(Sh(y_lhRj`kU%Jg?^C^U=e z?!W5zat)t+YWHZyLIVA!0Mq~#F)>{o4rWFJp`_mxv<5>$7jjzBy1{=S8E=8Q`hY-B z)t?;aEZo74ZBe?%2F-#OXtPTkT7{O~N@ZuUnsm-t?4AiOf4S!{3U=qfq??B2s{{O- z=c$xktu72C1#X3jWK2;js|-1jBdxciVsA{VmZ8uaLbsCd{ELsm~ z(~v)Xx#zb{pgWMNWk}75-n>wQWMpnS^n|Me`OxU%QF~h;BG?lr@%j}b*u}H2n^IG> z@XoN^B=*gjU$qhp@DncN3maWCD(f z>+aqKJb$@CrxR5%0keYO9Ev$+4Q%wDZ=R!Dx44uB0pSj$uNHZ*Y_;Q+AWlA zQtg--fpn`4w4G=+eY_m6d@2bzCL{?BJm^QpVPCReb^g7%klxT1NVujsRlF*wFh8;= zfFosd#tlVlEKBT2+S{}$d5}!7Mi)kb@q zY68!KtxYH(&N{C#1{6UFp%bN08VS4G{$gUJvwGl*D~j`Tq{}L7V`h2$aHukyaf>jd?0eHPjF`hyge7rSjzKcq`tWH>K_K0 zRJ7#b*Y-LwlY3ktqHXP(O1#iVtd(MC#v=Mx7QU`y|Nh{&CO!ffjtUf&j|U#kFusMt zo;H7TayLzlynFu+&x5_xa51L`>)T>qY+c#;a|kn~oHbt(@75}cig+c5YlDv}eyJ%y zGdu5}9fVl2^vQLB76uK3rO8i;RgcOD;8`8v5ZSUf=cjr5noXRXJf4?ovo9NBjULo5AT|$e((AVwLzzQQbONbD)H0Zg zZ%oO&fP38MC4p@|$5So6A33qY<8-BerxlCxb=@v3`PLpP14k}YB(gM$+6)6^J@gXu zof=X*MDXHk2HyuERL8ZK=Pp;4G0&;gG9Lix;wBDfFp{I0A(7JvT}^Xcot{=ETtNhV z^EARxS-|f(iCI;!Q!>~hb`KqXkAOBu%IU7|CzNFwu;=noP4UK>wzCrf4#jCK4fi$X z{C5OTCzC4h>0VS%DZ-~?BF&~xao#Ij(fSVZuG~Uvz(Z#VASn;*&(ptsX`G)-9t+i8 zb$FPUskn$P9z(Fp2ktK|PuXpooo=xIK%%i?y2pISN1c>1UF+TpNQaMOlCnL;IsYl5 zdM{1{9j_0vK(<032G;HNsptHwaSqZo4v`Nc(~Al>naaj_v0LFQ4?qC}dq4Ra-D{6l z^K;yeE8Ws%YnQ*?%f#l;I4B8C$I&Z?(}&+1dj05AizlD(HdN;i+G*B<0Qn1=0isGJ z=QYK1LL<3CWOU@)(4SjKDwWbTtFaLeSme+7=!y@78wSufx_~lFW3ys>Jl_tTyY@9= zA>l_M*z<6?>906fT9^c4=SFbfHt@y6L{Swdoksn?LQ5phi>NoE2%4?r>LEGSs) z(7OYv?SIxAb^AUBPYyNC3rvpLzC0D1{2ewCSrnzV-~vIpwEPi_k6c!F);iEXC5w3oaj%wUMS5U8F3l3~JaUcQP5G48C+Pfze@Cniea>}tnz8{hH zcMeOf+b?kfg-qu^xphqm*`ln%BtrdLa5T6>wv@ZcKTAC_r)UZk^wgSN*m*H+^_`rJ znDUz)2S$_wEPCmc<{#Kd1)>I;drOVg=`*@D5~pz5FH|bjG7bxRIu453&6rx=G4 z*8YD*!DpeSr3~>?fz>?QP4va8X;eVb8y((fQEt#TF9;=c<3qv4=0-|bR8FIqx}4|e zL_{{@^$Rvvi-T+hwHWZ{m3l@GBKhO9Lm+I*GoyE*k%mat2l|N~^Fn9VlAf=l4+H0f zvabgsAMqravuSG0n>2a)4q34a*pn3oQ@LIVqxM!KTZ@e(%Im>lo7Uk zS3F%0AO@KGcy2gJ^N053_*>1hn!7AvfQ8FSa@(&lBT%y)!6ob5P^WV*9(soPl)Q_x z40{N;<^5p!I@#$owF6RHzzZ|+#HjE)6pFQ{0O#WoTv)inBYF?Wpg9-eVP2^o!0Rao!Ws`uO4wpmmKWEW-b>dm{M~gsJ+SN>Cd)y1Sgr|s@D<@PQh63oCSmT5 z!X+x(eN51cw=v9BFkg#1;za3te*A6Xc|LhWa^Y`jyQl>{nd+R}`DN+rQF&t>T(n<` zTJ==R_2M$Xf-5eOC8(v&XitwSB$-nWY&HpobIk@M^3&GxhMc(9n|N8|O1~9xS4Kr0 z$7@tWF*wyXxr|z;<;7#K^1(|7H8-Yy>~V{ z-o&qfn-eP3ebzo3{+{vsdXu&Tx$$jaNkVNXI;d)?Q-S#Gk+3vh(HB*&OHOM6HAXxO zzs*QC?m_>pVHa+n+h$J`g!vuY9A;G0(Z)9#X;(tmsQ8`D8NT|N3yU}xgmS~)KcLM2 zh)Xs7sEs#WQj6tBL82#(d&$9v$w^I`Rwh9@LA_V|zb~*UQII^*QlZYGp$tMGeVEP8 zGQB_xH$2z@y=PoNs~Jxv-fQ8P>jA&Su>0x5oKNPViTa6}C4;>8w#xT^6!6eZs61v6 zM7w0!mts5Fg)m{(tMLY9%wv&y_Zs7GOD<#dF3d@B-5O!HOA6)85O87}&A1}ALU4JXB~Q>$Som8JR=a`=K_&O&j73((I7WZuo_Z*c2w9WlX>!ku8-@ zMHF>_e=IR=CJ2E&4rZMkEMRlZI47O;WyRGrvSDm|RHA!IrG@ly|B;|Hbmz_baz6Lr z&Fu__-p<|tD(VZ?&-4DW_K7@}JXy@=l(_FP=(@=T9{~Eo@$o^wTO-Mi8wKaV`bDQp z;iWTTPR>nVTUBtJdvL--s{rmM>X-Qf8MT6n1*q+|0;know4w{$=-y@ee8d{%@6Q?4 zUWM!4<5-#*SnnF3mhOWoN7YBt7i_YQip_t?xn#;LGbzu=spGUR?QEtA@!ty25 z(7lr-9{dT-ZY|}#wKVsT^=8LI0ko6S0hEa;@HU=FjA7b&P5ZmvPN+((wba{t{4)WP zTq2U9Lx()3i3%o>6WoqW+l+a&jKWhS9MQN6RY=fXP%Jn zb{g7TsQ4Mdbip$$<-V|(7gzEEM<)<$%&1W*DgGkbE1Uw=brOx$aHL5+ZFjUxudqKU zGEukr5-Hi#L*cE(e+PSg`Sx+yc*k?SAgMyi-jmE`9Yq(>c-cx{KPq?u3&?{Q0H{jA z+Y1?E+V9oibTpA;%UrLwO=rdXl?GGn=NnP)?|Xj1HpFw1f2O(1zCa`6M*Q_(-B;H4 zv`m+7Y`!4|MF#2qHr0o*s)*z*655fm68Z4qYeKP$=NIF)SKSEy(&l7e+ul8 z@k>{|#e$s|#=bQi4gdHDu=kCrD9AAH!Ntnvuq;I0CGb*GSz_Qjw_PTL6CC4FESIBq z(P-K`h&P=A^m#}5L5ub1IVgC2_NE3u-+{fs=c$dBBBy0TNpzMaHJ}vlXk=4Eh5RF1 z0OjdOse(*&pPKnTMYhQ*VPcL|kXJ8zpV!7#K$UQuq~G)F!gIhIZ}L0{IH9Y7<`M)~ zP$WZV(AE9~dOiO4$;oz_Eh*^rp@mj#pb;j>=r9$HJ%#pE(Eib_c>2f{kNnU*&sSOT z1N(9L>6bl)xoP~ zmU5_dV$k|tE)?t-K<{WF&Qe;;Qndn)(~1w%R{q@xaw~z@bvbc5d57LDrJa_4>&IHr zte)U=;$lupjr%muPnyGa`dB&r?_){{K@)81w^>KC*`BqFXqge=Kj(1w7o z*AyS}2okC9@pq3qfsdpbnT@A%^O+<)??q-r$&f)1{U>jCcrT#Ai+dpe@_y-uL#-tNC)iG+Y2bYwlViuz2X%Za+Q1@ABh7`rNWuf!uj2*b|#g1%nU%a-m zUefz1xw>ItFb&PVru);}D>61^=R6M0LfEszxvz9RYy+>0E21w3F{u2@*%}=p5A)Ira;=Mw+HYdyJ9`pE53wDnz=cG5H z`XxzM1Q7`r$%7>^@#RNbxh(}2ifC(wV}(?_LTO=j zPu^Pzg>Mo}etzUY>Trz47L*0XO%y&E)bp(RShcV3ItOdqihM%-)<_Qb2Z6sT7aAu; z4Z~Y2Na!%MgO!`B2x9pX-Fl7Ia)Slc~Xveb-jQEY}-90Drwe`qQaR;he(kY^2E3nDdMO}f#Z?Z8iZuZtXb z2Xip44sX^$a*5z;a##wK$ylzi;%K!~dXn_wiCgEqs>j;zNMLY#e`ny`zeQBY`{nRS zCpF~!9jbc_Ca1Ww+mSP?Gn2kbMWg$y7r*OE6!6Z0j=)u>A+i~@eXY=p{U9Un$vFoq zE-mV1TW|&+%kvV+eyp%ZoRdXOx#`pvUu)`bUzyu+)Pd9uMISMDv8_*9GxUl6EC6Jd z7cx6L2w5MzHoNX{T7aW$sp583IW;#rPOBqvw87w`R)@fL2aUjq4%7rxa<))wqe=S| zbha=qLeo6k1eq^{D*Smsu1(#%QIo!Em!KIdIIV6W=E>;!Hhsl)Vt?VY-faT(sXO2W zqZp%@_)fZ)(nJW*wRv-ngjAlF;1#>5b9pZmqdjp6F$nzTo=Ci|`Zib&l-TnE*0bkhTD-7~R1+)zo;qDzmr5nj_Z9L-+TE5g*e%7W8Cygt5qb3p%gBCufh z%f_IRpn_84Y|VJKvfN6B3k7+7<$Fc!3ucMNfW*|ATWj5B1-R2&RSgB$IX>Lfpadyr z&HRZ$zaoK0_B-K#+VyW8s4U z=i=WA%50v22_+AdI|_xQ@4Ouyg@k~Ek+SM= z2WneT*SqHb*GXp&J$>ya@>`j$ku9;B99{m&KOG?t-M)ET(~eH!eU7;}H%WoY)`k=M_08W>-d>0f55PmR)2>O0h0IdU^A`s{F@{RcXj*Ns%Ju!{Y1^(f#S>RkB>%IqjJw z*^C8gv+BOa%V(JdKd1hH2|qSB5}>1|NuAH{j;&9At!*fm(fnFTMbmhrlMfw>uPNvv zoTME_48?KqSp3Tj!|fi(Ea#_neO%#len7|J$C^w1j8#-EVs1(9{z!^^lx3hr4o6Xx zSIpS2OB_dRUv8Z&f80mL1`v?Z7}ZQLmOfsy%q`b=^vkw*Is)6&f1qaO;PLYZ7TGwn zoh|TJ(bQ~{ZCWh+N~GndNeNvQ^x#y=Zegf8f?9%&Z$g>j)-)vwvWg@2YDpO{+A1xF zKgdpuDig3E4rulJI)2MRa^$dxsei^9V)}Hh|1U))C1Ud#AG9i`LHo9M=rnkn7l{Y#;jLaS~v z{0mb;xd)upMZ&;XwL^W}o<8Tl3qa1c=p5%RPV6~f`atEO`p_8-)P&;Z)i~_cl$*Gt zSywv6>lx3$bv>SS!VaoiQj|W;eRbk*Wlgt4WG$yh)>(Be=xa#7YXMZ&ro+&0fe9b( z+9}~Hs#)TDqrFWEz@xfCU1{@`6BmtVz0ejchzbCSG;#|rNL~Y}7<#5XSy7M64Ox(k zTVcemhyp$+^KfXfKaw!nv%?|}pXUQ?#BDSryu*VRK4$)Qe}1|LvDG^FrC&CK8)!d? zvOnXEQ}xI{Gy3z(lRuAJUQaRuEjp{iDHo};Fn9dYCx>>O*(Ua0himvX2wCJ{g>!ok zuZA*#Q$8QCX5Kx$*ZDX=lP8W+u3R=6ba#4c5XcJhL=Pp(RJ`7NkEAv!&J9W=Ee}l% zd3i=WKq)&(Gn2{1`Uf^)vN%$Iz{Se8M~KVAN$XbD&4A}cS9e?yIPk|rBc^}K!9M}c z)^mYJvPu<`Uf0ZyD2YMCnmZ%KsFhtskC0&g!2yGWD=4gq$ome;1rEn@;s-dfy{K)+ zG-nhxSm|}Dh08TNMG5+Eqvo!u8m!IT&&@E2dv8#nplLq(+_S6_o`QS;Yk5t0u~BpQ zu{uF@v{bfk_*HwGFYMVHStWlQh3!nbUjj-jbyw^5ZGQ^c8zUm`P!hoHaRP<7OL4^1 z8btK8;?-Ci0_=2c(UyjZ#oJr?4->6`fkya5Lu7Qe2+ch=ZmDAV8w$x;{2JMZSo&|a zPxcL%>OwOjGQ`qgpAg_IksfMq(>w{@b>A4Spnb(uT`HvNBb(ERvracB3)DY3Llc zRzn}F@X1fbh9m1X&RSuuwR1%FunZ(2Qs^zg=VTwaYss+_o(I>=7YwX>+F}mt?x*N+?5cH` zom#QSL@B%->MpXKdw~$Cj(BGb?k0S2EZ!fT?us?#(>D!r4j9ef!g=G4cC2?t%f}VB zDV=V6>ntA89gUGO6AcTxP?4SDjl0iKZ-$)KDf6#{ais5?^+q|xeXAz8ox07J$Qxe^ zEDHyu?V6@0b={DC{s*`O-`?(bB?i8bJhwK#^u{R=lPvvA^|6XJCIm%#?unp-=Hd>5 z!NAh$n%IJnJHOcvTtlEY?p;PdnjxLlTTHQ27MK0VH){IPz_ldTKfyatI}WDva$CW=9M-;xf=0g4X!%Zs-AGn8{w~W8MaG#H33nO{u z8{yGTs250wsdlZ4j6G2M9ZoC2ZAUfvjU0)@nh$Nh^gIanu)YjJ z*7rugl-oj%3yTX2l6Pd_Bim&2lTo9lew*pz;WSb{l$tMg(qxTpKsja{N$hN2!)4Yxxz4qNo?F=l`oP_iao#(LI0)8zRUgCsushN! zCI2~{(6>1-ND#2?cE}#9Vfi(fF=6H=;?(DPjSq!fbe>L!6;wqYA7(Fg^|URagNn8n ztv9(LeMhgT^@>&w1*u06@;|I!z0Qb&gzuq8+>jT~>f7HtQO6nphD{A_aT};d4tx;V z8m?;kIdB)7W_-~>JQHh!{#p%Tb)cVu6E4=^3L68yECl-mdm~Lmi3ITRNm~( zZ!7a&F5#}+(H^1?$XdXn!Qo^h8wKMz^%SN__K_BktV z?X~|fr%eHMeb>1Ukg5YvT6ofD!@*cNGQ?8y(u=3_yUh3=9&F34aCY~3c3IX^a*+U_ zk^n*C-xk`G$ZAd2!wJPU1Dx(9x|8_!CLpJ7qJpju#x)DJ85jlI?0r@^1v%+tm!-Zt zSn_{L@_t?X90e|Z;_SnIACQiWLCHJW7dY9F7rOgWdmtg2mr1l&hSkRzr(5G)&uo^L z7*1xTsYtuZ>*Xrrt#_xQTDs3{W7mfX^-Ig_M0bA`Gd}NPz@k`J?9 zV+QIwvhxBAze@DmCqJ&e%kHK4gX7cLo?k}n!QREZzPsM*+2d0M9h<}Z_Ok&^|G9O3 zYJ$f)|*Z8WvfYbk5kC@;Rj$vhJ7dKVKX(ex> z#lfQ;#o7+a;|ZYJ!?+JWIT^zElk`cSr(;k`KvmJxPWHx6oNO;hZAIR*AK0CM=fyM@ z??A#Yb_6cvOPqGHh}>K$o=5=1OAp1U7}bw#beTHVCWX7^!vi@PomLz`49u9thz}n& zB*C9F1BVsL#5D5nADQwM4QnL!VD6k8+VehTb+FMGtWuJXQY`sewI2mYm3zi_KIaGu zxQ|W)OU1FO+wf%xwPFfc?>T=gwnRQC`}(~8Q1O$ww2`>)!atr%Zag^q`nf^IfXm#N zD+cBxnsCSb$e9Ot`X10cY=76A4#j(%C32|e9Zlp`2cfgQ@568yC@%nW}7E9ihEF-`zXV9*9M2;G|t})=cu@U^Pc!E&(yY-S3x$SX^ zfGhXeo;50Y^W}lxJPRps^sFR2-JDGjbUbQ+C_M5vDRj+u+njq>K??zEBIArumG}n* z;MN@_WCMcLl70gHxDVW7qwmZKZpUciH|3N6{qg^tKvC8;gvU`D%T1K1ZX=(!?A-Xi z_(3xHQs^c-6NyNvCrQI5f5g3cbInrcJpP>W%A{mmiagF|(VmWPuCs`H+)S%4#m1q9 z$$45GxO(xN>VYXW4>3qd+jRGPC>qy4(v7 zRAXvs9*c_eFxSPH>x4;YIxSSO%5^M(v0GJuLk55YN{i>IpdD>6uglGBXoT-e?xXnrE zF(}$t%yfocSOQB%$S;>RO}}n5rB-(t|dl8yrhXoF^mHxnid%0j}yz6WU^h z;{iHUg`{}3lAG^vL#?$@pa&S$3-?^-E8WhlG6cGk<6r#StZPk8`c6q+j(8^emO8~bWxI;k;*T4w6WU~C^6aqBw&DlenOEA@rg-0>{_we+aOn33 zHm)1j{YwnRLsXM}z4{!C27M&~{HwE$v7*Xse$Mg22CCna_*;jJT^61?my=HNlTNPH zY#adPPbaCK)eRK{N~nu!KKK@OHBhi&J(^YT4G>GoC@li^EpqTu4|h^Gv+LVLmAxo` z2fRs3M(DIT52W*xfp#F`%GTFGZ~e1of_Y7u!TXM-n)ZxS>qmj3ibhAH&BgWQbVlm43eRhFznUk=UUo_}s^ugmx#cTPd{2%!&?R(0n%Qwfw zKY4z(rMoOU{!yg(D26Q70z^1^{hx%*kJ)#$7E5SZq;IH=#)^c3q6%NcMJn>_~Qj|D-znGf1|;P%)z7HfZ0cx*JL|N&HqGy=Dz=e z=i!K6Jm0S_)9CM)kfZ|bc&_Lv>*W4X7UykV551U$5F8|^xiZ0sZc@VjRPhe*=*FEH3QgO0?sgRum4fpVPkXE&8Lua=XS0t-=o1Hj#Af;V z{=ucp-b6H9JO4lv2Y&G$=jliJ`22*BL3wRg@W5WxlfAgSp~n^fbrAw?mUB*&O7uS zFgyB`^-|JMNfOh+=g1GC8R8JT@XTsaNbTVlgUH{|fNCHiN^^VEiO)}0BYsyLF7(i$ zRsp!cawz_!r+zdC2w7m+UAuu6d7;CSD#>sWN5Y@8&5mzq|wOq1ki4yf!$<%dzd7A6w7RB4#K-Mm2o>I&HSg!7lOc4CiYdx0dq}!-Ct| z@ROmOx%ZtssjvIP)&rufba)TTy>!)9R-Uddx3M&}d1%kAG}NXq88kgpE=@bU>UPZ@ zj+|Voz5DTaOcM%ASbf~IlCS^QuZa4QzhT)qHJi&@?7`539&C#0j=tuj6>@_e@O1g= z_WQxwNgr}wEWrm%&U>?}VLMHvNGj8Q<{d0<)HjyOt1Mp=#{{G$TR+PTh6{Tx9vn=a5cU# zufIKZrQkj>ggAZ{_)`}HcRVR0XZwFOI-)!>!|hHh;;blt*D0L(*3Z+k5(C=l)&<1Y z1-|Wd=50#Qzs(adotde@@0#JVpuzuN@bO@pT19vWobYYSNEXp!6_ksiF)e5F6?Ld; zDek+Dz`dvL^W%n8tVzU#bq_nGJbi{+C!VoZ6(@^K2q>UjH~!g+^zjvcr?$2Lv^$@D zaqinGZ8)QT@lsdV#1(BZBixKci%`zmq>xKLmqglc%~?y zm8(ZxNMPeqi07IK`^+_P`xx~tuIGM7w`#_Fz8Z<0CzZXW21NgnXq=tt32*f3ZMY<- zA~yRyK`aO@^;YyWv!I1jXyqsKef`XXtsOoDRU+p%Faa^tGxj9fc4DA*tG?lR(or8$&ii-wi=P(-Ubk8+NdtAoLuOv^oAi<`AwFTzbZ@1QqfGyfr3e0<;u(nX5zXsxys7D zO|R0Nxj-d1PEsp12Ulg9Y0lJKxNzkv_ud;72OucpNB4by&hLBPf5AC$-kigEy`PWy z%so-UbP3`L;}J|gGnSw=k27dbc9t~)IsS1xQf!y!LZ<}4O}4 zPk?lI=u(_bb${~7B?0g)V%*5SvK%`Ese}^AI0q8D?__y5d*=Rl$qSTUj-dE-VtVp?WvhM5mhKk`q9KkNT zC^P6An)bA0cm8rxLAOuCdG*Wg{iOrS3j(MrOYG-}e=B@u8JPPaJX7SbiK_fI1*CbAF+l_Wz4- zlV`%J56!Y3C&be(2#)UFnFDjm$~=-3rkXeeqqUI5F$a5y=~iIPRg$dc9Yijl#%4ym zVYt@$(+<<-Y+!%a)sd<`bw}>Hc*Pe}3_ItuLb{+oICQi|j7M&YUW#aLwNcAoX2#2j zTmq*ZQlftN0ZvOgy{}`4?@GxTXH+eRX|+ZNRx}d9)1?ZM3O@RHN3om@J(6v9oK6Bc zWEzyhXPg{gu3WY-sP;i{s%^`!(Y zNjHk$Emvkw^LH)K8ktMMdt8B0iKfl6{2iwx1+X$r0B=VOAY*NTk+HtMG-tnid zaz{g6^`mM?;`%W4kK+%x{tUJMC4R?-zGud0#{|`u;C;r_HaGW<7aNqkNc7rrEVQQj z!D7PpneUzzhu*+(cGXpf$jOg2+4XCq{Nqb5^)g+aQ=)LeG!*;3Q2FXf?grA~2iV33 zgp2m{V^~9g&D)BZ@A-{8YK#Fu#CN(SYN>0A*L7uW&26`7?yQy4&Rqm9K;qwPF+L?Y z2je9LSLRL1*xJ?9gV484K9r>Z?PVLex4Rvx>OZYU-CZMN##hAC9p4YJ8Dq*L; z`1w6%IIH@W=q!odo2YD?{{a1ZCLmok8Ahyt5$8?On~0FGD|68e=oZka?_s2=bKa`m zb#%T@JjV^WSHQVaFsW$YW8RQ>qi)FxRyJ}peK`3JwAz2 z0;Ne^z;qLh*W$wDCp(Y3!ox@OF9_5HuE)(zBFk{0A``bjg?w#wYZMS9*^#Kj}1!k1O;X#s~1TF`Unb1Igaf)GoQC%F$KXI z29jae+R?j!q%rK@d3njrG^4b>;>JZ>vH58$p~1X5(^&l->n@8^1 z62nz~j6&`NaP#ZJ$_gCl+M5_U@~Y37kno;rr;~PEAG``=sScIT69ZlN2j1id!P=T) zKGw*D|FoDc#H!OZkz0aS|J1g*HgT;EpG9<~5+3o#_ajl}GcV?empf1haF4-ZGC2Mo z!*%qSdR!f{>F!zb>6Y2VX3IcTMyDgMh$M>Xz)8yKX;J(@M=Q;NS94f zFcH86W1m@zyLV3dai30;QxI*w%B#^ zo~BSotS3}s(b$afV$#UNZ`bWzke8Zz_0c|1DJEx@P3QjUXMHk2&W`xS~3jtWrPqcfY(K!QY7)bUj(I{3s$RlX^6IEYJ8rGPVbI zz89)3neWQ=(vZiI7!~QIf59>`)}Rh^($G*QynkzOu&(j7+?{Ff*Jx)9>QMP@QBMiz zc@H2~ZgK$zcKz8co1%A^Y7Q`Zk__d$bCUV&r|C=EkY)Y2cT37u;DsjP_b0+2A9!pC z0=XGE#clE#OKBJFoCT-?-UR382!q13B5yu)ElONfq56PK_FX#9Rqlw)khgC8=9m@N zQDc1}dHu64CCgm@{1*c`kwWB`wr}0A%@MUWU6;gQ7nGpokQLe@6^)cJZYx>Y!|$KZ ze`A2?Da^NjqHU11i292ueG0a*MLm*{dMu`dXE&9zt^oIKseejTCXL-Rx>L-(@GI3O zO&}^^^h77r@f~diXA&X;tFvV~PuX6^+*Q;fMlB3+m*DrDXEn^%W{aI1J&eD$37lFl zlh$_YqxvH5RG1{}40P|`6o}pkXPCQlDav< zi<}-M^@VsBjeE{m6RIUTiTl4FZwAMYhX(jC_ffA)u1|V0TK~;l_UJ@O^Gf|CC2g#; z<_*p^76g1~DZUHYJ9g)o_aczKaQ)PN^6abAWAKgJm(;*{Qc~Y*`XwLe=f*8UI@~q~ zKP?tK11JRF`?QZb1OPz=uX){%iSEdRk6aoowRT^%J8Og!cndF||afs7P_pE zkC#ueeZaRud`=BdFIV(PFb3oG5R$ZrKqwH`^xs-|d(Qm~G6`2-p(WAE3uYDqdnUzioW9YN!HT0N zzr39`jY15r#~MP7c=M8_i=e4d=Qe@MlJBFQR|(|ir0yhr_czW^(uc5JWq6!Qmw~L( zgSgL8SMp8`_u27+)uK$tK3{xrD#VYPf%X?!Us0;&8+mk1{g=|ngdwCoPO1FtEav)1 zmjm_qv|~{|=tq-E-9mb8H@jmRqO}ec)Nv_kR3pe28~#bb(N=_-FSx{fwFsz9R4~AL zGFD7U-Sf6=3%lOYz0{e1_}AY<*(2Lbvu$_K{`K$~2PwmmKn2KvXfm0jwR@HUckuW&U^vYDNe zG7;56Q61}pA6~;7p^8IfZI#qEo**|%AdwRO@ovkZMFuy!`w1_`p{22kX2IR-i((-U zLi_p^pOaos7KN$pAB%T%AR5P+eVsUga(A!N9{09LK4JGtOL~uey9dcDo1SnBvsk(noo8WUuA2~smJR+f4Bc3)0WcQ~`RnKL+mp>LYY;ie6=^A&hnGC2$MD)`B`^S8CzbeW>;)BMlm87IaLr`f2yneJu!pbh*4+(^ z!xoFo(#I)nSYv|=fO*=ltE6m02V2&%pn8-o*I9#_WnQH#jO4en8HrF3iO025SWzw} zwsCCCrC)t=T|pP?Iig=4yW|%Qp)jsL=x_+8bcXF_gwk4HW+lTDx?Kf*K~e>2DoC$n zJl`FD?HpSbB}XSfb`p~GfUS20pe^n{U|gA*d#h?-RlzYAn)?)3<<@ODu^W~n;S%rEVSAH+GEkXHz%T!QU&?+3wHWsQcj%vK?z@Yc{m5e=DcoDqDsl$rt1CzlR`f zQ*HA0DRyH60D$xv#)J0smM|mIeBZ}`zgo{xJ5MBed;#Zw*C7ojqL3q?%kbcgTgiO1 z@>51v7MejVKeNGx~8mzO2Ef%t{G6|Iyy~Pk-dpP^<>Ow;9hiI=%P|~mgxBPr@o-3tFz6Xqaa%79^vrQq+TT(WTS%gl-v`x$zNr(-CT2auLwFQrjrSH{u18qcE$&g{?t2f-b*xQD z_PJ5f)!Y!DBQn`Gas3e(4|>$cJTKV}0+bC^Zgg!Zu><9l+Yb`r5oLSe1HI20WU{3A z#yn3xRbZMi)%u$M$YUGGs(ZYNsuWGMg7A$^#yrzHs?0IC;ifQ4)P$|ao8|2Ns=!nT zA&|_o7H)jk?H_qI-4Wk&%oim*+8iDZCa%Du^z1}>xS>>nGzqDAlz5NY82fW|NG!}>b~BddRd`;hwofqRkT0RmpWv*-NwvR?VVy6W#O1O=TdZyy)p zR9w+p?j7>nOLENx!H+HRM8;v@o!mSUsn-QFB0ad?GW1yXz=Bo{jai@IB1%0S^7Iv>1}+#RNO2YnqB zcW1j)lOuw^x(Al&5H6`dnTX)GV{EFFCfy^P_ZQHhX?17Hhc)DU?SlZHG^9Bl-2I0%8jesXuiwasR*r1YxWA z)WISkTXBa{zD0Drq5z}T(;v_B=nN-tHpLkm&u#ctxn3WZ7ZenGc1?0rsf-i}@~1_6 z6SMa}o}xRs{UF$!9ZqA8tOFd~+hw$Cz{g{R8-li+d?D|;>Lg#qBU^Z*7p|%*Wv+gd zbhNV;QgCwKM&SCN*=G&$hU)L+dy>~=QesnI$VDjJP{~GVBfdZ~F+n?<$jXw_`Fmfr zt*z?&sF#mYjaS!wkt4QasKX0Pq4jSwIn?Fr2Nlec)gcq{XN&t8wxJqdP5|L1^MPVP zNF2&zwi{uIaP`bDIgR|Wq466$1zrQsd8*}5^=r#QmX&$zEm_ib@!n3 zVp(em+!!T=ST{?LP*=xm#-QyfMnhgGbVK5QcxZZ>?Rd&i?DFSvXlUCeg*9dTIP-L zK^TfUeAn|08tE89-CUV3$|pS98p<$E>3P2NHbDJzh6)GMOu+6nF3J>0HTJL!4SE(L z!4jN2wmAKbfy$SdO)<6ZS>8Mp?s2!F4_TmX!xa6AWMs7&EBu&3Oh8O2KGr2BbKE*r zC716FJ(<;Xy|qq+D2)%o`@XJrh-c?cwZ#y8v1C=0y$w1E7R|xkROpdUNe=l{#6(1! zC_Rq3+45oLnu$SB(S*e6)~}0ZA@)VYbU!bfrH>~(Rs8O~Xb=!G&MG9%bdLy92}S|K z^J>*q_;;0Z?T06B?WNS!gI@@9$dhFi-wSo&o=V%hUs>oLW%oTOO1B=n_ldKko0x`!p<7jh zb@P7{h5xF}4CB*T+D+?~Q4ywxkVJD#-^!Fiw{9&l#@ zRC@Ft3J9?|c?H$u;ekEi9r(tWY5A=Oa3y^{jX@}@@;eU@aHbCf{F=EGFT7Z0KIPss zwVod@19B-*p7yDZCSky(iT}I~iHt_on0tPmY72kyODG4VpOMKWOW<|pw>u%G_ElCn#v}G{ig;8+`sfSB@zyyQUnF++Q;)pBzgrH$#0<8^Ea-oODnaIIBJX ztbRtM0y19UdGsW-O@Y1PD1ZHx^Vr@E;8WQ# z;fV}NZQ|6%SySp-b4(7UEu;A3)N;=)38g{)3RGo96@;bk+7XYj>zB6NBCaoWl&O5p ztci8lNE5OdBQc$=&yC3e)hkTpx(hA+n9KPM7PF7589yzU-g7^4Bn3%?yJxvhah=-G zDe}%?POc528hTwK<#+uhERKk4B}iSZ5%PX6UfHBp-?*qJiXXT^D6XWupqD4(VK+$o z-#kM&+X9yVwRO+8Gj2n>0P_^;9(O-YiCBfXc%~^T@^Lfe*cVf4Hou4zX!lVQ=+%>> zTwUua|FPD?jzbG3?mZva1vNHM+)z)vIO5lTubpPl-0X>*J*s~YXj3ov$X;{75-q*M zK|}B;kE(+g*9j+0U3Rm9D3f$+#X+I7nhVW|&g+A+X|@vf8e zalwj8diCkMZ{zFRY=SqM*`Tgma$t}Dl3-aDKw}Nsu?H|S-&0uC8&mh0*%iL)d`aeY zY(7M9SNN%H3r~GFZ}Es#wIg(ak8LzGw%Gl)5{WIQ0fGr)yR90`%N`9}%0Bv=<}9q_ zL+;26hwt2QBK{1pLM�zzy{n0r>B+v~B=!sOMLT0C}^8!iv$T6wO ziY+hJ!GORHb4OuecL|jT;zyl_X_V=nhD(uQqA5m(Ztlwg=f!oO32Ioy<_671*$jY%23lcf5(e^_BMU(aJLeWc$uFLys zim0wv$n|5RpI!Ssw1tYw(n6wn6Ev!$p&BE7W37)$KI=se#^y5{i=$$)i<{j|?ju5U z))2GRzg9`3y_u_jTSNY`Q@PP4}pn8y?@lM0lR+5y^DgXe&sUu4Saz8!+W7f@1)# zzPcc*6-5WH#d zcg<5gwShKnB@^eAZN9XyZi3W*PAGcYR%}8^wQL2&4hbjg7O$+|bG)Y}-G4scn#|2G zJD0Q{JSwJX5j$(BvvXhdRyINqp(dTyWOUsT(cwcAo!9|TDI3U6J~RCdR9ME z!QZX706Koad`@0}YQGb$?Vaf8Nwwxu-0$l~cs$dY3%L+qgM5WYzK~c1x6C$Be11iw zM#4p6&ukxNOh@8rKRUiEl_+OWmt{wk>(_*<-;HymU!b{J_4k;u z8S$jhPNI-)y_khQa+|fHR@>g#HhLN>{-qF4+*u**xUER-ZQ(|LTBJcI#1hxjd`VKjiZeE+$qQ_p2m`3^+ ztMXfYN0e`jlkRy^N>0BE`fu*S^Enr&Mh{8sl22ZknHabE^I5+M{7P?40r2T@qw9MHcG>CKOwnDqcqd}e zM21Ty40BdfUE#xR{P@SCja{>UIx<93qO<`&%vgKgk`AB8mW1y2)dyEp2#HymioZZL z2Ves>z{XOy9>xgUomotOPA_ff$o#Xx@B(LQ#0am>r1cW$7HCQN8YIP@0hLEE|UduGYQeHPqmXFr&D|qWDmO7NJ4qaI9u2HY4%9mRJY}K>;$A*F zII#3@{`|L%^acM&z~rI4g%ivoSyU@mOAe0|ApD(v;cBV z?}i9&%N>s5W4hyr=@a=r^DEjnnq_Q_eVT^+wx1e0-oBDND`Wjh?7cnKG}zq=0u$9s zcTmV01C-S01#2#(HTo?{W<))&G3dPgdTyU06VBbj^?}=N{z-!x7=sC8Dt|BxZF$YN z$7Q5_me7T@ex=}IvI&;Xz#$Ifud7hj<0_{ldRs#wJ3BU<@{+bkJr$>t1J)wLU{72j z$*)C%Y0+l2ye~Vd-B)7lR4x>1L}pA>T#P*#rdJG2bW7P;G7hNjy?k^uSm1A6$R3|r zP+oXL&knxntq{-=q8-COWj|iOTk5x6882zr_hQNBYwN!y*^%I)cwdO^uql<@%@fEd zJbAQHC<)ivd0inHeYpVtn|mjN?V29}{h=AoZI9t<`xzlE9T!b{*AuC3%!Ug|)eCts zS?vc68bc9`?o><-e%}J8_u(DoAB74Vh05ZY$PlPyc3vOBe`QOH*-jDB`AaKx!Q18x z7->VYRNnWN6Kqr)y1PYbdfkAvk4wnUFB)I$!iOk!`7wIT^CT07D(c&Osrv@kog4E; z0u%>zH-`P5$ccIek7@dNCEfl{AgftWGB*sPa$0WO5B?TI3wQRxUT?L7f9BClHGgk{5!=XRIljD>{cP^) zljUB7I;Mt^7#k}=Ef3;9@Uf^$vMeYFCJQx=G{2Ns=)~u67G)mt=h5q*!W!Zcb{m@N zdHW4G`-}Gt`GN~q37x?RUFS4tkQU0ndr{JMEB{$g z%+K;o4tT)QhQ;-jxyul{9E?XeZRJ)^3xAV7;r8s`$3E<##gdI)gmJ5(-1@-%gey|OBrAA_M;kA1Ot)camS-4egdvbT1SXb>K~{IPY`4p z?9yEMZr~AHc+~rPRi`I(+&IgD!8#$dJNRf{`LvzyX5sJzuER3u^{RDd$(zA4{j-#O z+6u#Zl=9x~#1tLuD*CXM?tD8WPyuI%bmWLoj{S`@h13ew7V?GhLC(IOZ>(rvsM z>;!s}vRfM@RQi8VC{b7Hjt*Ch;YrQi51jLadu44*FDz&w4q4!*UsfIe9e-X|Q5=Qnu$wRmD^K%}V%6vrmx1NXfwFwkl8Aw(e zxAW7=(hq;V`D$9b|WG>j;zjPn}k zFI}t%9`liWADtvhAmoKk_|Y?uBxHk;;Uh_qBYKPutzT3&Z-2@%sO8sPiMB3VZ}pNi zLQO||`)eM(qe1)x(5DIqQTHk)1ylbmL!m5T8{B3`rk-n1pDOkm5z@k))lK320?EgF z(REY9+eIZ{zsiH94v*2UMb$4)bZ2W>ao)423cLA-v(0GgKg=Q1YwP9l>7zOJ*No7V zKEP|+%QkKvINRVf%{nfVBTA8*TI0%T9FB$k=tZv!TGWnb(!L z#i+|Ie9&E!J@H9KX(N@jYf$FHyDj7oU{W^GS&sr*J& zI}f;~tPRv||M}fOptFg0DcZ?vhJfRWbhkXn`eV;q)v;qp8Qm`{$n)XsK$sY<^D39p zr;>=e#Cn>1p5YzL9}0s9q^}s8?jO65o;QPv>aiyc#Y-(C;}#XVGz-pU8 znVFGEdc|st3Iym|PoTZ95tnqV&G)S2yP&t%XJeomgCjmZ^?KyL^MT~qY5lYrK>Q_9 zgWSJQ)H=$he%6__*6=A!BB2Ocx5oa&V;*iqaosc$TQ7hxjUEK`>*#0DWA4jj@Mvv< zW!4kP>l^e#nHHfoM8**=DX^?J3)WRO?6xEW$H#u)RMl1F@V_-3`dA2(pZq|yZd#>E~oSNisU-fHY zNnxIDoDmzVJeUmF{Gz8V(o@opwVTy+vPZ}t9*kF5NsP(3p{Gpj_F?GlpW4JiMU@h} zxvSLd^V6_Jc+%F#8dU;%>439_bTR60{E8cXC8~ck=D&6ePQph0I48)*^W6d=YqWgJ zr!K=o{9=(JeEEOzS4B!BW9y91M20qvPD zD#um1^Ks`amTD=>x$$VDEBBwJTU>$8l##?-o% zOF{a| zKUDlV$x$t~+(a`ttZKWWQnpH+o)0R<2)|jg@PLzf8Z?WPcT45b+u59OksnB{Cj)+o z`EFej_Z)5)u=1H29Rj`pX}zgaVy)owN*b4(P5NIGGq#ZTvsT*X!~xGn9O!DrpT>*X zmfY7yPpeqkscKN;)bh}he zB!3*{&=}_*XSe-P*y-X5Z;b7g#D@HM>mBm<^R_(vax>lxsi{w=?iM`X|5N^i6n^Hmie8o9}1VXji+idK>AgZfN?^zFEky z8I(apGfRMY_5qv4&*@RXy(r(ibm#@1TMxF|yKJ|&Q;4RNga$6ZF-w}fqD^g;XjvxR znLD9B$BD!@{cRKX8|L#KvHGT)CkaR3T^PoI3K;E;rrD@SLoYDUUs1Es+N4xfsu8S#y}neE z*Vo^v6_4>*D?etJD_--E&118WKzpD))bHval{$pjyM2A)$jQNr&_n`#6Av;l31?q? z{On@E^Wkmt#s&2xSal4UKzKMC-zYvZ(Q%u|53I1^^Lj_PPn(G0I0EO8n_*Ymz`*nt z?j#?M<3%r|Rx##oO?-f((({1gJa?g*&GZW=HhYNqJTApz9HGbWRX*#>;|kZr5=pgA zlZCskDYBk`>6zAS+o1mM_BXZ2w>E6fCf|e!Eko zQ#%O*@fbiTMPFIq1ie@sN*TfR-bOXkP-La(392E9vVH*1Gt^oK{bdX?jD#RDVyVG0 z`b#g3SaS+@ObOVGyOAd~_Qt!EOsjpDxN8m1V5D4^L~57MA8UDMD3Uq^$dGMmH%W;d z`t^=SYF)&v25h>sd>Llm&3pP}&c?QdtRc|YpJX|RM3thF75fL>0T?9Hv9ewzK4gD>M}0#Vmo)XLf3F~FLO2k$5X zsti|lXEF%u2gd(Z`XdS=bz)lgRHRXZrs)2=uji&L;!rTr2a*#W<8sB+9r3&A-& zEP{~+a(Q4y-N#oJL9aq*YEV11-F@LF{T@j0)VVVGy}_z+q;VMNuIL>;z=s!oU@5t4zq4M zjm3)yDE@LMT9h`+s$K+!>26kg9_D!>X&WW14T`f3yb!pj8`}A(JM34?(NOBltC`N( zLn}0K-i-CbE@`9Qs*rX3#yYyxw~CZMS>}(R;Yax8yI?PEbHnys@AC zW85~>1`XFh8aqgUE;XOOdS?4s?ZHzwg3$dWVEP+VjU*rJ)MqT=@uq{fe$z9YRL2+6 zc+P&>o=T|r_}V3};Cj^8D+qdTh$w!e)JxO-A;-AP-uk#qd?SLwennP5+upiK&qXLIZwxkRzZ!M}IbElL$@268E3_wbPFgLE+ zQ5;gYH=NCT+ORcl62L#x(_MYT4d0zsblFY6L5< zPHrS0ux3O&X6pAjEey191zPwd<3A~(|BUoH;JLg*nQZAR*N=DYI*p%0JD(A}-<*h+ zIUo5)?iDVK-iDgToWW!J!Ct0OXl&8Tgz&(oDl>STvJIRI25OMvF;_UXS_i*UDnHOI zqq}*ykOXft*d-xh-P*I5TV5$0CQGR(^;lEJo=j3l%4(>ry4O{$KE0F-xPa3}xsaA6 zBtWSPk`hf>&P{UXyk)>K)X!yAD7SISiW$iYhE&<+Hnpr*<{T#yIC>b-VO^H%nk99o z`DThuWR;;pn~L_RkfFjJ#<C!Mz zE8cHq_SA_J%=r%WJnMV#*Fko%K^?-|J1+5?OzUk1gVEc0I-+VPDfTy3%u2PBt3zT8>!j0>7zrNRve=Y zBd(d(V_2yn=yhYL5OLO>!W+W*i#Lj!; zsb?1jW%{4&DW{z3`0-Q*^)coErK!JEpuNM*YTC^oVEabx{TA=%``L7Dw!mq=<_&K4 z959&iluWZ)yF5$eVQe)rtAU}zC5Mmre|w7XJQ|#ms}AiO@@%;yMSlxse)#7YtAg2SOvs;_J z%4U3jvwI-PYM-Tv@pDEx)^O!&+k9*WdI7R-FpeL(vB0;U1mOS>w;UJLSlQSz&f;(ZprtOx=<+rh&Sa7`u=LomnCq1fLT=PZn z5BYA`Cz|=MM?R9t*}RuF=>OeVe6V+h(<3qlT$sRpGe2k6n`HjO?%Yr?y_dGA)VMHW zs;JibZHf|eINsXUCx3)&x1`8$%#*mG7*_-$Az0YKm~Cl_izfAMBp*4I9QNSw!U()8 z7W3;+G3{d-)5RUOOiK8-(ntNpq-p%zi%mJ$XbrLDp!P)kxt;B5D5{+=2x_8tTP?^T9wY5d}-PG&l=B%nIS2!wP{pc}cZ5OmFo<#hg zTP{N1zOkV1-^}&Uhu^P~^kEB8{mFs?dTUa5rQqqh7jB>T+SfkbbFtvbL=vZYgl)o5 zGnNOs>mg&M=Duq`$AsQEeKJ1KkARXT|1h%d16yw$uE(XK9q-kEO4MR>oTGih-v^HS zdq}#(EBZ5dqNe4?ntU#``|$?&Q#oG-t- zyWh{>xFMbt0s{+{S;3IDNh8foX1G1mnPq`6S@)&ChG>$fukHy$LyMl6ZE1YDI$3Au zhM8D@X_Ax{|Gp0Fx?ghf@B>i5=Gt;rpZ_%&#?nu@!RfMF<$k8J;< zfl-4M+G2hVthF*UZ!q);JAn-0UyPSJ5=E=Wbd#keS#3XV0hdc}i+ef{@RH;Q@tgd^ zet+5JY)%M~Cpv;<*7a_AFp3bFzTm2oGxedKXWkG4|Kl9kS-CT3$LdDcp97QF$RYwm*(5fT zf0j8t%iL)V9pkt1xBVD+E_h6U_7P3S%6~X7C{#Y=8oJ^dIu>k~#3BmN@UsU;Fa`lf z6B>mv?`v-uNU%FNUx7dFCt?>8=YHtQ>cLqoRr6Dg-Y^0O#2qK?4)3C$LMsodnN_pY z9WnHxXE4nPy@{18&jzE>TP4g&%2G|^egLZvv=%a19Wogq;y880xy3~UnPh>8AU`x5 zjjjG@ypYKVomXH)#cv+@_fUSUf5rYjE{UT%KYr^FNiQ?!8#!#?o}X^LfkWfmx6A{8 zoiRwgA>pNf(aGJWb3v-CWKlskz+C~!sVReh)WGOmtjtUM`8*g%m+*(($txH9W$&NJ zSfo918HbOh3o5l-@|Ff}!UYeP?{U+m93$2Ij|4}ll+R$fW- zShp!FHE;yTWa-EC9R?jn!5K*R!%J3lx$9W16r;R#HeeP6=+28Wswqn}`?tk%#t-34|Q2&8r~! zsv+B~)Y}s@6kkHi;Jc8LDu~GXR=Pq=6mz%6RUWVU1+pp7-SA-H(Xug8S`LC+ie=(~eropG_BiE44ym(4-y$gQ*KxGd3 zN0se@C;Rmg0n@K|&#t7%TCzk)`PLyLev&1FO7!>`)0b>8%6I6z>{~Ru1ZsHx(i-@` zuDIJ=TEgu>7XbH4FUuN!$7`R{@w5p)sd@l;A1eVr=C;-+MpM03ec*23qI2cD0rkDu zF8DYcep!_rp!4ANH_YEJ=WgOOUWZ18`}`?ewy(OFQOjd)x}rgUJvc7ZK@dBAt{}qL zv5!-244nkT^@#3Id=jN6ibfcRVajZ=x#x&QBsbn<`4%^bQ1s;l!fi7a(navuSiGSu zo9Bv91hQ>ZmCALkthB-DnfT*2%s);ch}WDbH0aY3sz0K^2GsMpnnpPgjQ>gF`==K7N^`&elBTuxhxWkdCQ`C^weQ&X#`> zE~a~brxmWR!4=n|(bVK;U}wfJ`WaB?>Ro!bh*!B`_)<`v?Iq2+x3P7S$&0&_odXJe zP*OpE-r*p$LhhFLx;ji?#`2@Ki^K-+s0NssvE5cT zw8QZ|P(7=G+~yR5^lh|CH3wfv|Dn15)F>w@1S+lh@OO8>?<vPE^Qw&K*_|Ct^eV~vXt9Lxrg3lQfX$jkmsV?w>dfhuhG_mU2|gOvKH|jSf8X0| zQh}{{!@1uE5zh#a?NyFL>&N^=JnRz#sU6C67IU zNO27J7Lk1AO&96la-La?u#6*jOH?IBs;@2b=gUaIX;-Jw^q*L3kj77K zuvC{TY7{x+Cry{Xt*hX0NYzs9# zo)Bj2Bv+Se4lgNtly(AjYsp7{$iDRub1mr*&e)tPOsg@xx~KE!#PcpaEkn=J_4OwT zgb*4|FsA|mR_fJgGDI|IlQ^SMp;oeO3rlu7=*sH?Lwj;}6pbl;<%D*CXK z!PmV#6PnPVLgJYlL!bl+^RkO{9{)CcxaLfrrxV36vKKjqUu<+cJ^gGw zIE{LKlSUB~F{bd=4dbD5Z#Rj}X4MTEr!DEJ+L{YViuo?d-_|m4@Rkt`6C9%qGO}^Y zZ`hV`I7q?tD>sZp9?R~uBwor*${f185X}CiefZC^@0?7#@@f8%zuD^t)Td)j_9{R%P zyq9wJwrLg1$HL7U&hX3oeQJj#O-AIR~U9CE*T#mx!5!IB`i53%p{WdpC>& zTKYi|;(@)$PZ9qJ>Ia*xW~PneN7055N^w?;ZWiD=1LnpMl@nv=Kmw=&Eui|+wbvrr6grd$TC^tT|$y&ETbq(ijaL7p=6IS z)>%S@>?FIXY*~iJzE6rlma*?!wy_Py*k;V|o9_F2e}0eW|1*!*W9D@|uXC<*opa2O zYL$YH1V=*T0#+TNl-`9*L&Jn}Kd!)CE>RO``1R-hI9~zyL<7j0d-dtm$Xc1DsyOz% z@6J?b;5hVhSLed7I%54rgW`E$W0n;l^Ei>+)6H*81RgQxZIpdfPr z6`wfY`W+#KfPv;EF)zAPPZzalCZ4bsmLt!2pB%2-g|X<4(wSm`D@We&*&at2&i1FfiMp{SrkgaG>v4q5Ab5#h0k7ulr! z?Ky=JWUu2McI>)8z4W0g+36;@<&gLIC)nbxEQJ<^+-4U({)^V5a>Kuc3o^4BV#!sB z9SAS;v{ogkW?ozJ~=_&7htkO7Sa0b5&4_P$+{K1}V z#UVGQImMqnP(iLAt78cVB?UF~3gkZC?%LqAB^5SiGS!Iaj-$v$-Cc{ATV{c3?C2bK zKx3ICW@^?)vo>3mz{Fbw z<^S{ln|k-_3a1(>xo$?&ew;HlW;$JuN0~tyM6l<% z$91<3!Gij|!q(9rU(?PwCpdGQiMQ<#9%V=zr&a2ATYay1smxy+_e11+H7bllY4@2q zoj{Z*JK)O#{PI;u_T5<VzU{m>ba^~ zF3qm~#fyL#+uKpaxWg?dkAI^)AWOGLXYK-GPlWw`MaXcT`J#U~9qfS5vO5|FrZrWs z8&139$Gon1ZngE|PP|+F-v1i%_k2P(lHQ-HP$fUENzI?;Cwq3;Kzlvi#!Emkz-n9| z)MthYn>w;7*{vVpmtw2#m}=iU$!2{__P5@fKdQWcU}xYUozlCtcbEP7Q|$6ycYExa zXw&cog(+i(FXLpg+>pp29r!aXfOq@JE(4j9GT)wTQSm$9Y4?_-n&+j2K?e|x zgEOYsUB*o3DHFpDRNw;P#Rg+9De8xQrN9L1LW6`z`bH1`gGx7WUCr% zs1ZFTRXy#Dc`=!=8QsMf*@qd$WeWeBT~}TI$s~ZN{!4p({gh*eD(5Sh#BOw&*!YBp zbDV)(vKJ{;U$!wii!(dF6p}do*TrX=ldVF&{zVJrdRrcr3X2yvUA|Z9PnWnnsv( zz5^#H&0m|6F!Guo3sSVdyY5s1`VB_m9WI8uSK*v}%;ep4G1*J7!P+M0Uj?PbN9nf{ z^(j4wiG>O17#n^^;hfSP)R@0OU6Y}$*q`oW#HFg#kbGQml1Khq^v}xE*Q#j|e z6VBGQhrzzRLP*|SmLG7WSG!*vus;4DhGIVG2#$vWG2GjCQx)RIF6Asci zkaHOcq@>}fa789o+*r4ERvQdMd8db%QU zl$DO8=je}R2Tm~_#Ve*?2!B^{e}1ikB7fAw8(#ZdtJS;-Uiyi<;r17OKR^c(5{;$O zZ!!BVj;GJ(xb-GnULXz<)u(}GB@p*)0^R~dfDu;vMRpMClS)17^#3hZp+kS11k$88 zH{6)vO@Hu5ff`$ZS2f5C>S>gttC=|2?4eE})(vDOfp?=2{iJuwNUxsbzi?-C^jnZs za+$?6i$8XbmlhiScU80iob&q!@)&;-$XN-yzCMbikIs?~cbc(`~L4bZM$0w%j5dZ=rQ zTY58K`Y@QjJ&2evJj%Y+G}3tLe87*8vYCCj4H1yza3r1qgA3ak)(za3UoIwG;Q$6*ooI~IfU4>kc%K1a-fVumB&;b7TO=ANvWe6kZot&1Aq~2d_&D zK|ZrJBsx{|$<_aP0ki;woG!>UJ_eU^K3IA9?KVhm(jzK(RPJ0w#LGVfWtQtp$zWhh zwwGM|?QWdMuJ-nqU;a$F^PjDWVjLJC9NX;&5(IY~VAvUfD3m72;le=DV>JN{0XI+K z1iRmpwO2>a$Zu`%i{X#{$T2(ad>3B!8S@wv32w#}5i@7^enWA(NM1y7Lk?|7D)y+I`R`{VnzZNZL$s0{0MbvrgDl`8rx}WK; z8-lUm$Q-6YuLGWy^p=Xead2?WLdD2eL)FKN_tymFKFy1N%Je}# zo|r7I>>2Ly6m$boME2NyN%F@zJyWTj4WO=}+4kZgwTEdl3%^XiE4*LRqeCB4$_`ep zwxg%Qgpj~8>VzA8K7I-?IJ~bq9i5@F;@rKiHddY5+Zx>mxW3S>+!L z$RpoM5$PdWGUtVX@NbE3e$8WL^&}qSa9OGrtRv-QHL5i8kf20)$H)0o(|29&fuKic z+5RX*f%I-W35HW!85FC^Uo0?NWW~fgAAU&@q_7sgj{}NouMHZGZKq_;7jq>^^ki)n zW;Hi9j&XR{%@b4tdWY3qo@~*IT!z@swTv4*_MVq-ANrX?j?Wm2Uun58L+k4Yv)oPE zuLgl#%hQFuZQEU3r*ZNiDYj0H`obvLD&^nLxsNwix23(B#~+63l`09fedEGT7=x){ z<_?|cUJW8dxkJi(9^5ox8%#pSu-C;QT-_;5gF}LV&MM(zX}V3+87=SPTri;D3nBh~ z!6F#vwyO}BIy0>8>M>l?+?ty-QK^$?jl0)iqu9eu-HDk!)Vo9~ux(siP+7R0qO(gu z`?ctv{7ZY78EiURN+~`R*B};(coY_|xoMf1FwfehnK+o+EJz~uGVq*!!!>@2maBM$ zES2S)OK^xrm1=qeWWcdTgyi72JubIG2m6$Svr`PSIm>QfxyLi0Fg(Ejl+7bxk66zPWKQYjddP76T`4HQDtL4-i`iRKHyR+FwE=63*)D zmgav=LXSRuyNbSckb$LS6t9C%sBT%L)+V0&*VJ<&iLv0lB6BpyJ@o5vLD*o%Ol8tN zH*|G4{AhC4E<-UpSZ4e_?m|XN^KRbJm2aEH4-JHv%8T)Z-1JJ$J$H|^;r1*=W6N?2 z4hs%twAkxNeVeZ*9IDinoY6vI`Wn`bzLhcMo9AT)v&6cp-mC$uHPFh)PnhB?mqsE4 zUuz!udOlfuRXyGQ={^l!Pe0w|eEz~LW#6D_P2=qu4_bLLGeuL3N01R$7xa@kmTtH>w6){7RId$%T?P@O(& z;ipO%W*S#_12QB;G!Ans~2dsfH}tQ9nuHLHN%Yh_|MTb6#XEN*Y3a=xx^@-WB1Cn%aNAwHP#l79hIdt zg4S{&sQ#pX^AuT6*yk?uG;dwFZ2x9E$ERi74Y@UogSyJ46XXShJ^r=h-COdWy)imM z6+wooO2ipDw`i*lX zyRU;8>~-8pBGPavnZ4}?i=_PhX-;(nmNb1s37+#77DC(3LCa8J`AcFoMjV$o0G{KB zOLN7^dtsX>sNE~1b%CF}psL*Rh_0gu*dRQR zM$Y0QHLCb(T785Xql;myVlR|Oj{B7!Cp>H}Yg_sy=V)M_u^!EdN``R>$#qLYS5mBD zqrh)w$~%rT{Yu&pQAOGc*nY9Q+fN4JzZ5s(*l@-0-6C$8jJRK_6gu*a&W8sj_u7ZR#cC ztuyk!n?6zg>H45L>q`+JTu`Tw(SCQ<9-}9>T%TdB#<~i62_8MGCB5p@W#wDT?%k&j ztbNiNPGY2l%Iu5I@i_^$dq2gFwfs;EH(&nn7(9Gjl3tu88|2=YDPVZ8{|zv$!al6u zU5r^?As5d=t}#{_k;mI#{1okO|EA|46k;_h#>AT;#%|eG`3H_x*&9omuAuoC(G>eR z0zbBDbET`xhBPI2P8@oGs<9qzatyR)6sP|bsA*CA;r!92P3X~5-0hdaCdHqd=6T9e zr_YgjktAMof^z!^dn8@WUCCma_*d2v^h??+u*-n@+Pin3a;Q`NmP5A@Dq8b5c zD49M{uPi>Q)U^6UYBFhrhZa92UvJ?`IDhmYezdN z*eMhzc1H?3V1DXoJ6vO`MswtC==qr?iUmlo$ zT5DMCwnKw&P8X%34qGM70F(|LvvvRy25!M$NXGdkaEJfU4bsxRyX>q8Tu(#K<;|Q8 zIm^h=2r038+GIhI*8efGzgH)FlEB7sA^s}I>E+gxed?+Ro1iW#4t^wY`U4*Xp=I$o zs`5u!j`G{4WQO7TCtj|whyEINeLHBMs{_mb28xawQ0MOr=ykwE6)OaPVH5Q!Mqd z^ki;Lcbi}i@2#<`Vh{sV?J7jDUAoS<35`jvAZVR8gC%n26haBgE?t6u+H09)foIu+ zmM>2)7I-A}=?Rkz&0|z=y9ZI*Oq0AFQv^dPTu8N*sn0;C;cvy!zs>|-z760bqcUn&@ zosyXCl>BYgH7Kr0mfo}?dcJ$rtl{vnz?~ml}vaHHX z-!Ag09~(mRf^u{!?xjuIEg}XYt<&6wcKj?0ZN*E#R=5yS+9vD02ZzyRHWkItMM0U@ ziPr8bt^%aEvK7-wrvzn8UXlf4F_3MQJL-GPR^f{y_2Wc=rWF;do(nh!(~@?~*nX{g z^E_EZmVEtyY1%4hthBfliI$^7rzlq>`EWAzvD8D}aKv&^CfDc%D{3)(=OrNX5F(5; zWwfHa*frE(+TDVK!<;SyyNm}&S9I-gD&~^GTHZR(ts;ZoKi`&DF#b<6bq=pN449(C z{)Bn@Og(GZKYY1R#ZtU7ptsRl92kjOFZqd+ROO4V>w6OHu>C~_GOQe}I8WAF1e`>w z?mZ4HXW!rHto`jg%hd zfSBtmc8jXVA%6P3@J$H87p}JP{g)Y%){bl}ziPV3>XDb%FtsF8T!yH^tVd0)Xe-?` zFHTM_jyE~pH9V;-yXOB+%I1rg_0_z;B2nP<{as6?>|R56mq_tf?@GfpXn%?cpo2Q0 zI7|tn0$LaURgyT8ac;^pL{EQf^`#qztHq1-aTI-Adpd97qt1^2ZFlp=3=7WQ3m;PI zx?haA>Y}Sxqn+N>y4jJ_vo_NApR_X)jHT9Pq;Odip+kgV2ES}ZiVph8o&7ziLrb{) z0;ZaQW$)nHnzjt4csg8Kc*JBg?Fg3kH^dA`GPjP1XHORrJ1$TRd!kq}g$S*$S$gMdQ+;S}r-eX3d~4JLL%% zQ!@KNF#_gRMfENBGGG)T^U@E^ne@b07uo8z_dnQ02$;PoqFk%(3qiKo$@pm6>uq*^ zsmX%4nt0r9y zaOvG>?e?1C>Ik0J?i=KW8A&K?<%NNOfAe$Ks@iT=`U8Zx<5hZHppMQICf`c6BnGIl z+x;%SaH_y}=?Av)jv4MFu~t+J4!UFZ`l3;~#0hT+ngo7qxZyTi$>EaijEg+9)W= zGq0(k$5OX^KBDT6`%1`^QHyzw67x`7G(|n%6(wcWO+#F>curPSKCs~TXl}>B z^oKXbYDdYCA^Crr*T(LeM@239-nakHnOoGcrwObeyhYueAEkG zLtkr#5=ZM?&zKZ*i=6xUZ&iQiZ(CQAX6>BG6WXn)!@8uIFPC2rg~ zrgepjdr3uHFNjvf@*4(jJF6x7$9{jq8%%x@ys0C1N2hYh?i+A0A&DZ!Mqm6KC(esm zjK6x=T^k_bQnQxf$aB2-B3AWkRKn+Hds)`2`iwZrnmY3>PQq(GQpvHr(vWD18hB-h zXEZeF=(Q?t%+xgx zS)=6mi-#U6aS1?V@SPumcYU6n5omvSh3UV?PxK9p#AvZ#RkQvYs>25I=87t;kxTnE zSb9G%JJahR~UD0nZX0Y$Bcy<>HE+GG(JrP+;_O4gR@?glbQN>^E|+D?bT;y~Q) zJ}C|fgGO0TIXttibTC(O>Y74@v?vFVh1#CONlbY1XHd^fQ zLXdweeu{|v8w5-J@U;3H)Z?AH)bhKJPQfO*wbP3&?1=5(>wiC4a(rs;oYIQ58ubV! zD|IxEAXlImDx!r!qAD}_k$}U~oiXX-ff|3Ncy+Xw0P;~|$meI!sxf%1$PyKBa zhc9^r{F?4o+!wKZ_u9ElyS3Cr(~k1nymW?~eNb6Os%9y5MYf6w_nZ556SS8FY}97$ z4uA6Q6Vg5O{$4?_Pye{(DMgJ~=*c=L{039=z-v+!mL4&wc-N{#0pM<*ma1iMbzet8 z#OsdMY}G{xrMgb58l?>qQD`eb2e}E~NkUD%B9;0#xj~ntz;#|)jYcl3Vgbh&v-6^q}6!O?;DgX zq^=B2)Gbzeo8=y-8~3*Rk09o2(u4G43FB}Yn+xjZ`))DANjX=YNB0#I*vk#hTfma+ zWm6v6>#>Q1^|9)SR~539T4qJCHE<-&g9T38-F{>TVdx;b+AR8&X0n`yCW1@V*H@pw z93MD$v%efaPBsK?cR6>67KTX+0Yz(^UgURUxqL*WJ$i=YEmh>ca~2it+SK4Iq#Z;H z3!7%7hP7ii)9M6s3YJ00Nx}SeHS&xN08<2N`y@3r>|&>UPD_qq#d13HDlRa^JKS*C zFqS_G0jSj8-?_LdO&K_7Z}LtIC$StiaDKRBmIlBjpo2dXG9UcL;6twbvUodXTHla0 zMxADc#o)_UiY?WGUKgbn_*IV|r{JUc8@yS&4}0W}-QYNbtWBjwb;nNxZQPGXa9l=2 ziXb^NGo!$_Cu2y)oN%=7ftOO@#O!3U+#jYV2Hyw*zW@B9TT876c6;B{Sf@uGa(gds z!J4j~MK>}aEe7icj;6A0Lmr-w4Xv~D!?BsxV- z$m$vjVGZAr>C zqWDR_GyBBMoq|QadHwDCY#$7%JjG)fVYlO1r{^wbh9McduiTlT!LZ@RK?&pn4%y-L z8oj__+ujWz_}=L)6-yKI>GDfGueGrM$uthd!IR{Vwv8n!wSxTSw>{iL8!&UfGLx|Z z(B{0}N0;@jVc&bExWyb}onDmKTM^FCyQQcJYF>BZqBHn#uf#9CINpBYp2YaJ98#F2 zuR{jS*4#+LRuP)rcW_Zp+I?;6ssDFIej25-ahOowYYR#=bd@8l^Yw|`{sw6oi|nV@ z+<#y^X!b6KWQ#xX1C3bk$=6g~KVZQ80=uB)Y|Qec)PhV6HM0}$LE{_=y#(Kj%eBOe z^_iknhTIKmb#$q&L>?2$FE!)6%7-alveX^p6cQ?F?15}+GjKZ31-gkC<-(%981R%-ln$+oRXC&Fa3exzzdqC?`mPq4@|{qO2Zx~@Md3fx%6 zQRC-(Gt+|3maVJB+x+@y=}QFhVggHopNxN97xB@Ril13%il)^qPikR(TY28~kYV2D z?RN%`An;CoyK?R~%^$aftujpG5TD`g_N)_O2`ALD6Dv9+>-(W(mpWr8uxY(8@jgjI ziXsv!s$&`Go{)6z)Ivru6aV^s_MpB~LSixL+$Y=YCwaGch6#oX&hQ3s6#bT*~iIlmZik^=|4RaX5#)Ma>> z`Vc#>r1Ckv#xra1il`jXMFnfmo_*X(j;Qv zC455sE>W)_kEHEt+`4Hb!iZi#yFIuVx~=UW8*tm@u{bMu?dT^ULDZ>e0uQq0Qqjp3 z={TMISiZ^=d-R9@4!5nGrsCMh#LS1M*5_?$5@r1ody*hqLM|>sSfk}e{f@EDc`5gG z#SBxFdgRvO!*ADk!q&jZ*Jzcb;avkAnQ<;ZPitKC4R>m)qM*M(ft87meTD@jKtfW* zSI;4W|9OVoBkLa7=J5d&4HN?e4rVtV>UST`_&O!Yr4RNSMq>om1XF{mXQkw#@!m6e z`OzY?X{@Bg#xPUZfV@(YRMXJ9ohVKUXdumP2Q?rIOc8b+Qh8tS%72vrM%5!S7I(ceA(^e zp);UFaOo#Irz{@J@GZfxNKD*eR($R%Xrc-yKt#=4jfw>qMMr@ftGTf&M`@ouN;IG= z0_2OH4or>|)00l6`qE$nYsc%O_T1B@gRTQ9C{y<}#jn0|VqiY<^9`2F#U?!;MS< zxkgR6>JxMKvvj?P#sejU?O=@8e!q~Ota~|JEbGzZ?-S#CBCZD?lnv1?U2=i%cjIZs z)~=Dw>q8&Yxn-^OCaM$Y5gGt@`x3x-EH*y1Xsfj=!IP)HFrh4(J4@xgud#I%J`lDz zVH8VZM%fo+A$sI7D;){3AB?A|30o+8>SRxQ>f>-o@$hN(m=+?V>R#K%! zUWX37Wn-S8<(fqnvITySLdw=Vyei|m=)qsCp)k-H&>tzR;mg%P9l*T@P{lEO!$&a& zuvY&VVkWwIF|!sIistPHLkgJ14o)4dn4fHLrO1F`wFm9BzJ#~anM}d(@t324u8bl$ zAQNZL*;wfo#{WfYEV3(ww8XmXTigtRCt&=DdqCb>%eyNhb#v)RQZYO6L2ReKx?@w) z$bJLi@)xOCvDw1Fwdwu!bUUAzY@4x8X=BJ2lw6XDPYHEHTY$phJ(eoTA6DpmWE*X- zRo2_(fnJUF#2L(pD;t`t&L)}fJegLl`c(Yi_w8&**!>?DG?Oj-ZdVt)-E;#VO+fh; z?-c(lV)S5U=TptD^aBWd$Vg|Y62MuwFDlZ}hbCmWbE<9zw5ygjAdhO0!7rqKaf?|d zueFXLWEH>5?E|QN;Uk_USD|~w;3jT!j(=wNt836l@iXp`5g&K^!aryw)IWG21Ajh7EqoD+Yn=NE}0*zVYux0oR;P6M4^+C0V zD|#~`bpr&b2{s!PZ4k4MvVcXOAJ|TV1`pcL)pE?0I(elD1ERReRl`>2$Ec12qj6qa;6$2Be3 zfJfJrAQVMdwcF)_9GJ|BX$!r@9nUlM`MnM!|C|r+_eZh5unFVI=-y*YB$i6m_eDlG z6b-|Sn##24Qvz#d$Yqwv>DR-6$MTyzVgw99sjk9^tX%MwUC9v@$fqjzmHXBqNwfns z?EHE(rEVefS@(juAY#2ShnW}1e9-ypS%#utrwBYv3D&8Fgei~6J$0szsm_W+1y>*V zn|Q8W0SkBb%cZ`A6&a}WIu4Cu3q7ggira6uRv-iEGdUU6plWN!bVVJ|c{pKrXbbd4 zKX-=Lp?JFSJdVT-Z=YKeRD?Eb&l*kBOx4Dt4?qA;icIv+Wfud5<`?MUF+X_5}yfOComEqQB&7uLo4%(;WEt?0IwKUl#G-Fse`h__`4$o z+X@IvFoP6fjkk8^*&tMryW%OGu0f+XJ5S7#zqsAEvI)s*H3u+Vg&pVH*paza=v*Y< z%1H^}RK$#a^46qAxc|P7^vUC%w6~%cYAkfTN0dTx!`v?)RZ6n!fSzC12x>vr2#!bU z`77OFZEAhbW77(GtSZEH3nKh=M7i2FFd)iMhuRWvwboyNC~JMxvm5#aLWCZ_5_31v z`u{yI73`zO4Ww- z2f*H=y+65f&f@9=FR1>^)rhef@wbIH!`r@Q%+i+Et4U0OESzHHlZvo@@5p7mV+=0T z-?E7m<7IO|GB@|tTCCfzDr@?E^DRLGY2E9uzcO;@GE0nj77(sF^?Z*yibaH{^OU={ ze8_aR;}q9m!aX#&%7ine%rJ=1%}xxa#=&y&Wa%jJ?gq(XVJq3UMrSnGnEcqNVwgz> z$dCt^F{bnI_T{~*X{I7IqKjd13fygImXbAV*|hOOb{$Ke5LP(2-Bv#_$@U8!RgaeC znz3t>AO8@?DWxeYIVv5sH=Mj`V}80~{b4odcXGCKF`iQl1fJ@;5ZSLG2H>5as znYhAaaGxoDb}dB-dsLoMhB%KPa`Y-a;BJsc<%~t z;Mx0MCKar6lj1L}?AKajZ(r+M%`u4-C-IlF?r3#z&|(t>tIEik{exg=+opJo*75bC1$x1$Phy=+Po71=emMJan@l|MY_N-5YY}j-?$y7m6 z${g>-eN#VB>>jAjKrBxS9w&t}Ud1mGico{ip-i6=j}|Ep1z7@P-F=UMcqwU!S#22r zM#@;jP3ImtOyVw3nW!srV9J%NBL5{!d=w!hH(xXc&vq-VqSxp8t{g zO6S=%5!Pp+_o-U3pSctNX?lX76vN$b0<}&go`*ksb6d_?EG7P~-{7S$qV*!Tl+Cyz zrfdd_t43+9_kH@}4u9s4@J?UV5>WSm)ME;D0^@;jiwNTN(RkUzgHZi8{bo*j#8Q6LhX{rMb;?qG={ye-_*+&H7Nx~iJaSpx zqW`+XhQvlrm;bI-s2&Uyngglo|AH4b`qxoB?uo)#w|Ws?48fX~@a^AbzU@Z_^E$>z znhPN;*Q-8##_eL=cWzbhu;*3YGU4_M_0&>Al^48b?x1%q#5{}GjyP~OT1{Yv?CvYh|wjzt@zW&KO zuK?QK`P!Jpzr$769nTow_&+EQP0pk>Sw|`urCZ0WuNoqQ>W|C{tNE(bT^K&0i-tZE zanla*D@1Kp-yw$wo&c$@L zm9<=e^p%T$MKYz+hoCxbh3}oAmuZ6|BzSTxm`*rU1~xm+#fqG9VMTPPh`h>uqQO#F zNSNBu?oN#Yx8DuGgrH$6n=!Xe|Jo&ky}PPxG#`8s5ba16php}Im$@~!!*m!uk;L`( z!;fbY40n8=v4<37hH`3@YJKX=jJ2zAq3@JHDpeJ=?uwadwS!#(MHD3nR2b zg>OlQfzc4Dy4?JsqO2@h(X&~Hfe@qbygBpfR9BEKV-Kdcw%vTK#Czb=)$s#9YdlLp zSA&DLPEUg#BV>MgKyaTV0#dJ1#vykEFfk_}fK}{njsU8qV7@mNDSJ=`0?mVw-_Mg` zvEG}5quOJVibt&s`X=sy^`1l*(f_b0OB}Gk!2mw{4a&??K5vpz8gZ0%RQ=+Uom-GB z!ew`3oi-)1mfhNCt()17y{1Zsl8CIqq=HUf_axMU1T4`R_b=#oKj41Ut@Hmqnfsi4 zCy1+^Ucu-eDF3NV|^WQ$O9?8LOrjVz@ebHx|6rR*lJ+C76I7TmD?sg zlA(^finSWRn*r)BQ$n+s|hI<FR5q;i+22?j#ym2&ZaI4imzUFP0<fZ3pHd-;{V!yd>b_Ns+$)&xG<1F5j#VkjMR?F8 z*rByoen~lkJ+he139vw7K0x4D)MxkVL-pOnaJRhyXi4Hk5_>~;3dj7-($y$-xriys zhX;m_;k$?4$i(T}mtO9g$kGqlBULbxaf89`*(Vcpta4F<<+F?>*#EdMO?w;KKOv_r zmnwIF&&&k60_co_;bd@BoxC1}PI3>ShH2F9{VJ-O6n!mgY<{JpZP_mGxCDqxPY;}#<~UbU8us`p zi^8tI7|-`SLrL)q58lNC63GD%TvtdLn-cS91%ha%r}fZv)JrWPVA;2<9+012Z@uI? zQuVjX6wZR7aiZD#E6U|)Mx|KU=4^^KqSYIC0`n#Km>JQV(5~Q*t?{ht)vjfAF^VL{ zuK!My?5KljQ@`_aN_|-H!RYtGufj|jLv#Q3M%>6~O=+Oar2>uN%Qgsymv%cIz2~}C zfIHJ$A;F0Si3Y1P8ACbv+7I`~%xL)|St|XLZwqM=IkjRSst@)BTcqB+3BrBJ{DnAvgo2^~ zm+=LPvWG7ok0>x4p>dyQXRQ@mp3>@oslY57`E&e9m7_V0D@%6qap_8tg7&`)rnKF) z8xED&hg!|(EvX1~!_qW=Y6GInXjTHY%6U9|tt@tWL|uN*6Wgv~3;BhK!=IzaAIyJ= z4fN=m>N#{e_FA=An*iKjKz6pZ{r>_950Q8*D$$gV@2QN@aSDkCzzP8aZ)EAkuMR|G zn>P+LXU-SCT+5o_Ja?#l_iG$%PC1chgK*FG(~KeEcJtfl-pV?DzkgX}xvsF8=Sy)X7s zmlAElj+`*$EHFvT-PrtkqVlL+=i~Vgz!4#EEJi=Jux|f)_#(Z8inWpNomTk;7*THKM>cf@&i|5OZ-+Q znVHwaci6l=1hr_~e#639%|?eI9(v8DJs%{~ieO{)5V`z8TC%Fme15S5{MO*M{$#*o zb=tDRh@gOXShAv{=zBjzQEk4yaqu>NEQ0%w)M1`9-!DnUmL!8+3RGss6C#ZeTI!LY zxT$iO`9!-H0m&a{qXjaCxZJ~nWB6eV`ks&w-|M;=Xi%i?$T1)YXaRv1jR1Pim4<)1kjPK6vG%CZ{W9-R!QF)Bj4AKNAiMesv_cHZ*hR>X#y}K2J!-WQok{ z5Z00?;9nVwVZcxBkZ;{f@NCM(1wNMi!B|n*vho+G&{1DA zT3@eAN;ovD$&CRSbBAV_t``kSOE6}h0{e#xYtK6@pisGevpTEAV4XRJ91kBPU^%nA zd82rT(%rhB_IK7`E*sFo>c^yAT692;?LA}jY<|*a<4h_lce=U{ZGF3);s`hzW9%CE zRUj^i_VZ)a+PaSAAP)W;r<(HwhiqdG<$XFOITT_sK%~ z-fnaQCJ_lMPbP$P+}d%1jGP16I;PYmUiDd&C%5pyPWBULj8=7^Ob~+ZgYS27I494U zG^MnzO=gAR`o$smT?qNcf|&j8;!C@2ug2cM-uL_<75tQPdi!*krZ5rg7PYz4>!*{FUx2^4(_<` zlo*+~w+W*L=U*JB4f0Dl#-}GGISP?aUL9mper-*u<)NI@oa?;bCm(fJ+!*VNdUhfb z@{&#BoDkX4BM^kM>#ZYH7K)LbXyZvonx5{5@@x>O`AHq1()R20;M&#e2k~;DhB*6@ z@x~5Dg>|QpXRD5D%icyV#i2M?mV*BC+Rjx*-aB5O<=wQaoB2pAo``FMk+55WM8hny zA$Iu$a@6muDW+B{(Q$TS1+maT+(Q))KNMj3(S&nLuLFRsj?`?FEs-r=8! z`qzuZJ%hvfCFM9x=k&Pz>nA9&yZ2FWK-_ZfNM4WHePPY&uPD;xlbhC#DHP+T!Re2V zu=nwaffs@mw~x(!SzO|;S-kn4s$DV^ANV4A(cSI^tQ$QuLyh%7?D>N|vMaF``vQ}N z{zvrbORPVcyQ5uvGJ>kNE~OUtXMXWwGo9|PiH?!GRDD}d2os5LAj}xaCm@&mDOV9b z5aiGD`JaH3SkfO*h$LJS+zj+5p4AEXkW;E=F402_zP=Yr?=RLl(A4ppU=nTJCrT5?3i9lD!)Lm3H^ks@uA*!_e#sDtT_(!+NeWRU*Xl$7U*oq7~y>$ zC6}5GN_Eud0qvm5`u>s$&G-jgk2TGJL-4gqci$G^Z zP!K-&fv#(}g^U%|U4(;9!hPuUOHK@31)MHu_^NVm%gv5o&D31ZyuR15)KqRXo*Gn@ zM+?}d7i*-UOSIJFHb15BCI@}H&$K3 z@6|uI>*6|5)iCEacEjniwFkSHAC3pgCgR>+X1I7B9`CwNCSDi?cQ}W5`=%}6G5!xA z6DJGk?Y_Ra{)CmGQ{cy`~8mfygm(<uV7r?^$}cD+emeE8bUhx0cpEY-ibhFH9TFsuDttKIKPqy@_F(&jt@f zx+CT53`DgON5@|!0Xy{5isUgWUb)r2sd9^=SK&@EhBF%C$hUvC%hF5$za@ARa!EyH zAr86t^Rkyk(z_HMxfSnIpXvdjB^^;A5vLk%52?D*mE3sA1mIfP8&(aN|5|u~e3&CV zee1AE4NZxBOyy$53xYyZX$$vM6H8%~@@19n6&wWpl(sWvfwGsct zIOI?Bb4Zzpe3wl4lg3tB@*M}&lc;vZkIEgZeNP9fdOy#khAPArk;w8M>Ke~(_dGL) z5r^k`xafAIiYfs3w#Vo%`=PJ(b`leKKr~YHev|Mxs>088SWH3N>A|8vq29(zLG_CX za}A40!-CE_tTRsy2DjRRJ0C%6lOJ+KLOkymsvI`v-Ylk@KUPM9&`$rviK@wZ?g7i| zCcwK-?N0KSYkF2`LMG<;n1`#C#>Gyh7~#P~V|*2Sj?+e;z@-B-4EvJReA9{&Wt|bd z4p#dpL^x*&=f=e_p2!*kHO5N}G)ewl)^*K&;M}4oS2?l522`qJvpFl_lgjUuyz44n z(G#o8rf<0v#IhtIZ=5Ogxhc!Zb78O$Gt?EHOzmv8vS1rLUbd7dSDhx8ToCfJkT@tg zr7~*B=ysN1F(*cCcllwBAXzg0p>#T-b7Nj<`9D#Z0?Ye#V>O)25?gt8@{ij} z>87YDQp)>XmX`k_<6Q9m7zu>2R#mu1J}KlnUe%1!IJp=INOR<-m3 zN3st}u4N}-gReJ(@s`IkR`TXvJYVei>fAjK)d*VIx@Sg|bew$E5?E4qia0`@E^<=L z#QktL#7#584y!!c1CJUC^7=wTYeai{Xf#;s_E`NyjRLm71+92bPqksr;MhvvLBYUCOV}MQ_%X^Gq*=Zp$|DJ@Fwo^`uf2gD<&5?-o(m%tV(ii7ouL56}5|D$w-1F%Z%F(+4s$+zA(IU_S|pwDt?GIO#RzO&IIwY}&t&Mq0# z>2w>4Dcfp0VG7Y&tA=g`N>)rr(DJu<1*I7~4`JKRW0d7e+->Que|^lR|B7{8Ewf;^ zge1w-u5^l>o@+b0N{y$}k4^2kG(M`E<&XG6-Sb!D_3f5D}NJf{V1*KX_GPzrcQ}@xhE3Fn9Oq(N?tJP zB4(>2*dS!1(7F`PF3u1vHwEQF2c9GHC*fQp)O_0+f;kt#o| zr5NUalS3bU!eMMu;cJuR-P)TcT3$x_HMf{>&m=q{8+8vH)~K;ObYkE3=FPv9Bepa1<`C6*2boBB%RV z>A6w^qb-7Tm2L3!uZtibWJ(^4#pa`|`rXxH_{kHhG5tuZqb!~?iR16TbO&&rmjhJ+ zsoAUR&CH)?5~$iM!`@yom99=st+<_x2XM%r>+}0fr@+!ENhg`bmtev~B=!t>*YCL`YYXK`g-_U&xsqA;h?xMal#F8H1KD zSeiW9&p0()uL|g@sbh5+RB((xTruUyIlEZmNE~it&hhMPhnFmBTNR`)TN{CO?j0i8 z)!`qigET{OB0{|WbxxSO=|98f=VI7`i>15bQNh)hmu-9~-2Zht39u7DNg^>= z(`T$nc+H~2c_#L_&F>kMe0=^1K2a&vC|eG|l6Ur$>v zu0Hw9h<7IL8(ZN|l*V}Dch2ZolF{Cn7|KE5Us*B{U+^Q>pH-r&?DX)B|$Je##>a2 zK&r3(5SE^T7*{N(La}65Ki)^)234U__#RAu{B8Wkma~P>cD?B|%f^l=WMx+-lY>!( zrSi*5Z_)O-b}E5(Fn_!KTgkNm}jL~wZw0k z2W=@-ZjS4YWOemcxVWpV&CK*?mz5SB+kl-_e6RQadDwd^4-$lE-6OwMeoKo}l?sva zN|E7_121R9{g?adkCk32v2rGI*ap->l)==L$I8UOr6DVe~6T;9GN6p7DFM6XB zRyk4HcEi$fRtqX^LmHu#04Z<_4}oAyA~Hf#buZ&t_d_4!^M!9l#ngp=FH&A2$hB_M ztblSvQVkCWM6TWlHhD7EC0U{d)aZ(z><0Enan;941#oW-sZB9QcI;~1<^7y&)%1Ca zP3Nppo{mG;CM(=#*VrY=QrZ>9_4pTOv3$#~hY`ipDQ9gn*>p(jjs7&Y%K=Yp9)K&~ zHYViP{jcu2HZDP$`wyW8k=`X%bIFVS5=$&L`LcqKM8%wS;Bv%Ouz>79k>Mw=1jI;J zybrtm$n2RkY$s!V&1|U)X<2PYSAIPxc~=1&jm-?l>diMb+1A^);;k~4Ku9Szu(K<) zfM+0MB`hH0+^so)fn~q-jG^@JGu)FaGJP)H)B7OaO6>XpI-l4V=5Ge)(rPZh)Xg;Luw?iu?F6l{C(?Xq^`VVt;A7Qk)h~H zEFJe$NAN5_RX}6+ln+KKm_OrU36M5|e?>1lQ%GNPnC>(H#}jIAN}Op zGip6$@r zz1D$xm6#f%mJO*>JA+u!omZu2&{t@uX30mvw911qHL~F@@Jd`)O(2dn#RDl0X_cNj z4k}9M@!uXs$;ahOT}{728^A|{;AuKo*Uc3=AS_h*y+qX3Fzo!5`6gC12K4TUFC+mr zdCExz=5G8GSA*2_8Ubm9K#xP5GZ=yc4)=YPM2|IN!fcj9qR0tZoG80tb`b@>QxS{fMEm7jK&41ch zLF$~dIZ+o;Z{z#FDp<{gqLid$PL$2c?+_Hl2@|50C0fI&g_`X7YHgDgd=V4rMlylx zX6P2>p*n>USsGnxZMvxjeKmzDZsQZ$-se5z+qLjQK~ZlP^l z?H@`rkF+UzsWz@rPUt`n)anv<_&GgB4RmM#YE+&R3)#`2<-X}vbhi9Gte!-rYyl6; z^{NnbS;&f7%jt?bp{8a8@Td-Yw;|Z4wq~8{io1yW8!d__LT|RB1=OWLnrNMy(<*U>Y@3Y#M~_1KbtbvFBCyJ%ZLKbiKuZM!`E1kkj02#4UpHIDpZ!I4u`J*yg`|k~E z3!F$k+`##{ z;Tp9|=&Mz(_J~ZeXk$n(ND90=MKY@WywK&Ygk=vG&`UV6DPH5OvD6_z0PgN@RPd`F z`_r))*DE%Qn1HPDi>{n$!lm~)PsUB0K2o-hEt%uvlTR(&I*rX6jnK2NQTO)0&V4eLkwj;3DG;LocyK8cxYJWnc? zz6krep84!+d0FhoyW_qXi&ZUECGoDr)8}QQiW~P8emK*+3~uXw%<~xw5)2%7+|e?# zK?)rXE}N{Qav67@x3M15?axlxTMjHUUgMvfQ`i>vbyQ?kAmnG0>!^X!>p_P93C4vf zHetTEb9#|7Y0=FbBt*m|)2i$LnHn5?5L2YkIn6EFl|fCrn8r3jwD%`hfkU-`bvbcJ z%=k|+puLpcm~=!(Ay$-uHf$JgwbC%Y`sJ%D9k^S5HKrKa|LDi+63d$+?ZeXcnNm^P zext``X8hP(6h==#mA2!Jv~g<(CW}#$>e8nzEn^Bo3z(QmEN=e=rb_IjmAyBNuo6})Jb_hk9ZsJX>Zprm-)I_XAU zH}-28%`?LHn?Ma{ZEe!^k3d3(P#Uw6zxcYkZ@I@8^6!q-k3jtC8GXr9#Yr?{L$9<< zBKQ5#CmZ99=~?d(6F?{}7oTZ#aU^l^BljjR3!8N%xn| z^vcIbsO4dW!l}33kIsZX*1wt+_>c~$y+E{h_Iz#yczH$X;u;=<3aCVxG+bem}pcfYMgNbCnksKrK0%4`La9-(^YlU7<(l*eY^p zMQo^Jhd9s|QO}oUCURv95lz@aP**{`6@)YmBdPH@HZ%kp;uVLgFkTWcnh^)u=2vUX z#Rnx!yD`%^=`Kops^?)!t+wpCyzsYDhMbrKjMRC)by?1Ii&ev99A=J6D+r_sNgg1} z!O&D1K(EH0ELM7aT8&Xs?ubt+_X*SkI)$5&>Ed%1Ar>p_Om(r<+2rlc#$Zj?04y-8 zvV>Lrx7FFAC~RUy`w7%@>f3w(mJ)OsNOE58nDZ{nB;fNJ$955hes%$b`-b)$bVv)? z(Y|u#&UrV2=-e%q0CF6pS@WQo<#tz#MXxIIL-a3yR4*+C>N;PRuPX-jEGq_LEusIm zL3_g;I(mjyMvu$hK3@4xUFB#=OnobvPEhNX(ycmcUcHE7@uNspYQo>SC{Fn^c5=Gm zk0{KO88wfsOnDa_aNx}1WPlBzCNql~@sd1`kw^jr6-^%ra4#S8_%taF2gYb2u{GTgP5ONA6> zK%5`LQcf~BLRAyS{hZ$`c--$c)ypna}up z(gF2MPwm#)*yrCjARZSmsCkcju{M-qw^DYf>9>)+H`R^GE(9Im+P7R5G@eFnkJNI$ zbLOh0M(YY=GJf3Ld*?jwKfN&iz$_{F)mpkxaYOkvBRS6+6?vE_hKhEt&s*|3@s@#` z)H3%K>q~B*SNf`6Rc<|^rp1ZW?umzdp8u}O^NMAq4A^ez7AuIFtP(pXJt3^P!fHk${F45!TE$;@Ti*M8RKBg_Y@La_ z>3H|qq7273_>i=pm4-m$_zayWth6akXX8{)1Re(pP?zpl9rvB&4O!%I!=|3>0;C<_ zoUqtuQ@jp)1%bXD4LlbWd6&5%TS;S$T1hoygxi>J?Gsp8<Nba||4o$D~>@1tvoC^^@?ImZ3WUCHBuhQ7Glbh&-!|Iz)j z*goKW=gejP5fRjcx0RJNDY$rxlPnHW9vt9DrUKC3J?4RN$+_kZxSF(`|N}~MF_EAdMR6jpY?fNQIj_Y$xUh+?h&-EUUSVZ zr4LUQnI^XK-GJs_jg{-M7u#f9%}C(^h+LPvp|*7L21{>Q3hKy#!!>|c^O%g+tA;F_ zdrX(QSzFinbBK#VCiHHG?Xu!Rt=VJj)LCQNN$5Rdbu)hDKT;Aa^fD(^q8}jmvroRW5x!vY* zR;&)hOQcPKQ-9bea#HF8|BHBhY7JT7EtFH5V# z_-n@+=D5-a+N`Tm%i)29vrS2$zQj*eHph25ckTr7GyXj&k>11rrcgaMMZow?ZtSwf z-LG}Fihbujsq37qI7w6UWxpeERLj(;bS5D)@~8cwv{2+`2?M&%Q)yvL@uzPmnM;#` z50xc`n_>Eg#ZGw&nij0IP^`G%nqYYOBklL_qV0ccvM<8RI3mGk6jqD02}5EOiw5UO zA4Slox^mz1vKt4;fBz)kq%$ZE(+e$GYrNiby7JTPbO)}|sK#zXf*SN>HR+?j>s?TS zZ1$!@zFPDwuUolXpc6s%jVwcNo=d2V585V%FL58ht!1{? ze7EkKn)jkOg)U9(6|jVd&i+=beiu2J>v~CQ zIQd~RSJWr=5@uw8DDhshKBqcqqo&lIOq8AiyzO`w4Z; zN@u#>9^-b(6r^%MZvT&rL&v__UgZ-~@oIS9UroxK$4%%r&Jay!10pniX+LOx`cgno z9Ov{*_}$qE`K-*&dU{Zr$iEzX72bm2z_Dh|D&DZ^_pU`h{x~pJ~mp757u?Af0Rm9I(M2Ne=jj!)O=isT`_zN2C8qVpS6*FjIZ zCO_)>a(;}EB-EGVaCv^&?Aj^Lcf4e#YiZ-0#&MqPp`J>@)&UxIz6$)f9P@m4z*n~- z#HK7#gejK6VEssrMP?IUtVet)in4qr*TcAbSI(qt;8=YSj~UGK-;1pDGGSpjjMhvI zY`o3mcF|d$@}{F(?|i&wLfC;#<7~6JsS)jIzB>AkLrJ|;Y~TkjX;lIxo-q~BQxG5U zyh?7LJS!@+iKYWRkK<*!h6Q8*7J;%QYfyABA>C=i_KLaZX^8v++_4!#VXgcSz6XIdjOcy-SB>#T?Y8EWvJLS@0N8 z$v;r;>$6u1@Belnz%gmF^Jx3T(6LHrpME%b2z8MX8f?_nJO!8cDHZXY%3flU07@$e zEMtIPNz6nioBY2($KPFQUShuaHu7^I02Y!x8D;BJQB{!jVA(_6!;w-)$eTYNaISrk z{ZD!3%4Y*@VF`7{;Bh=pKNvr2d@C|unr+|L-4ZH_8aoLixYSt;Se$;gx_Q)~VaW>{ zE4U|$$ElnGwm$YV&y~&`g3`A#aIwGy3Wg?@i#}sA=N6`23?sO^O$mIC;l& zcYEIU9*!u#j{Q7ihL)j7qZ`xSj7EbM`sIV)T=lazvZStZ8$w}gSj$Q7Q*U>IyLXW; zt}eJ(_^DeSeq<;p*k?Zn-IipC4^F7vRj!!rlgh!Rpy!T~8wNArw?7yV5bS$fox7t1 z~;?F zJ}tQtAwf}^o`dFnLlnSa7_kDaR%5F9Bpo;g{jyYHUZpKczXF4ovGRXW>zd+!a3gz4!q zI1jud?$NJwio7G0UQExHSq24$E@RTgkmk8iv;5${Pa)#&&fb;?p+i;Bar?SXU=|3g z3zr19BV3!Z`n;efhmJu|04l)8DApoh&h}n)k!>O-B$4VeApg*0bA_xwyR-9p5_MRR zfWNI;i`w`8{d;D0Hj)S8nQZ^BwJPjq);$8%Z$vZ{$<)daCI2#FT7Q`OryXZdq}(v! z*IOwSsa5K(G=|C$Pt`AF#1X5}Lm&Fz`+wL7drVVk*hqf*E%gxw9YQpnbW2}L1IInA zhS`bB>i~#n_|kG~k}1wNF-^+z4?H4~R|(W&ty@vO+=Ui|Q~{)Z_4sH$EGO;Q=!#^i zPnr|2b{=&A?p$Wo=<+g#C|i<0in$&$i`B5{F|g2R0u_Ye8`5o6>hQ$s0R2nd?AdLu z`{noQ-cq7YD4p&Fluk}#(fpTFL+L~iYemdwK;lLbyX=MIc*Vje=5U&Np7!KT!7FTy zEbA%CfBampb+_jtg&|lL%nb_Q5r{efcRUEqeRjycxiq$=aO}vZ&ZRvLU}y11O8)LR z)1#a@Q$SzTZ%$TyrV$DeXIP*RIyTib-c3-<4C39-ayt|X&HP9pvBVV}FK4koZNA?d zoVH6Vr`jE##TenXMK$S@Yp$XHajM5yp`0i15DA}B_wx8HOq(}0h-TG2NJ#SJ-1FAU z($Ms%PN;GLE=o`785vmNu~-&ZX*5^68hW>_3ftdWhwcBk@OnRqYTu1cH(%)tqm0EK zj!CpTc3?u-_hYNu;tk#@Cxi1|2Y+7Wi259QpTY)zeEJWB%({k*Q-sZD>}B4VFPl}}1QvX^{kTrYnlvgc4gO{QG!PmxuApUHFSy{B;=s0q}hNz5T* zTU5g+W^l?L$d~qA{$2AWe7?R`-1NXhx4_cKp>Xn&VV+tv)vDS}Cq{-gRp&5ors19y z5*YL1{79!ft}m+EQ@L`Ov__keDOOZd`q!s+F-6rkX&_#e;v(Z%uCt*0DxegZ?#9@Q zOQR=rOST+2wO<8Lp5MMr5p(R9!)sLKB+JorAE0`Ua$FzNCZcP3Yp7ihoTjB5jfXiYL%ZEd1UlVMM8{mEsd( zf5Wo^fLtCV<&Iysx6E8qQq_gsmhVv3Dj zP`F?$87nR~zQeXw%RCS)iSQGMxA4yv-B{6l4T;MA4CC>$>F-Bl=;FKj+f;zLJjr`^ zNO_x+f1X`7ONO5C!-2(`na%;sg|IG zjv&JUGZb0Gj&@*!PEz{c2d4i+mo%Y%|GqQl>cEe^<*e89dZrVDj?j*wjUhzxAUV^& z{GwDM&>|n^j)!M=k=iYRE?yQA0F)R3I|e7!jl)lG=1wleLQ% zsPSe6;e(X)rYqPM^iI>$WbjrlDQ`J6*HO^D6QTK!ucS%vSDK#P9Z}v)fQ7G*54YqW zq{~=R{+Y_6wbgn`T6RkTUV zV~Qi>4fYc}WFp^29l#NDb070}X+-V#fQSHQbPs|NNdPOG9NwdX=_ekD7o18R_p|tJ z-L@Ty^;(`pPzTd~WNW9+3!ER&+e%M4F?Lz7r2}Zrb4^Dg4UL_bO5*kPI*d&i_K@$Db1BxFk2^}R#-NkSeGR+E>TbUh5I7)ODNlfIogr~Xa zYguLK4M(N`FqYz_%cgY&^3S(jtyT3s;QqOj-s0O+1#_2#diK-VwRia2Y`=+G=r=nWc|F-KN5L3RtWixIs0?s<3`3s=> z0`o2~C*x#(t%jm_*JOA&|L^}gAMOyg-gHiURUFrDq*56*8YHQHM_a5+C|n|cn4IeR zoL^dJU-fahU0kw*a!VO6ml46Ooh{|u1j;$G?7I+X5-t#zXO{=rLoT}8JTO&9 z_uG%oH1%lgSXGOui^WOOpAgPcTaD3)TR;xZSbT#aGbz{9?^fe4tIyXVX*x)5SE+wJ zj)kjXg)~&=qd5}iSonQW)HdTI1jkaT}D1%FXp6+5ew9?xIk-- zaQK*)W~=D(O1?n@AYWD;Jp-E9qUWYMjmPsVNvgMePB|wH3tuOV+CU}u2XiB|$WnGK z5SQ1DVwL_QDXjqnePpZ^`0!Yi-AZN~F0(t@HkiaDma5vwLOwYrJ)K|$>AE37GWah` ztPF*xf?bIps}zo^PyQ&6q&iivSwfwDKJc6$aU>OKS%=Z$al1Q(Dq-W^jn{IadkY@x z9mgCA__3x|i~rMZxS^?06T7ZqIie;)r5h8kl2l&4MQH!TJ4aKM@)gO*Q}}R5b~57h z{s`(18$&%x3B)Rbro6lfDY=UX=2>dldK^vA=9x;M(xO-5MZJ^I@2MU@3z8$YrZW!G zMfR11z_GF!_lI1Ih~`ljZIfkrWXO)Uh_OF6@62OxtKg^MD#8 z_*I2fe8To__ub(~OT{F+F^piSyiWVC(k7S?ZxSwP$j!SMZ_Jw0Bm$cDRijJr4)HOGRPMd*cM!G8b8g1 zvOS8vNk4j{*{(ONnJ=X0i|`<-HyJQ7d@Jh@H?c(1lNq;=mHbqfe6e4T zEeF=ERYYNN^zk@GyN>#rV?nkK2}cF#iq0-@rBB04joMz;n2_;%C1+A@-4zbtM~5w; zcx-+MTai$SjF(u`ll(tEm3ct4v+p43*A+|g=0$w`s^wt$>s6d%mE74yoZ;b==zj?* z|AFKE_x0UtwVx|K52SY#l2GfvBW0?vL4kuVTpfA&8Xi5tN(*;m-baUYjB${^>z%K= ze56d6`iFAl`FU|Yt)-wk55b$n3)(zwoFaEm1!&w#K|_%IYix%xpXIB0YDtQ^uW5G4 zv1iUNHVqJ(mCK3wMN;{uBJ-3c%10-CJhZvyfBIxTDrn;}iahm8rw$~nx)Rv?RR$|hi?PFWhp?*G`T61!KZBvfE*$SK9LZCkLlvslBW^A#5Lo9}m1N6bVR>felKbE&|5I zaw`2gpl0m>x^|5}hW#3Q=kCRNLx3bKQdtH*e}BDCJT0Q^dLEdUX-Zf}{!3&D@@2(x zX8;wq3wSxV?GMZ<2+n^M>E)iQxx3<6SdefN)IWF8H~IU&n*cvCu44N2p7blE2bW;H z>WAc~f^T9)2=8)J?Y+%l^5pOGBnDAl4f1etMoo1&!o%|+ zsRnREcB~bfz!ip150=nKS|m>z@2BbJT(b{=0cQS=D9~LAc-#p~Ul;ej@{){z9aF`5q z*R;TO^CPxoM8A_3jP=2%dL6qOaP@pWR=~k_?gN9M{NS^HJW(0`0qWdndrLTJu(Ug_h{c)<)9W1{XmYZ-jWN};UIIfkz6qnBNW-^Q`U?QL7=nxt zbhDW7B30os=mQ1k{~~3WKU_cZ-_O!rd#)p@Dngv^KiWywd{MDSM5 z{ubj)LGJVVk)NuZ;_6dx=>Zin5O@;}SnAJqDR4QTsA_43e>$>ssgcD0D=)An>vO?_ z$Ak7)758S}G7q5~rMye3+#-{6rSs(v$^k^70jFgOhrA-N80lI>3<&Kub_J&Q{Dn+3oitmK~?*%ePal$r#lS-Mr&dv6J z?3IjOS~l@0$S1tlp`aicRy#4K;9*)q?0YpLKEG)Fc>SgA!zamwphTXBiQeCFNCI4zFu&uNW<@3h?s{Rtcp_fh0_J$ zB&`!O2@7;&JXV?Bmrtbb5=bua*pfb^SnUdp%$+hObN6p|+h=B+q(=5!#D+5biD-|? zgAghJnuux^&4CN;tD@+6Nkl$c(e+@jw+(JJuTFHjp5gL!f}SNC`B6P-XEJl$sxAA}(&x}L-}`G4&@s^A zv1G4q?yy_1tMk)zxpP^^M)hvKzhnAa07DMus^CFqLyO+Dt37ew@lk&#_<%z0WtN3A zvW-v%%wc95hjty;hL=tb0z1|`Ao~xa=R9x_wb3d2+`mo#{dx7i{M-skyVg0wk3noE zwcg3f@wm8HG7#mAcyGKwRWM4}{$7ZX*l?9>cK;Tw`36S0Vz(iTinT%wcnK4 zb_b+{JOCv3%dL|$jWv0bqfEL9o{;~LxU!e~K&_~zm^HOuqsGJek-zl4Dk7!zIG!f6 zJFC*y*U_i>xL8(=N1>bigtinyn3JgyFg$$X`cnjJIJ?GFh2hXtqojZSk50Y_)Vbbk z9eu1Kup1+~|L)!^Qx~mD$;1;KHj-2r)(rT4r0Y?V;yW9Lf(ydEA-E^FqWXIUT`Svj zyQ|ZeK|Kr@OgB1PV}Q%&)2X=LL2NlqaT*$60VZxIAyc-B&zJUu3Z%l zf{Z@7?LjLUA_uo~TecvB!5{1w|AB_+SH?D);RRB^wss|+fuIL0p_g-(?N|J;$njbfA z-GacRyxib^Sj!8WFOE17pQmg|&8yin)iWbna-&bL48YD88!sy_%4l{mGiJ7L(mtD~ z1|=Rh(Z9X@`h(Gz|M#MoP553qNwRUo%q_&@_-wrfEgu58I)^!<&p|f}^wPh@rw0EK z|0s%$t7Kb$kYc6X&J|Ut|Ne5#)y3^I#?Y2qi^TjDp4B@ ztO#&CvSzyXSe&E#Y+7#0uG;JAqJ;YfVF59OApVxgEw#O`?WfQkQnP^UPRUM+g0XZCvXesZ5$KR8?@+DW|#$pK`yKukF21}MT1j&i2 z!7hKap|My6lh6G!r!&UW>ZU{uV_@U!8tl7?vsF0+%SuuAQ+AKTv-lE6Ym)&BSDb>6 zmv3+hmGIhfqqE=x9}t3L#p!;G^Tg0fVzzZD-%3^=GNI~Xc`&5P_=w+L2tWRHhe78`L>%gU*X|W-lCJ!WR3(^ist6tTV&;|Fek8 zeqQJ){b{PcIs#qmWo$p+?hL&s{o2h?#`%S^wks)+TE%UFej%2=Sdp7EQa+?QOmvOz z1p)7`Cc^uWakx}+yeCnpzv>s0`jPPnpD1BXhe%#R3*yweao=Qhc(TOK?f&1oTXN@r zMhTwYHmSdzMEpHgr-@$H>+W4OBEXo@R|!CmF)PG4uW)!JnhHjs_*49~7in%x-^C71 zPCs2jdqw^k=@|I|Hq+?WecdZtH^Ik)iPc`MBxHQ0*~gMWJY@;d5Hp$t%Ds$m zhI{?L`fXS5zePfbD)mQgj+t%=AQWBc`MvVDzpv`CECpIXd6HA*yVZDX_jg}FTPo-Z zBv6puf0?KEh`UQo#-3MoDt`JtCOvt-6Bad**>8NSag5rc)WLRcx9ZK2j>Uw_Q|8!7 zM&Q22GoTPsRu{F8)qK1-sTVvtPvPW4*(E9sX#Jda-UvWW!M}ZgLC<2|xIL9<-jnd? zDVnLH(eu*#D|e!EcW=b^-RlGa+rCFH+#w$=p3MXD&$YJ*qPxEe1^fCi=s3Ieb+bZE zH?vCyTOGPmnnoY4q=E{zo=t8&!nqCbg#^z2XP@1%5xn)_;IS);5hz2 zZ^;YQ0w-RH5d7^p9q>5P_CQ&Cu|R@IxR0sv8}yF%{i#bXgf_|PpFw%5xE&Ycp9KC~ z!qyr_ti7H`1nGoUw=-ilDc%y}Zh_v`$pPm>_2fBYZ&B}01vdm02=Yq3Nl!sl>wQT$ zHEAPoy`{_l#IKIZUkpXH!^8ak=~UotDXm){K&hiDP}|$U9tz(&(ggWHZ~r{SL6WPO zYEEI#k1iHWf}Uy```MYsC%pTN$r#Im;d^bi%ij)?wq4iMJbd^5t=#1&9KB`NMZJSK zS)l(iPijlK0i_2L2rF`*5eJI>9SfMMPYLz@cojP!b$ScdpC==9@W3p|`&;S0(H5P` zm$s?O*=&sk&E0;nkDiD3C@3Cw9c|DgnyR^krSR~d)$HW#vNeQdiq-2+n&y=>Et`O! zUG|u|4Kj16V3{A5_s<)SO31+>ALq@NUw`fdJR8s{T^0_eMd%v9T#UZHQt4r;H2hkP zc?_2)nGW%GG+UPhpGCe5s%B|2iYanyCz;(f7xV@jklM^YdoI@c6SQr`T0isbT8=*8Q!x=GA$0c$?tg+i381Oi(fCR^96!|t`?|Lpa&bkiy2MRDkB?5Lw+ z#%cD?=~V->4e{;RKbhaFhVWxRl2Guu>4|{Xl5r%QERxDa^eu%tUu3}e`D{z;3}hmn z(>OkLocFyNH(ZB>N zBbqhlqi)@@9uy`pIdzNw^uV-eS~8(aMEy|pj#%XoABZVBINL?5bG$Q<+Hc^}wXtrj z5eO@I6`Tr&f3NaUtCPlmyuP!q!gk#Ml(AO>`J{x4 z$tMB8Q^2a{F8dcfm`{L|87RiI*kx&4Ab3cy-Wa+q!VI2IT`&GfJnR3nUy-XU4< zqfMYv8u-uBqu~C<#pebiNs?243z|08Atz+EWGl20xczoGPcm&%0%I{FDa5>e7pqC_ z+9fHG+o>)}V19Scy0EjTYhrDF zIn5dX(Y=m$+U*wtzMii{B)Jn^N34u8zVb!i3$2-nhioJZ_TRS;-%j#wXiX~LdSFx- zwOtZzgB98CtT*5PLa-W!D+E>;wT!%OcW#GeZVmo+E!P2Bub)m3`tvgtzm>p9_usZy z63GV1vj#fB#KyoI2XB8{+oeu{n+>so+$aMPeP^4AZY5!l>284?`x zHEH=?J5m%8JYYD!K+P*Y7|qAc@7qC~qZW z+~6OxsEXe8$li~52ZUkM;Fb(#|Ige~*?9WTmhp|Jr2odLs#W$XZCF4hqjX>KXYjiI zTe??Itrk;<91Ls&X5wbgVfZgMvW(k&8CoVj+Y(K3*xctXPX*k`SKFwF}AP3 zhE<>x;Pn!K`1Tb49B)ozDd^rJJR+3DaWupgtkxzB;KEKm-<;zTeRN%L+wMKeUc+tD zds{?}?2h`BY81IC&j1Y{qE$kqGWaqs2g1bcKb>ja`eW1vyMSB0FQUEiHGZ1VepM1O zuUeN9k&bz77~L)~S>g)TQLk4q?|9zXvzq|%=P%w^XP+yf8^{2^K(&&Ccfn~*YSQMv zRL&gs-1A|}d7`oLVfgv&g8J-&(^0!k?DlYohZ(+ie(%~Wp)=q>_}Rq)BHizIDn%$` zCBOfVLu+lDmdGzeWZvb_&dWq$QY2&>tbsqE!u^{hyz88N6klIVXXA}FuV4HcJu4g; zz^5H!!1-dVOL*2ull(GXSp4$D1F$y>gTu&fci< z!Ta)vd*VHJSMit_2DW2wubhCdb zYj3v_Xih8$sh<>z!>RSPI|ahhHXk=j;sK$RLCi}7&bf6{-G5}|68Cp8hh%GpmGXmj zl(Q&G6|$fl*mQHs`_)y9LPMo+Yr@5NCEJFDEs4_;SY@Kf@X#B`mgqUrfUMH~fC-I& z8Tzyl)@lQyE8#%!<=JcbMblB;OSjHm)JhII_}MwXvirLB*}tHbY!;}$=BIft1GF*T zj8NN2GyMnBYL5UNn0Fgd^p$AaO{oddd5nIYZVa^bs=C7WE`98(pnBaaR^b;?fRj$r z)?YPp&dQxUHYwC|(pVznmfdH-Bfi&p6AkRxFJd{&!64Gx_WC%ym2{%hkt9MLp)LccRr!WjnY8GsUp&Qwl!~&A!1_yzBGaTuu{B3QHinll+w~M5DYX ze&+vnTuvLFi4&6NS!MHilIomi3p-nk6>!pW^T8|g*6h`;Anb&b(S+%%{H^;;&%boD zhp1gVMCqedF=PHKNsqF{ggRspN!hFn2eLVgPN}F2sre%TYZa<<)fFi6& zd8$q1Z0vT9p$$mxu&G7`PRkyUp7g^id5wIgHTuzsh%1{-QqD=;X<6ZtYMt5#AKrK> zmh0+e*tRZw`+k%@0I(+r_D7t)v`+J>niWbb=QM~^Xb{>EClm1-F#+kQsjo)C)XKKX z5s=I8q{FhESl@4Jf)h>TkoO_t6;bm`pYjrFDm4Wp4e1>{Rv-678twKJsD2q^|Bh46 z%n%{w^*5g?yg;Ohg8i>5?zMV&iLes@C|V4b1HJ@o1Js~OknSjM=xGlQ)f-3F-jEZ` zu|uE>C43aM=&?F}^j)aPqw;6WDJY>wTKb{JZ>;LCAa{on&e+`?#=o9^PKrk9MO%ld zLyh;tZq)~}JXni0t)n@*=bLt;WEY75sip?=5`y>xW;jZ&Bxlt!c%5(c5;E4E#NPns z5X3HPBuXC4B~>YK<3cD`rSCKMfpGZJwY?MK=Sw45Tc-4uH|1@t-(OWb)4czLPacwI z2tfb|QkxQ2OUY~3qJ?;Vva=L@{iGhJYxfIk-|KFl8RX-2Fn=Z^ug|tx=NB{x>E>U) zdO1FSIvx@}-$+~5+O0RtC#3bf8G)O0CkGyq+eJOimN3N)LBXL&T1;7$bHYXUT^HA1 zC-4vWsWOEdpOyglsg_SJRJAQ#X{5i*l%ZB{&xQ0fyu+eNJLPO4etXdipL3?1u)&uC zh+H@6_ELGBP`;Y9WZ^%LRpaQ$WAQSyaq71!`C93S?8Ban?k_jaCPLDTaMBm1PmZ?j z4$iDW4@9KyZT%0qGD={)vT_HsZX+{JpVEJ-HMr~4>gZQ71pRJGT;+42;L7gLI}C_x zLd}#UTL-E=Fu%;$UQ-{w1@$}tvaa4F@vd?J&PiwSeSb_o0kCJyG5GEZ5CK-vh_%Id zwt4=!FE=cD$=~g`JdSyMxuMW|b&t$q2vmZqSN0O@Q&NMY57>{X@oOkUlXCwwnps5; z>d2PzJqULzd9=mdqp&4b0}@S&AZaz-y;5vFA-K1%Ciczshtd>{3zgxXC*K+_O4CdbY*wcl-9ubw4EqP(65G)2 z#Bk=si*0J;#BN`r{DxrTo0=qq$9=CH$Wk*~j-;cVwa;%&!uGx)it1&*L+k3XsgD0p zSyO!URrDGwaPWI;s#!*d=36(F*)^FRwp-Rz3Ay=~wdDQYEX;y_^A?zFF>6PavuE7x z8tkj$;@%nm#qJkzh!GXpD6$coOe^F($kZd|+L#0pX7cGyW07aCmba-3r{x_^XqGRF znw>eKl#H2(^y^+(Zw+qUt5cOc64`VbnRX2-^?Ew#t;J%Oj?om^=_|gd6DVkDRT&gp zv*tEV%Z++`RK0RUZZ_VL+(?z$L#@QVc*ADn>f3`tzNkxCS;#4C>69 z1Grz$ugdUGl4Bd#A6!Vt|9v_GEKo$T_U_^)fpR*7w}*)AX@%NgTTtQZ$xxgj+9b6d zZttt#Fb}r@NK!8Bh8g95X7(t05r6vo#TvhF$2E`(!W)bSdhDC!nIoU$t2d7nGVY95 zOo|+7(7v^4jZ<9B^g8ba`%FvIpQO7yVK`7aWH(Njw<-Hd>L9nquWRpbT-G7FgucSi zI_kNK6d*w-&g@ZlTp>N*7ak5@zdhL)px!lNzMfMqEE;W2^TE#~dvTh|y~986d6ydb zxLL<~zV?TEiGJF2vb__-BTMtc90R}&BNe7a_NZDEYqeq`ZKto7__ z(EP@4CKB5EDg@qJZavxySp+-gVe;AqSF^MymbUtB-RJP{2KW9~itK!@t4IoC-17c;#o(WBueG@^!2J;0r0DxLGe6DV`!QO_j=TmTG7 zr9PNCwy$uO!107vARBaN4PRhB$ZGhwmeNM}A&GtjO6d^RFAaN`r>yj>kg$5+6>PmL zBO9ZW8P+@K^=v45s??4U5F8wjIWRSXD3$`Qe#3l6-&Qne>z-Rb4UiO1D|BWh0J(Cr zE#E#%RUoLMpX91-qsLRT(Q8qc`}p=BYXi^9N2)23Gj-o!gi_3>8=YXgUZRb$&v6Hu zDg;a_c0-eE8C6WRypsEVt_rRvX^_61#@hdMOZ^ZY5ds(TQ1TYtCC5jZM^)SShs(;v z?fd%)7TffbyWm=4y3so{L(ZlPC;FD}XOu?|Hl3BnAI@Zs37df{QxU?8y*Xy0XIzgL z)Kik%l(at)BSL+DI^yS@^B^Nf56qe#E$-{bxYJa-wFA3uUqzqgZC1CPWNHb2W3=R2 zL#@7hWl^+38#Z7VwY-SA&t2^`es>H9Q+myIDw?JnG`lxm8{kHI=8l}zD6@S{zpr(Y zalY$+{GM0>=umEYMw?p;xlh|n@J1s+F8Ln!CRslN7$GgE%Bb8E=W#k{(JorP-Ft6z z4h--GBug)zu@zopV_`G)WXbu4_YlpvIegu$kaP3L_E4VdKFSdE4iE%N0X&4`COBx| z`X4s?ipP%gyuCdRb#BsH9j5Vj^;{1ok6nLE&$gXV8HP%cc{6vr8G9`BlE?e$%y2P;tMQca@~YR4sK>+O!P=Y7Mrs{z@bTtz*-Kl^UPbmU(vVDq$Jpp=4g?hqDD=o}R zyEr8g?BMSdcV!#07H)p?ZZ$36aijqjk~%z!^};@y74JuiXa`gx7pi4_YscE_NwvEk z?I%)#mI%r;L+kEFCs_le&uHsSf`Q)xExPw~Vpk!xcIb)rM$W~S@a+?;+kKYqtyq)) z7Y=L)JvVwFna(m%q5d!Rll~PXTDIhb8OH&~0$)*`au0txFh<&MI&Pf|)L6C%gygLV zGnqc_1-Q)wW8W;K)?G!ho@L^=W@+ibi{^`}O2Y!CZoTttrV4Z*11upOGUN??ZxU*ZusIJqBoHe93N4xg6VXmOibSvt0c#B zPqe^2=x@{9ya;q%3BTNa&BJba>D2Pq*1EEsoJ#7frFvIP_v$kssFLu#xsXs_&A@lK zF){lDH*y9>{q7ZnLBrVg&Hj0c=zL8!<(GJ0fEK6Byl(NMjh|-A81GkQpRSV)tQHp- zV5ZiGmB8`4BE1V(_o;-1LF%)Gqr*Q%W-=K!%KpWYFqY{2#E?>Tkm?WZ0#GW`7xO=5 z01@%y#y?-ebZnt4#+fgSUHjwS%$<14u0Q?N78)KYRt*>42Inw?!bdBfo`G^5^I~~7 zFX^b9?DpzGbf*7GlUN7nJTPVd5D+|Pw#S@xGCTEy!WoCgl(9`c@T0XPn75O=K5~%h zVz}EVHNjJOXHdGe#uL0U2r8j6PQ_T3>xLF4mie{~7I#mlL3{vLO{2JQQ=|LM;mVVM z2l#E*)B;qr)m&p=-)z3z+yKEa@~5)h$JBaLpuzn$W=*fL@BJw0?cB#o?XYf)J8m$I z0oWNV(;3vl5(+<;M32?2)=5*TITTkSJ=MadQsK|$9crinA+2GDo$#oLYJ_+PE^$_C z!pmTBGdlvAq*BwG8(=w$!jsdlZuV?GdaTeeV?Q>3vwNg}QyT$n_rfP{9~?S!cD-zFN%x z4=jX(kZpiMTjcJHO&VSDX^Gal$Xwj6OE&qrb{!a0mt@l?2xW+qs=D2vaYrY^hyqcYDOX+j8IepzG;P(41lsC zCB#}`k%+@b@)2Wjap@4Phk7VU41e!*5pIFTLTjx2Vc|{C73qC$ztZ{Do~^1uXL0=` z|58*ao-l{xFymjAvr1MsT=iNEKw z_sG8?GbOqB(hl^t@vhhjQ|Nom2c1rGz3lHEia<@!`cU=1M3(0hqPzzylxg4())>*k z;9VuK?XK6Qf}7_5c1$K`HlOLlfoY%}+a3EW;`+VecyeFBgrSWWU?D%|c+jDPtUu)X zYi?q`W){Y?Z@=ztaKs~rvLRuo=$+~1Sb-u&)?`%=G@}8f7TgW=i8Td+_vT_935wf5 zALn!F08LbSAky!9)vx&1`?w48sl5-Qngysm^2jQ9gu4)2Lq;T<+W?|XmvmBNd3Wka zFy||F;ntP?6pq4z!Y5qtgA0&y)TA(HY_vS7j_b4lZ)Rkx z;%plj0S%ubbh7J%h0BlHu;nu*!ljUI{YVQ+uUgIL%Wiq2k3*_vA@>Vx*TDFLYq157?Nn32OYSB3cGOrNQzRybI&ED19C+8U)FW04r zrtSKTn(bM!7H)`zLEZTtOlXiS3qBghEg$^*Tq4+MgFw{|P}S4%DWkNN@mhlbI{rVc z1nuGFj)~!Hc!kCwlzZCW?ff2P}IzkQG$?uSWf}@F1F0dtF7U}kMt#Gi@U6t z@1s1F-2E(G*-Cx%Vu(o!{PtNYDvAXrMgTs-tVidu(^MMkRR5AHUE%IjQ7AsGlbaLT)UfX%S zjcJ}*`=)?BY4N6KOAR)otW^7(KU;9R<)9d6S+QB30(p3f=W)xzyvs=cZyz9h73%?J zH=_|v<3z=}MtPPa^!3!u#&~&P~W>~_VC=?T;Lnp|kAnB-LFUVHek57fIBgxrTnHn@{`KQaj^oaw^xLVLK6h~sAtU)l$-iPfWjD{ycU&{V#d)jE{m6r` zO5s-D!l>=uG%T<(ezox5N1ZhVPOvlrdzd~9*4-kpl5|OIe4PgjR~gJW9D_b zuP&t;aRdY5o%tu?*7_iYPg$vr)W^6`Kk+!IEc)d%8vz*g#VA7v;y=L_HSl{_ulQOQ zAzZ~lwf*q;hm)B)5^J3vPzI{=1hJlOJ2SXC(>-jG4yhNCN3>KZ{l55E>w$1+hr9#}&*SBhI87&?eS@3c_c zY4*c2*pd04WCY$ckYyWJGng0A;)|Co>_I{zE)Y#-rAvW%eRRELdnF2&I7iO3da3Q+ zqIjkAicoBjU9ok@TO(d=w`*p6(VAD5Yjvd^3=DM9@7&zh@5!Jmpx;WYX3vQyOf4$b z^d4{_GiyCfx{H6#$Q(?&Uu`3_Zzfe>3I`C2-UaH>BcZj*{e}5k_m{jy8rtF^L`s9o z;4osi+Mzctgwo_Ien|S|8~w|8<-M=IP`I?xFBrI*`8m$+h4*fH59y3j1@V6j^}Z@Q z?NL$Cz6-4E&@olbZ%hEO<&S*fCfsZ(I+11}p3PJ#nmZWogrDFZ=>&s>j~h27x(hG41X4tf^-{ zuE@Q7H<{+AL`}7#j6{bFD-0yym5HMA_zi~4?FbpZk+yTR+=>fblO3ztO8sS=oTFcDcTuwu>gMR0pNRLXsrSG?CCb05jFlals2_* zm&%yz1M@uR8{P=@@k}+kMU&QdBm872D8MV%sIeAbF*vj+)!uaxCPD|_p+zd z+oS^RXz9>K8gaU<=7%KE-%rkx$l*gVOCvu5o-IWac}&)y7BE|VII|?R&oi7iuYUMv z=0o?}u;|I$Luc-L*D%2Mn^Rbk6RqEYp>shjVuj2+K+)aJB%9u`)LKPkcRm^!>2BrV zvIxF2FPV)}0bU*q>{P|QnR)-24$D|r4QyKad_Ciil9w{8k5U7Aod>axA5NND#hT;d z3%|OoXLdX35;88(+X6)z`YyJUoX{2Zre+1**=+|^=`9;sgx$=}Ws7?TUu_AQ7XFN> z(TnZ*Q!(R2)6-@&shW8yj4md+9DF|SO)sRMHA6ai8TM396ZR$&61^n)C}pJgO+(s? zV@ie2ZSATUjETtxL8l)2Ca~JNg>sQ@hLTs)HR`w7Z$Q4fGx%lO>SiaxAEU@hg=*oj z2;(o}-XTRBliF)NRz+BW`K`LQh-%&XXLH+awtKSfHV0o+a-xUcC|HA zOK3Vu-Rp^w!bEs{)i3K$@zY2H6Hl1OTW#d7N|v|cF;|K=ir>K{$pH`rF+KV@`JCT~ z5|vJ78FWTfC`C`T+}ABJ(5wu&6D6>B{KTZJUoNwfFAMMO=D9QY&$#k7E2gY>QH2A* zFbdTJ{M~*X#En1<31&iUe=Gh?{ zpF&j6`_l#(BmWh0>pf54)poc=LC>i(zE%zu8-ZmFHp02RCPPp?q5|&UUBP?fQ?0_#f8b z(PgIi!;$|L-l`(ssyDe=!uU*JQ1b86zna^B)VdS9_cQ9sW}OJ?U5p>m-ky4Xbg<*z z*KqYApbdFA+j0DC#{A8%ZWabMugJ^tv8&bj#9w;H$IEhjRd=2Lb7va_`Ff%>ZQpJ0hf4$jMzW7+hCd5tKMe!r1BREw`me z9C{zAUsO?ZQ^>9E~6ChNWn)8HISh;6lJrS65HQFyQV1!0r5rI}_Y;thFv^ z6Dh?HrcI#mdS&Pvd5|VhNzwRC!_JbB>d(9fLU(eaLxrn@RCHW{;x$ETr3fWShhom@ zhkJ$K1Ex!#`Q(_VnJk)q>jRPX4Oz4>A%_J93a7vuva{|k*a!_(qy;%!NLGA6FycLTf7FogD-80h^Kd`$h}`vlxOAn zP}UFUtpu2nm7M4VCH1s;6=jkHQb`RHsxpLvHJcR8Nh7TYNmPGo z#Eg1O<#Q8>b-AnK<;$P>v^EAF78aZGQgs#=x4cIq(Cu@wZ2$|Zb?Ccj_)Ot;7+0Z6eE-k`lU%LxoCrQH{n{SyMUm`?fzR(97&~OwfTa5l zsQ*M7sIYgaLvmxrf@=zO(#`Yn0veNqUc|)ACkmQyD(B#vxOH-6$>QpraF|C0ff!L z8K6Ff!tPhbkBhzNDWE@8$k+D^+-wJoV~ipb=l#Sv#W~C=otg4vevAUzLduwFyD{$} z@C!+DlHHk)ba9n6=L>_UFk$FCT@8I_uqq-}W=>|+v{$?oeyb%Ms80kF?6K->so&v`eBUioQK3YV zzgL0XNpqJU`VJX+{gF@iFJG$ot>qwj(GoIi6}>@D@;xZ8T*w^i3)wC1cRYwH^ivl( zt-B9@xs?B2etF3G)W8+#+lJPpgO!ti)f7%|Jf9dpn*w!d*iX+6UT`uEm$M_x7m)0%nezn!B5gk@nK zdp;GF(r**AMENV*I#FM}a`z?xr6VpF2)KumoxqHr`4-Ms3F-qR_(AXc99IMt0lQIg zelk$jS^(!#a`guA6-ms=?z9GH2k66}*wDR+lWV=Ox&(mKI`^$OWX73Rb^}xCfYl}y z)Uj_LVV~lKi=&Q6w*UfaPxs-7qfwv;+NC7F#4(i*{>Lnpa;a6mz&f^)h2F(|g?V5t zX~lHP`L>Br;xItFRFIWaoT4|2(>f9wg~b#OC5Jf+syb)Kl}})+OS>CAl^T?N8<4#p zMu$)z*Leyc`zz4%$lkouL8Yb=7F{U?TY(RZ78f#aPIW)`Y4@cFeT{2hf1n2}kZR%A za(Onc$r^piX_JWc(;zp-8PycOn{2but^vC=rD3e~WYqYhoGU7I2J4Y@;lhWguHBLz z-BzuB=jF%tJ(J!lZ(OZ-=_rBL?85eCCnfhipRrvn9S9{}dqZF1(8X#h8n5coj%=$_ zeJ^Yuiw}Ag%(#5Y+YsC1M|T>QUbz+ipHlAsrcFGl;S*}ya#t&U+-cW3Ub6NT2TgTy zUS3=1?-c6n<5s{JVpfHvjDg=9Fk<5%_Xle?I~TU)b8HXSfyRIbeiOG_wNL0|eBkTK zDMg#2T=`UULulZ96ub&l{maNMisb&t6G3zkQ)RZ}J;s&#@KYJJ?8fbA+pxjZb_cel zeqqXNt0k=Ra~w-sIm0S|Zq_i?<;4xO2eW=WDQPRcdM$+8_*jp^pVUwa3KPkFsn}wK zsf?^yEYGD)srup4i%Fe7e-&xJH+XV^Ft4`N*D>5O-R)cuHV?atLf9#V{OkW%@BK?c{^VWQS5-3&#MOZT<>jSe^bd) z!QOLOtYbWXBW+DJJ~;KTD`A-cc1YXm)~!x;5nA$Y-A!HSC0fqPYyqqC`-^v5woVUx z%&J6amp*!Gu*e|r*^0F*(_OR?lX6-m^bvWdgqz;H2H-sc%AAPbJ5=W}H*sb7#Q4er z45AjC%(4|9Hc)+Txc~HDH#}HvH*BNWq@qK36Ck;2^gt{8p_5vxypEU~4LLQw_jNdK z3{a4BRqq>FTIqxInl{&0Z@hCH&oD=#bPNAG1|EORdl@x?u5B-K2`K2r7 z5b4h8jcsQ?6s&A}RpV2`qVp@gK!kSTbZa8TAam*EOC6Er32|lLv35Y+++7o)thH)* zl6NUE7OtiCEY&x?PIsOC3%Q`MYOye< zuY+qMz?ig>52|8Zr!Hj~&l8d*Bg5Qf$(nwr|5nT?K7hi*V{P2zq2Tn;0YUNZgj)K> zR1b}h&ehYw_VQ0^FYY4B^VjJw%rLeqnV~3NQlAy)5-1PgR00oTSQ5bgfXIRL!9qt% z^ttWmrR3)WZKg>Ak-=@Q*VSRkl`O?Q^3w%qAE^mm)QV~l$_1TSv)Vm17oG)t3d*@1 zi+U73Y+kG3)df1)AM#)|!DlaY{y z0Xw)dMsi$NETz-zra$BRbA=S0fJ4qw*$Tr#L85~}oQ8ZxFqP?80=4OT<>>>UbCbW* zY+^;*y;!dfClzkAiNZ=QCeWx=vlIS`R-bID1g3nAhH3F4$i~g2s^Y-tkkZU?X~qe* z{b{}XVE2<*{}Ja{WoCvi!*4wH@V9NC_Yb+kM?|0z6ABcLN=GS6FUBI#762)%2-mfDtXMs6#6NjG>GWKyQ~A2}W2pvE)3;Bd8S0)*4!y_39U=i9^uqF2o-CI( zvssTzD%X^Vn13;X-x?#=uneOx0Efxmo~||seYHS&V811NNqq&_;QVoy%Y*!;oIiUX z1q;TaEvO|wkR;_37G53Wd3M3y;>_^UYyG=Cyu*A?G?N~ znY83?Nj1|meu!=(&2{cOWkWsfK3{VE{Is&Si>_rQ&U`C&S_o{{dK0=`IW%=st5?;8 zS9(8QK(t~zTAyD1AAJA+l#Nk5s2`6`u?XQ&yUVXoiyMq?kFRaDikX0g#B+G6fw@byQiGis=sNU=L)RkV5D zinHyKz2(-Eo&BzGlew`Li&rgmIIN?O`!SfE~ znN>VAm-Z#vbmzL2&&4*B^qKQG*e9=H!q!iZ6ZsY?RW?2qjtI>uaDD!BM-A_$iRSn$ z*^uRqR9m-RMh`3FnXFo|Dvx)tMO;v0q~dyO7R2dQZ5Al;1PR?XK^9yOUiZOvz(N~3 z_g2UgDTn=bwR_bRNo{_*r`DOrbmM>Vs_c7cNqV~CsmEodC6ymPTM<5URv1u9NTKeo^nD1kSL z=A(n*tSu4=eD@;UN*T>Vd-0d;;|p{^*XLb^!e;UWrf}{|#7zH-OW&+e_+`ik zUCNL6=-}ekdkE?aWvV?Z8_f9bIMgFol6Za=YKah@%q^r{*+CE+bo7>;#Q9R#os#^K zGY%IDO>sSZql3TAM6w`MVWz4NMg&vK?c@P_EaEE#Qt;-Jj9bwMiGvcjKjVqc%q@SH zWj_kE-p?lU+D_&|7HU+%p6iuV6E|veG?cfJt~b+MHakXDw#>N6VW zl{0CnPAL}KYP;#9W^IH}A>fVqhg(&!Ag|78V)v|SP`xv9Mr+Si@f{v7Af}~Gz(0!E zdb09=A_6=qR%*3l=ZhwYc(~RGhlzh;Utuz5SLFdPb?{`|(MOw*)a-;I0C9~(nNm~& zNfl^ObfYWTX+r2wtf;r27EqNZ9PFkV_7ru+6@>l98Dp~d^mZP5QRi*ZMI2rLbi{=6 z)IW}n8|Z$cJ*<#%ArOt2CgAB|&rtFJNdUjVo7R=6Dw3h}L+|Hmj7)}`V1rtA4H4ay z_H5IF`eJa;wUu=qJrved(huDV;s`gnx@N7!DgJOv-_@sAQtf%FBjvJkX(u+C7nRxG z5v8+cId#wWpx!=RfxIlAxrlalaYb;c*_3s1xF#l>?|oxSbVh=f%WVddSFAGER0@*h zS=S$U=Mwpl5rnM@P;z-+3&wH0>YLKxvbry%aL(ItHnkK^r7J8Gn(hX>Qbw%lE3P8V zO89gF{*EBNo{H2dw&M2X`cSrTSqYIkP|4L}>%*lgG$#}I3*sSS-Fs(oH$N(ba${_L zx3$(+lc!NEYNTQG=brY_!`{IEY<;&)CZuh5f4>|<@iIirvKBRIM*qBUvhP5^v43_` zKE|C^$(>E!V4IJFHR-xx?)_Wy#s2`fBPJP#k7w7W8aA3wfcG{qYUp^-QGSQz7Rw|+ z4BZCR=Dse1el(B{l-y@MWV_?X`QUN0yIuMK%QcRD9_*sjmun89C^j*MNmm<{3{R+< z5~KZqtTtPDavmd=46v$u(~p0#Z??7j{08O;5<&;mb8WMAJmr#nVh;V1Z}Zf3Xc)dI z<>M=?HMXmu;|dW_uU4@;6M|MY-7}Rg|Duf2gcX6d7R9AX$tM#=2D|;!a6&1CGwHgi z=`MNPN5CG@yzaS=YRB$eN|^ho3WD!m;aYosKEDA5J3uU%zBRI%+|GakF& zRam-((G`zSG-jUw?cYE94PXjXv&XstuB>8^{SyfTAazBDz%*k1MZ0chu7^3%oT z(*B(LpogIAra3IBKW#Ha4=>(P4A#}PJYZPJPWB7lx#hn--<)1*uLTYzJ!^|FC=Siy zm7y`|hAW8+3b^9yTJ{**$|PSmAwK$kL;0t;4fyYOUK($s++e#!wvVMOs8eyz=is;1 z)xyJAC?De|-#eAGD|xXz$h}b0S|)_|NHy>B^YzaPp%2g(elazW{Xij8f#wK|4ls9G zAZV{w*#aa0p1;uFV~BT4PHu)WqIO1a2UTbXD*DhqH)6`$iSdXeLg6sbO=0YPCgN%V zwB$|y-pZnaKm1#C==eJA7PxSvZ(GMGzEC0CJN>nFb*Sk$RYxz?-U9gh=j5v8fm+X3o8B+V}?a`rLAgwkb+Rt6K>d4UXR?`V( zRLmjF)1Kqhno%7JBE8R(!TkIq4XnnjKPBt8ZAQDL4R`>%%+a#>9I8`!+PGO27Mds5 z+m^0F(wu0Hbxt;~)3gUOxUp+GTPY4D7D~U>kZ6PT2tqI_h~d~`TRr|fvIhNbz=h(H z&6id$<4%ez+#ChE_4-e0;4TDwV;$~{t5#EO)~W$pj}iua(QDCHmd7${B`D(kH@Di! z`-mndd;~j!>;%AP&VI6S;a6=0jQ8^=4mT05Jn5P*92CJ|kX zzdBy-Y3rvUMy!1#HhIFK-@YBTU*4COMSX&=ZW(P0rP#IFVx517QlAlVo?Wppq3RM- z#`e5+73u4QBlqv-l`1jfqqoUJ#QnNC?L&@&vHhLUgC%bKhl!eJaWDV#sxe6Rd=%Dy zXj!9W*u6F+=&IiawDw=6Mer5&;DZI*qmIzNlW&83M4g7d4L4j<5?jdNtO?0q-y`3mj)1cx!}B3LS~gJW-CumBgbCiYg!gMS!-DLYPc;Jc7Hd3@N%$?U0{F~YC2h8VDe)`YO1SK+mEawx3V z5}{{kAHKryu5E?H$2t^Z(laE4*0JTob?*Btfne)Sr5KZ~l(k>wVYywhYdIzcspt|O zT}s`qWUiJURgeMfXysn;n~XHtVDCvLPEKd_B7U>K0BRr%)^aak@~d>Pc`hZaCkXwy zBWc{4F5}I@@BEqRpX0pYFQSF^>i>yl*vilW>kx3?zcSj`<=H~%WOXqHI~PRecY1n; zQ>q1~^v;D!7{h+pfOFG11QOLTVgY**CTfS({^R8VduB%syUdpytN&^>|7|1L1t?qP zz>hB-U%jM%XA9L2+!^(Ew_(UTgTSncY&oWNAA?>`-DEc11mjZ&!?cT}gwAxF@L*B| zmFC5mz&zQFKPJLx7vv0>2SL?L`fSsu?u5(GT;0h0W)qh@PS`rNsueb0uZ%C-K$kfC zIW}KaWA3`UJXv5-2v#`AT{AK+hp*e0ZHU!Q&S+rDnVEq|QSL_W&5{?oZ)PiZ>K(3j zT{GHE#aqq0Yi^g|z46?tXXYIaSxi#0Y)8Mb>Fr4>EZ1s9A8>qb66P_6WSbQ~zId zk|e1M+_7ieCc%wcmxdM>kgJb=NHG+B92&&5zdTT|Js%)6e}Y2}{N<1EfRwZYx*X)2 zk1^=YCcv2l@-iBe0Hm)Z?S4Du1Mn#=CN}1Wz^$IRNk4#oYiP7~Mlzq{-9_fonQKhVM?IuO7z|m}xwcFVQLTQZy(kvL#t z(Fd~(>4rKpX`jbWcs>&t(e4U1N#^jrlW8&TRe*c*v}QnZyvL7V zghDVMUOOGMMY378$@rayhpdTN7A|F!(RR$z@^8(iKe#?L@FZm@yEhi}Dx=!UjPH14 zyVZ{GSnN&3b+wI>$^zG1f;_VM0yPY0t5*?^bVo?9(|8< z&UbzPdWFkp@4faa_qx};kDef=_y}xn-v72dO=gybX42xRCs+95x&6I7Tdvx@il{65 zY>q+ZMXNdQ!_iI2o;IJ*Z1R)8_(PbWB%%gJ|P*OVsD7C(MSXh}uYI zv)e5+yO!29tcf836zW?suWpd6Fq{||2kWL%x8qhcJ$wKkLnXwWRmtXtZ&&E+C-8^n z#rgX&Zc=$jd7W@MEM_57+Nku8cXEyfoIN=VlFl==R*m@Jd~)`~Jx5GRO1YfvjCZB_ zx67Kwp;H=KD^e{v>EjR5CiEc>1pLKKgVz`CL9>HK;FDDr`c)UR1xKw_flZyqtb_-t zO|#|emo!SBk}SC=0A;XG#Y#kM1}VpBrInNG+^#L>d`Tp5u@|(Wd3uAA`S93muxvCS z2hZUm?R=8?#NX*2{#&^tiYf_gqRKf{5774F zIr)9GY_2NOc<{m|%`tkGBf%L%%0Di0=Ff03c$U%Px=n4-KB|(z#ueo?R*06aW>}K6 zRbIjdhcA*$#j9WBoW9<;v8bT812x``*j<8VT+QN!J_1y&;CZ)vjE`@;Q(xrvkJDZ0 zJjyYluHRM|&TLl0fUJ9|tz$kgAk;AN5J@z?Eu|q|09$`%J})M5{nQmh<5n_A6)#(^ zbfrUf(ZVktsbZR*V2vX>>w2sxWBy2^dag`~(Y(3$J)|p}mv^JNauJZCC{H3lx<3ql z;t02XGMio(NA`-V<%;izJtEy~sj=yC#1J{ zmBrjZW?W(4(N-@@YvQT(5)DBN_t z0IH{?p>3z&ILmg=o(v)@A35peT#^tJBekhji|BlM1&ypM67%P8&Usi{wCAguK}&M9 z*|W^rG74R81~A{SEa|n0E^yy)ioW8Y6JsTgtQWU?-N?CSbP4tKl{P(p|$_P z)DrQ`A2oG?Yd-0CZ6_~TFMl8ayF2(Vk^74gmDP9W=Oz_R3@y|WG!B!&OO5}D&GCOy zA1s{R?AQ9Ry0caZkZwZHlmPA+N6<7o)!)!SrribeId4FpnQ}(ih__vh6Mq!3tshs` zfoj&|vC9xY+1AxT($YtjC4{88@zSt2QW(;$zh{Wwm%22hJiJ~`)ss9!Z3j9Ix&>FOv4C`Q2cs&|7_-5gR zK7@4LcLR9xK$!YsqBe}z5fT4{ZW2nnCnC3j! zF+z7ffh^l>M<|r?mF_t!JfV(XI?*lgcqvgvG5{C++j#4{%a#c1zOh2EuO)K zymt*=m3KSTq7$>wBNkiK!SaZ5WT>?ew5W{t@4c;bqx2tdVAj7}CfGZM9MY*BMYg)F z24+mB#1a+a%rKiDI!w-9z_)mKXzEp~B4O%Z0{^Gs<&1)=Go0RyV5MQFAGrEMVXiju z!AuHNfxnRF4>pR10LKyy!P?#K6P2UAvinRj)q`qSc8s|tfP5r=p)thyRwSC|DOXe7 zkTC4{yqzMsOkHH2r8bjErDvZo+$RUwyZ9}wK|>$Yj{4F*TZo&C6A0W`kcK86%?kMF zYw(W3@>j9&hIuNR=!dIiT@U#-4O^tc^Nm_cw@{1Y>Su?ePqj|EGE^Z~7g=0|wt9t2=xF;H0p?~#K4x#~>HR+|wf#DM@Nc(1+? z0X#8v@_xOyWBmua^JnT5mdPG;0lIU*Bd6XKn4_N*p5y)#oBB~87>WQ{k`~g3;{7v; z752Gg$it|~bwpnPu}x5k++d8LP~_vAkDaf4q6Nkx;#RyOrOcP#>AuvB0TEb~2z#{* zl~8w@i}WtklAuR-9Qx8*2Y4diB%TMw@6AHxEr$~mQrUl&t0q}Q*k z`#DE2u(ywLyJ&p4@{kyOqOkkn0IF}#^rdR_DNe|ria~iND^n`nZ2SRgiEPz^f*P+r z9p*i7V^$ip)k6>0RIjL)Vh5Z4$w0?@@;=L~wbuM+jIi+ACqGuk&z_8(omeY10DW2_ zZ?ytpXM`Kg81c6Kq#VPO&|mAqAJ21ghXKqK@{XZ{A#)3kqunD;3AB~9H;1_bJ^9ULQr$Lm4MU)~>QwH%E4FFncia6}t23yH zood5nhRwhfnn+rDO@s-UKxn3+enaP9QyClBG%ndaeFnz?7*hy!*qVuBG=$!5Z2QKy zl4Dc=$Mf%ko0s?*!|0*ZU35iVX4TuYtJHyG-S;o`ya=m1=RMcpLPpAC-%~a^rz533fFQ@(1^|QI$5Cr&E^~9KaIh$gE*yzO0 zl1cbD5`Ei?(Bisv!%~qhuj+~)Z6K)EP0#4-Z}DoYvEuRIFj4`UHmm4p6^x|Uo3CH6 zqLg85=@VWfS)8QRzw7JQ3 z^~kZmdZn);f(d~PmtRpIFm?Qe#{yp6j9Oi}l4g>GjbGgOW4Jq_%5zine>c9G3BZ#) z_SOe~7H?mQTdss}MztoKj5ccND(v^6A^^V8`KCCrfpXdm(eQ~_jpxZTc!QD(-OF38(!Q}gb ztfu_330H%hF%CEL0EaXNQ%2_;$EyPh!GX}HdCe_<@hg*D_hFW+w2eH@>y+zk5v!|s z#aPIcUNdb-ou@#NVtfr_98YaOs6`ALXkfGUQBSHEBi8fGEq;GS{b(Zxd&J|E;-pdR5N{0P(W;YtpbJ>KMgj}@$BY~4*=vVx=LnDNb^U9UiOiQtZy z(@tT3U5LeYk>X)*wo|*BEJ$`<@?05Nv7L1o!?Prc>8iPQ zzB+G1KKdc%Qp*~@)qBcvM@O7giOZ#nkvSM9ehV)CP5lK}H2#Qrb$?*R3o74jb0D63 zVeH>f4Ge@_P}nhG;O9uV%F1|w=7M?aRkey}wsblddWnl=#?P(+9Xt(?nyG~2iTmcY z4s%%MSTwZ@#}@`V+BZtH=0ow`l5f{;ZAR-?a~;lqz>%qR(I`c3-u3DEznO<%P= z6~wAzv1d2T=)!4?3*SQaY%!<-K7%J#CajlEikbt8Qq(*S=_sc;)Eth5=={a0dZoNe z3QuBWtQVC=w*%rHdaS|6N7YvTwmog=Tlh135Pr_{;x>blY5HSkjF=k)+Z1WK`wTT5 z^L|45j&7jHm?JafDyc8k8Y)7u$F*Hrnve}}YYTIQ^Z3t>m3InqMdO>e=c?SxA^sk# zb0>db(o{xXS|BoM^kS`LV<9s-@OG|mzJblp1g=3omMFMl+*jw_R(n8=CZlSzGV5?I z|GJyRVfrw^72K4`{IUPPUYsOE1ubRGhNCsU{DO|~9GihTKU;mgAyXAV?@m4Qhu*JC z2nPJ~&-6jm{I6`)+{ovweyi6aZ=PQ`?BP54-g^+f=FNM~dBK>3PNzv$CMT2J>B9#UB9yOwA9s)3Frd)rMKuVCji{mZMqriah(CD zUEFK^nfMdhy*4=?gRqSmjarKvB2-yK<9C1{;hm5E$AT(nwA_jeLy1QAu{oYq&)L$K zZ|Twk3g{O8aS@U_ryVet6|)JZl6G%oQU*cl3XFF7c<2xrvTdJ*8>ay~>c!U5c*b?Y zZS3FbDJMj4j8ugI@c2acn4kPPZ*0bwm`1t%x2*w-S+RGVDby2V(IP$*d~xL8!T*?A zzxOUKnK07dMjYQ1+22Cx(NyHP-sasyUZJFQscf0wJg?mk+_Umrt7n)i*=io{tVKk$ zJld1rdFJDa6Fd6DnQK(Bek^?Bbu(n=QuE=v#yU%xRnLD zRt)pfkb;#!e}~=;p#GZ%zj^W4LX-E5=5#471ua~Vymfd5O2@*~Q(4}_PVq>h4MoYu zkI~m{jUR5)ZW%)btEJesVqn-&!`gNdb&H^g_XiU}=r@Fwv}>r}Um%Npt0dLlS7*MIe`_ zC-vkgDM@lHVguacYd6f#o^NlT3BJO%RARvKk zxu($_gznWgg(BU3?Ch$c&+UF6_l%@H{d@-9mPUg1Qn^H z0QP-xpFo@ni$5-T6e_A4vnsAJ$Q}Am5O(>f$!%|GaFAH-)}>HA=-4-<&=fUYw6o>p zar|wHdzF{A%fxDdebZU5)tLIK>1h%mAiRd;2gPcZp|2=7H7wm&Lvp8`YPfaCUx9vZ zm~+UTJ4+~9tz!DdqZJo;mQ;Uo)KndrqZn}6#ceOsb2p6=g5&{w;8Cwz#BRJEd9Qh; zKZ<)d!doL8HXbG?SW`#5EW4&z3{qC>Ku3JVmZFbk|H>MlT&nP=YnwcvJqa=9UlN07}w9*jENePsdiouxf5tEMp;?Y>? z=B0WKrBZ7^zVgC=-({8|!OeaEG=V~dHLk%m__812~Zc71W(>Kt{m1=24* zj=Rx3IaOdxi3bJYd;5Ie>r2dXU ztutl9935+-a(x>gW$caQ-rS^~fP>p|zFrL9C#g-_oSLyN5lmOJnUwQ_Cf^#fllEG> z39)B+aOaG|M9NW(lxv)0U0TU3TAgRldty~(q+fr$fDf)!_Rfv1EN$EiVe)yL#T39r z1V|e$bb3bW)mLyf9R}BDXQgF1Jy%kRbLA+>z}Q!NN=RAsMDar@H4QPv0=?naYb-sz zNB@5!ERb#c-u?WK*R_z6UVDklO`r!M=Yk-Bn*=v*T_^rTF^x>@Ue2f_+MhDQv_8;$ z*YKIx=VRn>p-zWB?91dR2I$5&476JcG5XNdG8y@Pc#Ah0InxhJT*)}|gP`2iGoPdr z1kW_p{pYLRe+~nl8h<0Y6y12Th-P~q)2o-%tfe!Gn{gw$@EvxJDsZsQ#`kq#3Wpi* zcdst~mbcX{XovjM+n$;MRtYui8>gA5_i`WpNW`fZAI}o3a{#f zyD)FHefm*ce_m19+hOf<$Ycl7p8s9&{cbA2gy<9Oqi zd$Y;)HlD-DziA;C8#;RETEEb1Yk~U~#<;>XkQ~hPCaw07Hjy+tpZZhp+n< zr869SAozs76<5SUi%PLroOIyEpluG-2#w9M0DR(Osl#mS*)igS5(A;m&rYWB5>Mq& z=FxCKi!1K2%HVFMi9JKy4bq1RX)kLa5agRlANXP5nckY%-k&3 z_#$l)sv2YuP+3eXZ&@xx&|68*xMLtDcW1G z`nGf|W+=A*BJZGETZ=mXg{6*M{eUoqCfb)yarxqAmpCzzw)Xn(ieSUsHA53By!jO= z&1Nyk`0PFp@TDN<^Gr{fM`~|NSGr$siEMt06Wh4Vsi(sg%e^Q)!RNd_O;X3Kpd>_k zINYo487m&>I}AO0IX#4RfJhVLth=rC^gl)*1#82%isRG64+61NSs}gJ5gyB#b$KO0 z6_FMvqvUZ#3Fj*0K7fcUf{aM3c{OhR;>vY>DI~w`&g%8oM}Bi>K^xNt&&4Kw%%S{} z@iC%5nd}dQ@6``_pCALhOe2$Qngnm-_~^rLoOfl~B{XqxzeCks?zy&v72u9zCU3(reB>vR&?mma=axSV71-Nt+kt>@~?5HfoCuU^Ni^D*Lj&t z7txCjo-Mr4t25e_&6WOQwFE}BPE8e9)B?iVAx6V^b&lgvv^DPAV#{CklMea<{2;q} zIE54leJ?pA#R$vxIHDh>Z@aW>LZMstyN6x~tG4eXJ>aQluvWC)8IlJ-Ypx#oq(ECi z&zTRx#GjR#l*C#P(Xu9PDX1m=?vY&J^_LJ?Im)06KFZ;wq}aNfa}}5Dn>&a8K-I-{ z-WTYX*zDUUgRA*7?yak{?H&qqYonxl;NkgU)hRZD94ZeeN>E;2JZ?H(DczZDK0}2a z<=%0|6J*wZf#g2`J}>so#(50hCQJRGGIP(LEsSvjaJO(b`pM>#*MA*I5G_x%+QGN3 z)68!u+SS;!%@|or@wsjTebj>O>YbnPVQwzX*)!IeuY6%Ed?6f|WzX>p9AVPZEJD+uyOQ4!N&S)BMdjQtfM}mn>q1&}B!R0<|}% z_k2mDIeWA7n3v!ZQzvC&{T}&`H`@ePHTAB`vdSsoC`yjr3~puZwmhe9j?v75YR=r2 zqb{J4MiVdyw#&j_swPY=^TxOxPTqR==}MXiXrSAE7YDz`7F-*yeR5u=t%bL^k4w!t zGIvLfGJJlL-4UO=We83z-k;tqaeVL(Y54SDg>NL_@{m;EN^Gi)n@_C3w7)3uzuzvR z3MefqQH2;qIA@mgTbB1)Idg=UVr;>>eU);ri_Wfc%xp~TafhOG1G)a#e4xrZNj>MD+v41-f8P_f-=&Q<89ZY=U%I}y;s|UG zoy)i?qC%K4k$S&uKtJyj;17x}9?niO6)BoLD=O15!9HjEeBqE55W-%wXi3-5!+Q+# znO)O}HR-&yNR|VrHSDyWYkAcOHdg((J;Ri(j7bQ~;UBNd;v6v&;+u)s%(EV<*DgGS z!?)ELuUlLP)dI#J4nvKp$Nj>mI(X;Ubz60vMK0bU9*(=tSmT-TOY`(^i2G z0gAv!Xeh_nTCDu^m&OQW?V4T9S$!YkX+(G?Q`kb(U~bXZ{a+78tg>&$U+U!x&HY5} zMe~(@MA5QQn59)_B43Mk=!VE6Yj;>di?}VPhIz-tUgsfyNhL?!?z{dnos^(}GY8Al zc~0A{XXoB0u~+Wvqc1Z*5QiqDI$(=QIgQt>4@XuMu@3_9CyE)mPafVudnCeEmJHtHbLh-T4+{3mxO)fSF{RTJ6Y}^?MupF{k-U?9KH- zByjB4EyROW4+mf;1mO-hm5&A_M6sofoJS>C`;AG|6&|+pjD2*C?86n;Md|7z(msZA zjZiZkLhg*%U|alOe!}9P)aSg16Zob8E|H+Q#qjk-FF4Z+w=uz=VJC=z$*SJD*7m5e zo9(>!_(mJyepm?m5(iOD)qg4jp#K6j0hQ93*6(x3YbgvZ=&RTC|{TY1*Jb(639dOoOFL&-ctxU^W&##H8=J{k|It>8+* z)s<})_It-Q0bbMol3&HgRPHM2M{gQAD=pGMtyPuSR}~yutL3V%QuxBh8B~DhkCc{a z?`B$DE>~5=gIdR)?=@+dZcFsfN(S!pbh|m20H_;cw^PNe{hgOG(%$_+b(!&yqfoGj^7Kr0T^@8Z3B{LKzB z-y86!zXQ2)Lb@3L;-gSbxu$n%g#RgPp3{j6L%N!@TAW_y_s$;yclp=G_2r7gkFXVlLH zU-g~`L=@ir9L>?bBCHo?meLum&;Prnh|IV;twq^Wr^34;>gMe$J#pTmXUJ&!f#4G( z>{exkU@8)VQPKk5@Cw{t5DCduot0G`+sY|w z#+MI1&h*{ zE_4{=6K{eycJxB4UDtoBCy%QxJ?5oviw_+qqng!h)rU49SO5B7CPJ-2OaGC3sX3+Q zeiL?A+o!)hIUoVBRwOhuQ7f|uw;>uXI83zusN)UC&@|2xDoq-&KO0apY@eXwdX_YN z1BoehD`#S_TmAw7(hZGI+_t7+U0Y0aNHy{o-1OGP>#=Jq@(_E#r$G%c$gMVf9f$MF z4ZPuP1AmK5lNvyHBs$K5%vdflYf~fFY#d zB4KGMDXG~c+>WvFnUoadp!TUmZX)fB$KW^5hB~iw>Wq5H+yh^oW6+aqzHS2P2=HmO!7kf#VH zD)7z8gQ_cMWlF?(E{vSbu%~Mdx-8WJ(4{J*jO653m95~3m5BtRnR~b{p@e9eynAlI z%_h5nyYRw@SNe|vsWJg570m^Hi69yfDgEvTqG`rOvFOh+R5boiS{$;6$X))Wzi~6d z{(`pD(ESafW7V)5ppzqp?aX3LPg@WLnnq_YA5bg!7NIzq**+wrj5*jz8hnhMZ5kf| zWX;q&kk!aM$8kQ#6ojh5LVvI6B+S+mzBdGPDH-%Q8YNXwl_2&H4iTkV#8tzO5I z<*ZlbY#|+=&~-MB@{x`5h5-PnsRjP-iDmC zeGB9twH|qUJ!^_*`Zo~D*JdJH##q~ws>r9i}@%|Qd^7B*y-xREB;&)Jl+7D-jEN)VA0{D^?q z?_lpX!QkglZ-CTs({%x38jd_I#UWi2Y~9D+?Q0b!`=Z_Njx0?0=J3KAqPev2P(I7O zg3fu%8^P*uX=G_XsB`al!+forNzFKCt5Py}dk*2YL$ocHzUL=3iVBnr~;PO+e6uTTSmNE4FYeV<_+~}ERgq5J2XE^`vVy?ukI3Jg%&0C&~M1H7RW(K;{Q{*|Apg)|4aYVPzqx_xFVz_V~5%0(cf8 zjK>t~KVxB@gZ4ydb*fnaB;k^3q47tjgW=1yZpm&9bsH9?BE@TB+cCgM zBbnC@lW?T|(~;1%J>od8oGyH|K=a|WIs<3>cKH)(MW$oJO+$0SteV=Z?Uj{1HOV{# zX!HHPVMbj5)>C++x>!1H{88&)+G^FbWn8tfYA&^6>0Q_=!%`}tv;#@@Rd#j8zJt(v z#IBKCuP4^QcmkrV-?glp(xHN>5pGUWGLG+yqB-SWEetQjK2;RNqxMYBgh>HzwZdaM zgk)or2z|X}=gy;$K=ad_Lr%vCX-7ixV*fLX;?YK3d3xO}InirE9e;KGF@}8misOet z16J*F%iublyh8_blV#PWgEUQ)Sl-s#JNI1b7L=Fn?neA`f%Q=Cnci(4P2k551@l^; zPEGw;jx%yztZWsnd?7&fnA(QrFTd7`1I~L!@`RLq@JuXi>QKukvu4^9c7wvaB2F`o z?xRCV4WV_i!xt;A^tltW=BQ-*-1ud+yPP_(N7}9uPX%hWR*oHPJluQme)r(jvZ6P; zYsdum@qp~1Rs6a?>X6%9XE7_ zr|5DqeJRZv*FCqr``fCM%CGC5XG?FN5JQ=PrgP4b?8?$jLDLjd`kkQAQafPl0~t9m zYzA``6R5Jru0XdMQQnK zI;%qB6b&r*GKP*~;b~PiX+{w3bqDiP*Kx%5*+~lSJ>MS1kNlV&VMh z_!A@sMVMD=$Q}Jqq3s1%BZ!e9H*aU!PTiRwP1&(2uPj4~NiRYsUfp~y9icAL$zy{% zzj$^IyTSa*+U(E&IPx=&H=0r`_T~<;vfC`k({byp$Vm1RqwAdW8eH@WLg}AC8nID< zD>UKv$aNr-ZbwaHz=41smWX{8N=0V1_s)0G95wWMKAkZ#hg<1+R=iK`?h+82Lc@5e0}?Fy4znU6xZB+0z#jc%N#Dr5KTfskAyP5kf6R z_tJYc|8z}=>k?_VlVjXVroR}RMK(+@Hm`k%?1fo(?SxPB5~m){W0+XAyQ?#K`e9mX zHlWjW_;JA;_M@AkXFVZdp9ntW6f0I5s_60$_6~%ag{XK2ZUe4wr+a$pAfT(x22<&k zs?LKwecA|a^+p&8jV?m+U+);pEi6)4aK9ax=BkB)A%3=#O;vih-;%mTP<=K8@&sZ{ zvUXZd$2r$OK+dyfiZp#B)#@ei=-aOteDROG8A19>Lb|yswP;qGJ?kZ0w|?g}KH;6^ zwE!Vrf6izb(GELA9o4P10u8>~H6O~unCtxc4VVvIM~r!xjjw$If%G#s_GI`wu_N`} z3Y)^0pZb~afux<6tvg$-V=>$KrGl%Dr`7c}-=+*>V6NVguEG^=9tf%{2 z6>%l(-R{=#9Yn=M`SEUT(89iZMTRu0mxT~`-7)F^yz#F^`raORoR`H_P83T{za`~7 z;~04nxSy?hcc*^%;gH({ZgO9`>S@`HNqO?}dcn&APsubR-0AsteCl;*u}9zLISh=i zJzoqsgvh{5Cu+(smm4{bm}2e=t<`K#+Ta(#7~A+KJR#Z7bGdtx8H|q1gy)oiEG3rf zm2D$L!8wSSzUWxfNk-D3?2qayytI^CZOe!Ef{7JsMTDyI8+60wjG*Gaw@=5NA5{0% z_>XT@MW__5csMgC<1=!NaA|iAY>(s~bq7tYjfQX~EJ}q0W?92WQ%*A6%Pu#wc&e|o zC2ec%-A&R;XBqzd_m9K^a63`A@_YvewfL+F=BMdpH@@%M$kBQ+7x{fQFn`H1<_k@a z%wtbBx>m&&HEoqf8_CUI5Qe|8V>M0SW}n}ysBpD*)buZxZ>rbh6YzT1EmqusAOR`X z1lmiTQn)8%`?y?kd?YwGrx)v^8X!haiuxpTDetMWUb*c&gjqDLP_L^42&H9>YOlzk z8yM@ga$!`(zWzZx9E3L?E#95{;xS1ClcxwEQ{|%d`h6s#a{Z&>{00vq;nxB#?AAh? z8G}v8U5{Cg8{jg0u($j(=1Z2kW*?iB24cwyY~Oo^Uz-3K6y~bE@{dJplxQ&jU7E9n ze6jIChmreeFqTCOA%ddfEkv~BcZ_zF(nedPfF<73qN-}5p!N`p0CP{&r569%Addq) zIE10TqGHR;A|P}`@b(75%)OzG7rFce+^XlRt}Z82E$bEEy(~EdeV7KWU7Az-53~Z&v{*Cajwf{bDx>i41eKF+^(< ztSLgdx#U@jAt;=5q2c%cOAmolGrsUbO1T>G2IiA_Lr>`PN9`zI?lGejzNpv6a--wG z_bAP1&0F^J7pYoJgWE_7;!X8oAOqujWVMpAq~o+ZX+}-7S%{TDb4ErtzQy44I&38( zCUM_zb5!`go>J|jtKvfO(ck-__c52O8pH3n~ddnZLtR0{hH&7 zpj;=-C*6cw$MV{j;f$EaB_L=f*{xU{A)cyF$pbu!{{A5Gb7GX4rM1lO0EepLXEF<{ zgH1Qj!^C~h5D!U4crA6xZ$|>P4JdDA0~GtFSv7g;Z+X+3U9GI?-N2#sI{L7@{$Z=E zbnHEqn+8;gk>3!JaaNO>N*3N6VFe+wq*SLT^^SMV$`E19cZRmW%>7KM45GCjJ%Ki5 z&G-5tn{r5zG3^qu86io&aYNbUEv@HOob_wIGdZ{4or;L;dpX%P{=?xXC~ z`I0^vjVHm{!YGF?r)md|SohUID^pOT7 zflpZlj>~PpM^P2pk9m$>=6vwPpOpB5-SM8)<>v&Kk$*&fCACnEn>0BL{}jrz2tft= z(&zbeWR`k$eOuybqi37MlA4qgx?t+d`(M^cA}GXVF;s}GK>9@9TnoBmV$+WQYpchg1Jvr+?3yTsj1EvENs zluRe|9aElLnATLo58q{slVA0b3yb-3Fid{#BT6dDH1X6G7k(u@5{C2>88{1wAgkBM zaGE7k^^0vS@-$P&%RPgmHkVirN8O*TX#Grog=R{EL7)AUJo{$~HEK;-JzNKpBgcns z%yTOkU)!C#QQv^PxB0CxnlLHHC|#=)cHo<{=n$j|iJvF0?1g*PkCIp7)=UzN>(-BV zZK{#?`aMOJz3Ox+BDbk6VtYQAdU^{6p~s%}sez?Gv{&RV>jk@Rn%%ojDLJXLt2UF% zOa(?*ix;J3hqk)5lQ#<1!tX6}S7zv|WOyU98fS9fg)44*{{!BR`;U(VDzH1${c(^r z4Bmuce59AZuqD4vgI89JfxWO*$1yfAA)z{jUM852F=qKt8w}HRi%O*ZI$0#cDm=Oj z=(WA3e20t#7FL~?eJSv2Bjdj01_Nf_5!3NhPx!*7QAYgV_@p5Kh2dEWajk>Cz7#vvTNjpCcG z4Ui+qn_0mU0-hf!)}uwS{{fO$&W@84+!Wf#Cm3D(+o@P)v+KwgUZHYuZ)&t_MAJ&C zJhA_QgXdf+S)ZU-@mxS_p5tDzv1wqLsbi_r(7QD-WLlo9Bu;O8xc}pR;E<$4p3|k_ zusby$AkJsoA6GMpJv~lm;i>M3Fg$T&m#s;iTf)>l@$#>Pn~G+N)rUV@oPVB(Ez%c9 zT2QNJKJKPgEqwV(${``aNw8D|#ea*@sGVPsp0iNsa-IK?%mty*8|1qp@X;IH?&FlI zo0|o8Duq?Lv6Bm?jMf;VaG5oOnKWlL5%+j<47GjVlWt}8RIvAKbsedsp<5{kB^D%! zv?V)Y0~H%7mBsT_SGwMHYK9_=y1{VV7i+8mtBrZ^UO#*>s0@Ji3SEbb3dh^)&R8Jm z)OQc*FJ1ce=^A1KuKd|ert^5eA{|XuYPwf+VQUYi^~gmEw?)a`j(F?9UVN}76>J^I zM@ChjxcyA1ISVxe1D}yuxinmzzC6A%2}h#W>&8Pz%>#p+$|~~YYr0{EnWv(COp9yF z^-&oU7aRP{afWyX&%l|ai4S>7g%uF1;;6Ery`P}aJI%npcWdI87KBT+u)E9wYT@@5 z9r^B)!Sh4gYpJ+kH;&EXIvfk!sHTL+Xv6Z_0@VMZsGjqGzEj`dJypS4(+7y^sHOnRbgctoaSb-qrrn5S86v zBO8@jQ`$h2+!dK-e`+y`=2$JT6UCH@DW>D{gNVCk;fh(Yo7 zjs1Enb7kvkf)xI;xZGjvKh2!dNqyf zQhR|*yciDaPRCwR(cpm41R{K<|M53=VwZyM{9UI-1Cn*c_sV%rBCX{`vI6#XTpTqG zUFTU0+&O|cXpyTfiX)SFG0X-P5Qd zz8v>uycOanVPu|=P^0KlBD{GQ`2_ZiNeyvnU`UO{V90NLD`jJ@`imEU6_V;{KP2OY zmxQNy<4H3zI8wFJlIZ1%=rvy+Nqw+Fr{Uw)@a3(WmYKHJHlCjl!=YbtEWAOZYgMjO zM}Fp6pL9Wc#c;M7A<+$`?!LzAcBleUWZb%&i}J#qd$_l5MDFT;!{mK_n|{N--Dq>< z)}MACFDSJ1R(`=S`G}C^rJGVp5?;5A0j1Z?`dhL;!J=IQewJJd$XkzKEkkB6H;yuP z+Iaf}qYHm^)$@>dt1DDE0-B;~R~L>k>v^B*w5&R|nP#8bHo(a-8|a4ZVG_2&W^^m1 zE3%Etk_)wx6xo-`?e;oqrK({d8=FyAMqnlHG`ad@?6>1dLax1!u)&r1AJ6 zF7^2br<8=8xaJpq$0N+Yzj`8*Y85z9T$PQVPvJD7Z#s%Bo=&L)!@lW$=lNx38g^k9 zb%1)RTJtRYkb=GAcoT`~cmq|~;Eqa7zD^r&b*D)={c)=?OJX#ER1uYxY${K(^oWYv za43KoBe3A(v1xfv{Zb|Nglh*|%zl#kSbx9IX`_V8PDX^T;~Aw>8yur|{Dxcc#F+4> zKJfA3TF{rWxZ=}pUTH)~a+jK*T=e2+(C3^;Vs7nerVf1Al!!k$gm=8>k~Z1lEOi#6)S%>#4%&`cX5#w{(`!JK-!m^we!Tt+HEg~D&2`OUrK=Sqk zs9mu5kkV~5+`>dVYB+gO$zdRakjxO&u7p7)k$r+urWiv4kIUltUIPj^H?{t*(n}#b zYHH#2ju<%a#Vop#+PV|Sr_0j)YR;b~{~S*)fxAqMEJv$Tn3I)K@>PD>mV__e!=0%5 z5R`Iv&OuXfMtH$X1ZvjGh%mPy z-HpY|r0dqA`^-$b%k{gr6fE?e9~#({-Xs_o%`4cxeDo_8P|-ldKOZe*rG=}H9JI=g zn&mISM=wT;D43*1Ta{i;M z9urlrwe86@Wr+aJ_Iw-KWoxk`Udj~nbg0Y$)M^!bHP)2T4Wk^DP79n!v;9=AXIWaj zJ#K6AHRs6!6}lp?@YP-(6vTQ3<{^~I^Oqr z-jI-gpX}SHz1*IiT6uLLbPLGke=SA>cq!>kUgGgtM)-jT=)NFP*MT*81(aZ9E|C=l zAEGL+IFSLkZt@HPqH64UMHN7!q=9T|Q_3&5QLF^cmU+}oI9;z+Fz`zGpdl+>A47D2 zPZlALL2`Ys09^2B+2Q_FxmkooF`@9lY`VyoXg1od7NA^IZc9NR>w-!~mKUXnu1gO{ zKGq7eAKyO+`k8$G*seR}8lV|VP&}mfj?LHap$;jdb_>OLj{=Fzn=K!6BZ)HmqlE2R;J7L5y|ZY{s*?N4JKDmGG^=bzt$>iQnl6i>YV~ z^QNF*^qR7SKM7sle`;XNbz&k%NGB}t3)5Q)6;kB zl#~?l9iJnamNKAKA-b_wK{D?GyEYPv){|DsFC7jC)ZEtu*L#4}9A0-7c9LA3v>rK{ z{kwe(Pg?YqZ}jS_rmj4XAwhyR^kG4b9c;gS5(MKtlxmMk)j=+@G*KcF=Q9C0xj(m! zIR?sm-~8Zi*R2av9}Ev(Dp>TMoR)>*=Hu0lm+W`mz*1-s6B#KzFk6uVZ|+ac_wk*(}5*J#_gFfz?`eY2eRD?citYM(pk5@ZB2o%2OR z*mjIfH2s88)gw5bsP&QXy~@>-BFIy9Q1!ibr6<*aS2=-4MtHeP`Z+bd9hK&8{aaPP zFwzLjE)L8oZuL>hrHz|V4B*-=f}Htbdsd6w%R9Z8z>p=R(L=g#wc+Y_jN*5VpGuh4 zKc=<@Kc==|vNM8akPo(oPPug!>|~H4E~WRr=p2a^hXCEE4W(i8apURA(%^`cSn0<0 z{r2sZN{aEq#_5O`M$2)SRIhVfe%z%4*?I(rj} z8j-w`I&u0Zq+j(Qrq4l-DotFjO*{IMsj!iJXij8fGP{Dweh|K6^t)+_sZaJ(W$KXJ z#&Rd8ss=H9sh?wOVDJY;addANA}a&wMaW*6I~YnBE}*RQtqgYsYdDPO1?z1+l(5QK zHfZfk61?8teJ$)aEkgQ2oz5GTe^FD|+duL)CWni3OIH({`N;L?q@DG!-eeUKhGd|; zGyR;B7};87C7YE&TV)SnYI_RIfp&EvvUeOdExCFM{q&fmh68wv9=8Vg5+AMuX`V7v z)D!R1+41xq5m83qrNBoQB^P?{uQdQ7)Z=8Ey{G$?=Q$bILr)` zXKc%CkF=NN_$oQO<)XKu|Ldw+(&=}trEO4KvfNY8B~v%xnP5F?6W;nyAAEK4!QMB! zeT}b_sKQJf`Vkgkxm8nm4+UqC#hk#vIpBl$&J_z7e{Q(Hd5ii?sNK9~%BWV);pj7h z%4au><#C~EDtx!)bX<&)+klUMk%v34XnfQ{)qaH9ZNjwG0|fJ9>KkTun_l0mumd2= z@#!Il*HrRX>;+X%b*5J`YI*|5?*ifPeitxL!0#>(Cr$2~f^EWbRUd6nGpF8b{Z;Gn zLtFAL#O%-BUonyme<}guS~5v3KAz2g#v~K^{O)H_)$lUQZWg^^ zKTyuZ!rU=Ih@7X~du`E95f1=S&dhZ?V}&?Qp;4pi$-c|hQt7gSOQbD>V{VZjPL9!L@$tk3=D5571^X15aN`!3VvJLT+o{)>@O8O?fNfD#xK!A!zF!a z|62S-P$OIF4QVHt-mV$D@(OG~NEU4)JO4g5%gL$#?0e>HLetC#=3bxu0Rx{=%d@@0 z)q{B5E;)1mt~VK-(LArOaepgpFp)-<#6+$qPv&-5?39XMEfFnc>192bv9;x5a>_Dh zk9OixE{WG(W_J5gJ*LFA`mRks!dZ21X0CPkJ#+gam-@e3Ck{i0!vnlay*9V@O*Wb` z3PH7qH5LYu@?K*OkM~3*8RW2MAgx;d`)6ePW1v4sE37d?faV6(#s08ZiE#QZ2H7q9 zN6Mmn?s=v)3^H!+%R1D#^nI28fL^%CtZlY)nRY(7`Q*PD9AFufzimEvs}Lu2DNf{O zO+l-b>pP5fC0cX~aj*IDLgH||_PZAi>z5?U>vQttdH#a=Wxm%v#l_x7H}lBb_p#^) zD+T=00w5|mhU^bwrlk>6xVQ?ffey(;@yK=+G2YndtepQY^UfzIR(i$eByK}eHxn)u z6!w0hrqX6FrzrW9=VDBOa>Cou{6bd_0RTw_tN$oYekf<`6&!t~jZ zRKD5cG@wiPOruo4|L}Go9loZ|4%aehBjGR=ysJ$1&`dL=OEr8nVZ+M~IURH}8#ySK zuhR@tstxvfTeG4p4akr)bvn9_t$!)Vy>#??w}rcHC1%>CALQ5GMtY6|3(60+ohKQ- z{*yTU39Np9)qnptN`AGU{}<1MU+DcuB>k76`;Y&7J`kv}gM;eW#{d4f|M>ME4^vj6 zl6tMS5hXtoR{r-_{QN5|by9nhJ#ha|lPUxlY|C%iZdcX)Z)@;l(0?ZNb2IBBsL9sOhiNFJ`jDxmJG7Qw6aS7+j^~e;&n87_z^~9rabHe6d_W;@6AV zxM^AyH>OtY_UHVY6Y`6z{q>#QKkzg2%;iKq)ci35-sI{~u}bMvvP(_kt#|9W1N=%x zV$5u?DM-X}M8Ia6&=z9m-Pluv$AMjfo4PdT#prk&dYsLB-2Q#gkNMb%RL7+iA3X=$ zqyRbqj#J!KHGX8ovPD;4Ijj)nKjWU#I577}>TW66hRg{zZ8@~x!sp(99&;tYJbSHs zK{Kk9oOh!My;<1g1fqMZf|bK4X?wZUa);I2`yXSjqK9CR^1c&USo)lP3hVAnQ9h#Q z+v{A2be#`n);@B?;7pX@yF3Zc`TaCv6_lGLB;>|Occ_-!bdg)9H7s^*F=Z#|Xp~g0 z7UMIOR57~I*-%6Lb?Vxn+|H$np`}QE?#qM0hiw>z-zZ<&*<7>*ztk&PY>^w=YdD;p zFTuM(NZF2pftHu=J{nc6cTR)=~rBR?D$fo9A4Y`@sS@ zq2XPhFMYZ9!Rg&d3`Z;MALm_3Wu^%u3f``>U%tvw_AJIkYWSA_Qm4<~{W+xrreOxR zNPUiZXfkPiGA~60H~z9^^fxX)=&$q1TJ zbk#Jt{>u?@@*`1t1@&CMO})_BLfh7ss;EoW1sAm;Ol_19ZNHaxAB=6U2?-{sF%q!a zt;=AbCE?q-UKK*%NG%mWrHB%~&5q`JWv$_IGjV9I0Uw`uFe%)A%qwD+wv?b9{4uBJ zr(DpyzXb7SYq{UHY`VmZYrL=P5YxZedq6(ouwZ#c1$HtID+2jQL$yKjK(1KHg4APU ztmqTY`=3trc4X;UUHZ+3@E5K2#w;FReXO*33OusEpwrMy#LOn%T2N?pOlc>+V*Z({ zaLHuCr~ZNE5=(y<;Uu5Y_}&5ESz^~e9bK--#l(1*4nC^%8PLd26y12__g$iRQF1eH z&-B}+Ju5g9F8y+O-#DY_+f;ic!@3mZDkXNCNa&@{B|Vm*CAXP-GE8P9dgR7agA2-L zE6eUzNe}m*4i&y>l1q}3pR1|~c0p-ioS=+qoIw;%kTng*o__$?q?U#~Z3!n+Jl~v-yiDI#( zn_~gnwu(msljra*PrFt7h_=Eb;I=r|VonNXI8zLl>_%xhN74eir0Cd-l*Eys5-jw0`*IkpI4+~D|;5P z*p_2R+33kRzVDg1;zn<4%*;YeY5BR7UZ_>{)(S}}@uKw&y?5X~uGqEO^Uu6~3vz4; zReRfbjLmE=wu*UnG@rsDkL7tWacHn4BiG3A>n3dD)@E#f&viy?CjOf1g1xYMv@$Uo_?FW?Ti-lsyjaX??ddMj22688y; zzNx0^4Pq3HBPZqJz-5WAYtJM`4)mtU#c#$F9{#PgMb_>eiM2?Ed9g(NPhl?-J=dUIg4CumTI-t`5Q~X zf+tPA&=CC`bL+mk?fkCWe*O@3k%J@KMtC(JPc7^a#svny=>bPmS*8`Hct_g7M3zXx_%Bt7mL@zR1E!`ly)5&b%Ei zIC!Zi)t&%sHG1D?^u70gscwrQ=R=(qzJ2x~J=J8P?#3yUy4b;UXMdQo%wwzg*vcq! zV>$t=xRf5@k~bjRmxa zq*JF2`Qzr{dA_jzkug;j*q%PHTcKAb#oyv@2r*B(sT``>AkG#5qbNZ@YB4P1h3_$n zmETLwSTBdIG0Y`hR~P5EyD=9hp_fb_h5Mw>{E1fyW+%Jy#XS^8HHc#Y!(stzjiK_? zq5pgAM|1XAmHJF7#iVI}14}PT+Sy)NG{IW+25giDG}em&RpM(XIn(ycE8(10p`M(; zDpcwURS^IE{klfreA9SqESFI>X|7dnw*2KW|CD4Icboj&&vYIYjGwA$-YkN`1u++zUsQZ_fcNz)|k5 z6+z84E`d-j$<#3Sj<+rnNN#gD=sAcQ@ifUy(wMP(vTzc5wdyWk2daav00^NHL zXQiNctss-%q>XhBSZCoBS)X=Q)@IxV>-+bgT*z@SwT0&AoM;&5LgWkg&q@E5nrHMj;LQ7SzExSjGQ0oc+=B!X!rQJC#fQP>s(BbDv$H~KSea!)wvv2py((r2GJ}o zgU%KB3QS`jV?&p|IKUyw!V3(-v88Sz=)aUnS)*5z(Xun`fS%iY?lw6i%h&cK3aozj{;*$qMY z&Q?hI0xP@FJI`Wr_qqIA&4?JMu|uAc%3ku*K+UfmrlH}xN2gFK+jBvo_4dC$Hy|VBpZIpj>EY~w*(Yk7bmG`vg&$#k#-CW zqxiI;$-F5?6h&BvK9ab9f3!}MZGYylvFo5T5kt^EySB#(3 zKXE~B+Vjg=>IMrnXJfZB#X?0&jFDfzEdlHQq~O9Z(CbZDrfx_AP>3k5`Lw``up~TM zACYb+^`a#)lKvHf;#{WUI-tFUCuMN^zg~{XaM_-Y>eS}?!Coz75F+Jii>anc^3>di!>x@I`LPVb*^D*>7za3NmQ)wv(&50T5!avX)8}~+ zR7AzZz))^JbM-qzMPfHf4Xg#LH6yIr(~N*ls(Be#IY(79&9(xBq83;04;)XOLE+Bo za*tJnM455-0sDGRuUm~}AD9QF{MRW{pUy-$;iLzjO7tfLJ(=JV%JvKxlOuf|;?y2c zs+^?x2ri(RXN{HtxOLip+FTt^#}cECEp$F2C(z#R$4uGdo*+ane?Ug0%u zfTD8RKEW@P%G0K&C6nQ#m%Q?>)nBv>9KFKBeuK2VfAJ;#-Fnt|*_#XI{d~jN_LPqA z(usTc>YJ#DG<7qld{sBE9!#Q(?2vnJP8GVn=$Zdd(}z-FzCBgfBZ6EmyWmN%3YNEN zYqsZ0@w;C#n&4;{UL%of@J=hTPL-WohWxVUriRE!XZQeVb5x3TdSg_oDjtfKE$*?X zL9mP0ySe4N*{N9N?X2hRG|Kg`L_2K<-mYOB(h_ZMq5GrYY{Z z_TOzYXD%04aRp@0)>S4i)Eq{huTuPqljP_Jti@z!ZdCxd{-QMxn`x)5rKHss*S`jifQTL6w&a6{1=L%tZ=yYWBm_n`?n`z!VWp zlEDWdkFZae*_BhfP7o=C0<5H!j)9RlIj;ndq7@dRAWGmnjL2Vp&*vYR%C$RGwhJ>NAh3kOkjp+urS4~04MS^Ax#PRB$iwh zpIE^&oMz%j_~-Hkv&0xYJH~Q_VO~VI&vG7+b-xnnE$67_QS2phdAQ`(j8~Oo_T_-! zG@LgcY_ufFvBaT=jKQU0jO+L3nV8+EKj(q*fUrT|4Pm zaEDfa^Wod=XJbSC?q3Yqgr+v!wNa+34Y|C;+3s8JP+fH0tF*rmdXNNZI2(mvIH>Ec>eSCpm_K+6v0GtN+i08No0JiN+MAE9C9@(;+3bm&#r%S|KRmoY8sldp*oM#n47* zF@IhvY#1x-B?svw#8@ELhK^QcYp4X_c#L9@>x2{=+_0b`OUdh7uw*XX$@Jxkq-VS` zcI_LOLVC|(*LwKku7A{42{8I2W~eQMX%!7n|avMD{P-J+CM z0VF8^;AJ#1a>ZLAV@6#w=MWNLLS|KrX8a4__J|2}t+iO_ya}wXW@wUnqktX$PCymY z%wLEw5I5>H{M$*BHnhmZ>xs~yhEjqMkJ{||={z}QkFmFrG%ifnN0|DDdtWnsiNUa1 z_+8jJRVJLNDPHGuO?no~BLsOVBk)O6$A^txHfZ6r%x7t|v*_|7a1EP9K)G$$ zn8_N0ux2F9kR|@im6cxG6#2jz;8A~)sMh024r=u{G?ItXL2LJpiE~|Hz3VYB!~=Ei zM4M1SwxZlWHnpJvQVDCR!nMj0&B|;_R~ULW-m_NfOksvI_S)9gU1DY097{*mg=)08 zj={tiWyrT`tJ_^Rr{B8#=KabZe?n3jHqI(EeWVfVFJ&{;HFqK^Ke5i=T0JG$1?ci^ zY0*l+LN*@iy{J7bq|n!)9-)K}qi7|xzgOSFOB+NObt)0xruT{ud9VF_<D9oBSf6BEMx*sYVQ6&Ujx$j1_)K2*%CtdmwAUwl$eEbZkjFtyA(Lg!NXN zYNCyn=*6A=03cLULTy14S7!PmdqT0b2V0!?RrC%&3I8tp-Mm50K zx>4<6j#f}|(FX>8LQ$QyJqp6|Dd2(O-*J&#M3d#=LN0jrU|L~|Y@=$Z7cr{<(0Htq zw+7N z^$VCK`z=rmEmSb9df3*XrN>cxQISDJ ziD5HMR{$ytUHyhRRAVNx6QaxcKmLNHfy&){qmmrvxpHPYk={_B` z^VF#PWlxg%hTx?plPB77?R3W~G=XDtS=WF4E#C4GgFQTal@ZcTpDl7KMTpMrTlZ)pFcN7)bUzjTmzR|mM7Ka z$T!a4%nve!v9`DUrCi|`7u;c>?g!wB5##n9l#(u>gxl>M4mw!qRZQ^%3QDrkbl z=fp*?<~Ww%%&4sR21~p|w-!SYA$F-Cot!jGl(DvsV&x{rCo?*C^V~o|5oet>=idi& z$*j-rY|pwaKCwO+Cvhb?Qq(-t1q+UAM<{d_$3GnS9^Pew?J>O6?Ab&}b1Hk@pm{6r zfaM+%@WT>oFX3+ab-Y@{@4063G4!G+R*ha@92#+|Q)cUvn&J=?S`(ydvyTyHC5oJp zJc!^^PiY2_qqs*Eo)O?qT?H)6nTPp^>67A5-Os6;Zs+7);p|dZ@`0 z&|?vWp)1B}ugmz)KYb|8XfA&^`$oh790Aa*g>3O#dQ<{L6nf#v&He`ZHw}P!8eUA5 z0*#13T;|!sf4aRDpEez0HH5)CEh8F5%Y@~&D1^|ZY{)@xxQm429Zf`$BRi99R~6b= z={H=D{_DJ_)f7fsv$Wo2-tvXZL^g-3*K!D_D7vK*Gq2Vu?dP2*oC1dx>ZDMa$C5s5 zJl^VtEw1gLkpb(>kSj!6VM+h>qusnJ6c@vRA#dLMbsUm-5)r0F@|Ps!=W0tUfbp6) zkw7TyD`v^N5J7nK4>K`vR^PK?95^-&ihvOs?WW%b)P23G+EOL(jHISZ!8Y7AIDYnH z5N`A(-}c8z=qzXlS13}_bDE`bwT1MthJl6_6x8)sCnIF3IHiM;zs;d)YuQmSy*|Ls zVT6+Z2>MXa5YHWJm8#h*N)3qtzd`Sa`Q}&R+WZ!vRr;O;o^fNvh{9LeWm})yH#j16MQlFYcZ3L#@S2r8;l5~I{X5y(z2TKT0V`!BPPbdJ z&SMJZ>DTvN?b`vwvOso-_+)K-n?&IC;P+3_k64(PvXt`xYR&y*4uV5ZtwqvDWyQ#8 zcIGR%Ze~^kkM5g@_EYqg@H70H(@(vm?G={A4<*f1?Q9>d2}5Y+(4(x?$3*F8qYsWt z-0nPMYW1`xnk?*>f2*vK9EL)K&8h6;jEX+!u(Z}?t6SJt%cw3Jvwe6qU}tvcne>$L zj^{P~>-0Z8`#|OXA1C<(emcLOhEZ6@kDe7!>Xz>k8D;AL3q{h;;4-6p;NCtTD#iCM zuxwYl@M;TMY)28@r<);&d!3V#ea)=^hanipbdUp>e~8|G1eUDIAU zEWjtqeh=0odA=Q`f~3I2K5Ak*5W2j$POuA+o(-PT|Ies{5eL; zZNjP9umqgxu>Du{suTOt8JTcq_4dGhZdo4ibmj#?C1A8SH(ew6oS4x?tKy3e#iH#Hvly zZNWraTr%rx*IjG+<@~j06A0Vbr299JHFm&rY;93k`LgMjON}or_O1Zmz6_(1(IAxS zR@i)rF!V$MD3_zmzeTOuo`0vUh$qX)A6Q)v-Ej3FkALWW`NqcSi4-zxm1?Av>eQ{o zAg}BnEdV?(TXh>#9w&k6dL}nNddhVL>R{eo*5KfmUZ3so1sf>!tXI;iOs{h|xg7l) zCq2V(pwFAPlK_WoX{GS`cXu|ABMaIkIbxpJ*c&BG=&8V2k*9O73L%tWT@&HTnE&BoaL5BDzxxzJOM0 zCTz&~gt1O_GA)`((@^qw98D$FwQ;Uo8}+G9ou~x7t~f-7Ji~gB z;=!k#1ksw70T#6(LH8Ru2(`Y{U{qP~iE#OVpWfdI>)9$nGYmK1L`Kr&Tb@rYXWBo; zQyVIZ<3AjRSABRc#`yH%%X*1~bcBRXNW^LAzB>$02~#fC>Jg(I`B6+AV`+b#_Er%f z-5FQ)S@-iUky3CmKs`upg*dHYp5}WgE%#>;!0;Q1{a$n)OGj@nANgWpH2fip zX(rh;reWNM!j@jrbg}VUcZc8fjpMW*Ig~x1AF6g@At7sT=Qj#-eaAN{bIJ0+w_TIW z^lnyDv9*HY9$_30Bp9EO;KK#6s83{UO0vk?!5}DHKxZb@V$(7i8+wT0efAB|{1KV2 zu418OPr|{!s(M|n9V`WGb%u$KNeoq62r}|YBK?C@PYa6AJT_1L!6vI*olaixYT)I)a|6Pm+&*9gq8xJh=qa&iUR!nvRv%e$=U&Z9eD*- zC;jGXIG@EgOz<53H<$=2j6GXBnnnxyNiI_{dZQBhfE$2s!bn`W=>l>!zs#0x3e4maN z=abn@Ys9xxfYpk8#C2jBKL}6c`ku}mrm9@10|WO@tvTJHnn4>{LMvzB{6u%26(!rt+YR z%AyL%H*37C$zTZ^Cyn;>P?J0?yYi_m_*Li7G|`k13$M6IaM#5(&^|`|oikA8P_O<~ z;}uds!1kxs)m0s5Lc}VAFkw&xqajuy(eGBzxo9QSBc^DnI-;M_Fn&}ld9UzH-m+bn z{MuwotoOdZ8_Bt>#0wdWEI<_s7|1oen4#z@6VAondO<{POrLpb_vWQqhh3n)=Riep zrRR81o!r2Dc*EjwLBa*CU^(b~1wl(rMsB0YR#C4BgE>JsdDWPSZ6xSRg#5xMKUb6N zybEM;u{4kKS+^CpG>Gth1YI@GXrq7Bhz=7j=g_pxm8oirB?appSjA|iM~0{e$xEVn zcf-WA71!=^6=hYcHoJ_Gt0nt`6jKhhX#}&rI@AwX_3ZV}zY6Y-u%W-3E;1jMnBV`} zq>vhf=OMwY)D=waT++*Ca6(EkHhkU2s;mVp#sImR>R{H8^!f|~D#I6WvQh>!0v z<(yg@OKGKTD6g_fpZGR&)ZeH3@Zj|3Ai6{~!oIy~^|}tk%7PR|UKYJDK}Qu)O{_37EJVwJ0w1N6<`SY~x@JdHpY_;>J#E+GLvTde=!Fqlyv zP^UH*wtoAH+{A@+1`mDG-J`R)_bVI-U{PcDok5t+9%F`Iaoo$03L5_jRBvDe#w)5$MS+&y`UyUyi9$}xW>?x=jw<_H!|zWl=F&BR@n z@jE#_WfTzImnH~z-N0q>w|oOWl(y{(09o4{59*5G!#JOQ#_(N6^foHwVzO>PG6)=R z%8lD0H}Y_vFA;F4D#@Lix0GXu@lMkaP7f4^jF9V|Yt=u?PkvY2W?2*luR?ls;l@-(=U+O=AUQ5 z>$4rbh{yzWZEa=~aFm?@b1>!O>H zDVt~ImOAtbfuC?T#bE6W_uBPoVYfcl@^h7o5vAgneUHRRfl8QW4zKX<#ewHrAVaYu zZUY_R2~zNh4$YnG`zbeWzucatG=w64U-UfZ zunQE}^if`wi|F|6CHEn@0HqX#zHsFgy3t86iOho=wq6Q+4$f}$^VFEyOV3E;@0pozX#B>wd-SyHkJ^Ewl@k*Dz278C zg(VKeN!zzTeDkHT&ECCq6Oh&Ztd%>zH6-$atZw(hT0>0xfx~FxF+E?;Ef!<4JANP$ zA*ZUnwnN;w$PgMUkooHTI%w*9OONw+=!R;KBbidpW23{HAbYnxM`*-Ric9 z&KsRGN5p~@RtJM6&-MP7WZ^A-@LZ(4W6=rJ47HxfU(ewxHH~Im4+SzgadvCq{slAP zLXeWNM6)^FvGIoamsw0Lqw(9B${o9!Bcgb01vrbXT;Q;V^ck!_{FSN(8mcnPp#yk8 zdhE5^4*&!6v#TB066+Wf47gm=HaUK4L@Yp{3rra0049*_+_PK7-sZozGQgAMak@*| z9-rl8wwQ@rEcHe0uGV){@~glS>_7lzvNQHK7teV^R4k`NRGsNx z8$Sxk&g;?{8sB+11*EVw8N^4OK%lZY#?&Dz1TVRcQq~6IOBN7|-mN-23*M=zY%IbY zCt-gRqc&^~?=T{UUqZh zN51XuRQ~r*sYcX{RHK59(BB`tds_eVr?%sO9YDOlk@27Z{3|>CchTBIQ$an>e_lmh z=syd%D>;9};@8ppBNl(e;+HV&YSBMp@&7?ALL7W8_w1oFR==vE|6lFY?#p&R{ZUfC u0&akU$oK6$%cse?dY<3s`u+LLbve>ow)@=Yyv{kV z^Sa-^y15+FUuL{aOG``t#PJ{9fnR5}v~;qTE&;Bv4C-BgUyCB#j~&)3Z2fK+_(xc< z_lXc^XRRH;o@5e_MZPQx(*L7_zt(&1*I{*2c`L~<@?=J8& z|C;~!t^H8@zn-Cg_Tl3HdaRQ*|J1F*g!#|ayLj9uLQ8AymieDWC)_uWX=yoVo%r$4 z>F7nHYJW;@H*Ujpc-!X9^!IBs_azi}g?J=f`_=ONtIClbr#7Ue8EvdtYOr`o^Zq!$ zb&EF*-16?MS-0Q7Hoah*)ltLn4_mM4PA9aBP3nJnzv$;A9q&K#M}Aw}_DhiW!>WDH zRxR7KegCVxv$#FNZc@%jXO$p{Uf^OSx}sU@2A7s9G-?_hFU1;ftocfW6I8AH>blk< zZJnj7HacAW|NMae{n%W2?EA(4?(3JXI-(7*i!fKOlY zf1mzuIKcl|{{Ou0-+_#!{}*xpmf8M)z`9tiMgN;Xk5iP?^i@SA^{{X%`QilID9OQS z!sJlPfJ#WjO71GGF7!r^Wm_#w8B%J{id3ol@nPDs9xR@GFNmAAgsTVVVJ+o!|PajEsP zs2+RiJDh^!*QBiCr2LNf@$&^o@6_Qd7`Mi~DSv*y@IC&1r3P#`O|632O5RyrAPi4q zG<@#Y?1PH8we41Rm{~Rag8Bnk@MzHB`37x02b{7!$8NWrLxtXq`aTxinAlHCh2du| z4%FQqM@knipfG)YPpSeRIDP&VKij>en>FrTh{)ia*_oz|oO+9EJ~QxD8!yte+4h>* z{?=^Uo6=*``L^G@%#Zi_e3xouhJXd<6mP!%`^rA|HiviNYY&)(r5MmG@yhql3gQdq zUcoe8H&Fy(=Kw3~7y3So;{ylIy*uosS!QqEwYOwit+ZfFwZ$01E}^fVO{(=Rj7IVn z_#NLvTiaVo?g=mb9~`8}?4nE84u;EOuOJOUHG}#tpki9#!5W+DKf<1SH(mf^sdEQJ zhIUaiHXH?RvCoVezWbWrl117P+CYp#V&`qYmNh%kbkoG@LbLz#Q?eeGy5}Z|h#46> zVn21n9F5|ZxkJ-qf@jY z6B@Ybf3WLVc7r6R$h&zYwQ*?^v zdM#o{?}CQ}UKVqGECs#A+HPht)hq}%TX$^bUoJPUP%1+`6jA|U^BuuVw!nI}g*9Wf zk~Lw@ns~?-+G}QN&Fp6yzGCG}4$TO2?0YLBgzvtkM09kl{>-O!Hkih(JZR@sy!vyG z{d{9Cg0)wglr@g(``vz#w)JD&b)2`s=5I(sq!7 zf_LhcFvlvIq`?1z8s1l0u*Q>XutzqMV8I`z3U00!OxIa`&6;vzU71!9G*bJpTdzbV zc{)WG1Wa7?aBk{aWbE+;5ji%!?7$+?IK#^G=z_ofeRmAB2gs+k3v8_sVKaZZNacaI zBP0~#Qbxx#j&Bv4qR@97RP&}y?Nrh{mxjg$FFlyL!{Q)r`c0|*bB<&>#|Xu+(mHr7(#38??)`$b&UIkZa8(COmzwK^l_IseiRseRY%W z;AzKZwL5qkZ|w+P744@h0sO?z_ppmF(Nz7`6f#R?$;!Xh%;sn&Ye{x8axF~hL>L|; zX{K`Q%BJ}WW|T`o9}htO?~sz-ic)C7-@o2`>U%dn5AW5Io1I^-B;N6s?>pbwo?7yV zUc(*@F(_-C9_Ai#7#Q|UeU!h_CEHy_wE)T&A-+GTpj{rH|8155K7-uS00p5T&gP^T zadvfztTn4TMZNjST9$lP)&py-zex+wDdk997yFQ#LVffZgHdsmZR@!H@6=L)rk|#! zvm$4sDpLg-wSdrm%yjnMSAwZ3*usp>x1CR?+1PN>XHoo_Z#};pmq0BXIbE=GtigpxaH2t_;8zO@Joh`vBY`Ey`a~!koQD`z`<{?z~?eoVBFHeXJpebLJNdcAs33Hln~w75t8&A53RnXW|KIu*-+NI2eVbul7u#ERgfwcmeLm#VlGx zi7`+4^3tN&%l4fIWAo3rQkHoiww=L=@8=Tp??3EJwx9iB<>8-9S>yfAQeZj8&m<&# z|8OThXc{r}fxHhf9l8R%%lV?q^bQE9jKi}IaqV#Iv$AlR<1Nd{g=?QCt6^$WqTLd@ z@oS#VCUfrKIm$Raj?&Afj`fazZ@=_lowi`6HH!bN(q2ZI>&V0_mM@5k8tk!m^Zb}J zTg=}T&UMcm1raYj$)<~K^bfUUZ38Fdm$?u!$f9b!fEJyU5{XWV zrQC(wIBSI6L@nGzFID9X_l%v<_J_I*)5|$aN{pX3fQXF1-3W*5W)T@T0u(?-W8xCXUIjqZD@VF+^rC}D=|Y45j=5i7Y|HvVTR1O9$unZ4fV@?IdnTokru*Uo1c zpxf+Xo&DoIog094lhxYK)+T+&#Jp)XS?F{~vyE5L;tT1z3-i&ZK=Cd%j)>?i0wCBa z8^F11o<=an(opn=%5k2Tl!2Me=gv-vHf8G|?>r)!swDkp1W6DhRt7c$t6%?g94USB zbl-9BP>ZMcD@@vaMcrd(4tt3!)_iR2)d!1p=^@(A!>MoEpULI-d6K^{at4ilCfR`v zDWT9Igwe77fm!_!Sv=*S#Q^AXN3U?$BTE z7cJFWdX}M(RILBv`o?w}p6?o&Z*rcuQ}>*7B|DzovFK^ylJlHG=rZVfi|_FJw4ND$ z+qi|?;=SB)Ju6mSzfQ~8-c*oec8)6huGXv6425>p+Ze7(_q8fjjGB>72UDnYtcjwN z{UE@WaNXSrtftVhaU~t3Nb}9yjh^B5(>Vde4IM&q1d&;&TjI_olDN~^HMX0^`lJDK z;UR^l({ZNJAGX6{$6o~{WpRQ(-H!ioO+O0&F+^w<<-y-0>+@%>)G>ZGLiU{b*z<>ECn*^tS5gMz@b&bn1tUE{MA^#xyQbwn%K-qAl&?!2fyX8&Ee&Z`A za7SU2c8jvo$2uz>5#wpA*JkKx_^BV7X(k6ckt3k&EIe=X2i-IwB0%caPz8_(b|GtUa*Mt!rQNJ`h|9F+jsRFlX?x zGJl+hOeHdKtd!3PT4{C`Vwt<-^M%Ak`8jBb)|**v`Z8#i>9*=YZ$Baf zwZ%!A2Bm4Vw{@Ai!DhrT{U%#(Rhki|8-kvW95IZ`-Ww(x8Yx)*E)iG%kQ3nj^u-@K zDP>1pN_opnvlD0`hkq&+eJ2X3_0l5*Hv_=4&GYe8rk-PqPGwsGuKXH`+>|EvDbUfW zX9z1=?Bgg-uJ&xh<3-ySQL3V(8;oDuu$oM~X6K}ux$CgP@2_@V>^egczj#^w4-t3_ zXw~@Ngd~6Nm4COvS)a2jvY))4W6*Z3-MTMjeCndjnc5xJYd-EfCmUgSCCLWRdOm}K zbw3&}-c!Ha1;6nTw-w>YE}Lz(0{Ky++Xf$F1p=>5$G)^Ny4NY7aCB zVR9ak7KU^UQhRFULp(G$q!qhS!!%N704%Ghv(<)=+Qopq~YunI4Y zr~FbKRZoSM-&sWIDMy(GV1{##B}^+ca{~L1JqlraHWIH_~?=#2xt|#{N!1-khPPK{Glw2#XyYgYQ1$AX3L%M=aYSr1$fh8f*K#2y&~HpupS?c1MUL>m!}s zT4(`*RV>}O<@B>C_cKGST_c%NI47+Vv-J(VoL`JbQis#`_$XV_BK4sVP}GrNEPB93 zk>%c_x(I|p)PT1yy!JP;(;GzO2Pyf*tDG1$DoE~&=DNO1hZC)LD+d9_=YvPl+pdif221H&O)71Wg0SiO>I^^mS$A=%2m=iC^Zf{R|Z=am{ zK-Re^Kl;+r2>U3bAlE&0uSaHufgR5me#G7tqWW`uDmUHjRAIxsRXnT=V)Nk$ z6Rjp0^6{c49W>on?#W|k>Y9y1Fp&^QE+SmQp$mx~BObN}Z6?&ZnRND@Gbzh98HKQ&Qddd<4B#x@iAfxe?IPaQx0u<3r{5^CMm*TZ0aC^Z|A z;^qW|#-VKa$#7T45x5f#qCQ^+KkR@dg$5H_?IYaVPKF~^hX7Up(BP)*N&eoB(r21h zMyD0`HX~IzRr)pp#i$5bv5|U%S5o&};p1B%GCo?s;B~uocSY1)8L~E_$~o!6xggDp z_IkL~-u_@z?+8u6JZOXz#Z%@OKmtg$0mwTmm&7Ik=@LWt0>3(D1*{JsM=k;d@i7}+ z5nD53Pnd4>VqVs%06OR+FV5-fT{rqWLd@qpF7&VRl8asL>H0sEXD(;=hjqR??NWw# zv!kB`Bs?^_xMHhJFl|PNUa>~~0qF1P`(cTT(ilaJ*7VXBf5@|}PmSawdn`7Q>5L@R zIsxiY>l9Ue^5Hgmc2yjiE{pWn|Bc!id|M1IO^7(`>@VW=`VUY%Mxa%StgO=3Zld0~ zS08%>F=+6*ZioyhowG`U{aoX+4s^b0SF04J7=yp47ocgIYp*S5sm1@%Pbba3PRi;N z&GSW@S3F3r2#D^#QA&$iICiKR*tGGgEV}orf`y&xCs*F>{WdzS=!^^h#9f5=TS{1I zKPA_L%5T+b_Vgyt3R_u*`6mPk!68FApsYCf?r6m^|wnk6< zYL-Ou3BN%#ex`g(iya#qeHH3yTDdeecJ_UI0lnCeIgHk)A4I=J?RpyFn<-Gdv?*Pb z6|s1a-m;c>T8J`I3RN3p9Ph#-T|u~@u1ZJ~X9yg$Vv#?7f|G+~sBrQP?dfCCotGzg zd`!D43rw6G4eu-1(QKyzv5#XpNTf9A^<^oFiPJ6FF+R<*n{P9I!rQy#7z3O;K)m+X z-8M28vw!gzByX!Nh0~KuXWJ8nj_ok z-8z?L@YII1KQNoR*QDk|k_BE~&>olHJ=S?rumn-24IvM2cPzys1o6L>xxf&8#9;Y7 z-ktR1d^yG}y5r3mwLQ?vM*jASJ9TmzfnsE1&7a=a^G#J?*#X=tG_B1nC^DNY=9f!6 zupDz1?Y7SM26tz>2;)=zGpTm5=Qo(V)5?aBMT1#d&HUkGknP(hB{$AmYMO_S4TGyl9dje>xuLfR z3ragP#=n62meb%wZur>X1$sex8`{L0lvcp!F!=W%Q|l!O^1@IlUDW{|QuUISG+T0c zU7~K0Z+UrL;U=!QGjFQ`A8tRxd_2kSXtl%oi%zneSpYAnQ&O2)ZL=QP5q*#^U|<#N zUN$25za|$E33D^G*|t%4G6`lw*n{_}*FLXg$0Wco0;3QF*>3(E>p$^@G2-g;XTJ|D zNIa(JH+UKt^ra+U#^kp)4np1T zzHlb{H50^nd8OXM0^eK+l997#up`6}j&%<}eYom%FqGM-^{x$#aEP9{G_`ITP{-UI zYD3#qVdD12r$|u8_UwrG@MKqiK>>?kvmgRCdA_5HiO7k`?z7O*=jx4dbm;s@lZ zcewUpUHK$m^%OPdpxkGf)4rW0hMp&`a|(5TyOX>EjSc)=-h{@|$tS!w_mikK?jy#Y z>3!`dwYK7__}xtCqq4%iIPO6ZjLT6*WR)S!L&uhmJ(fWyt^Vw6 zim2pro~k|Be)ga!Vd(L28hixkW*DeGYY10D;HGmiEjD5*2vW*Pq!U?lN&-Qy?c-8! zV~K|ms;A%Is=s@B?eD8R>y}$oUyQ|dnUY=u6Nq9h_7Bs^^!ePF|+s%>3P3Z2Nf5DD`+QkVE*lwm58bGmtS zl|cQ0j_ecpkO1r}hx$0rEXVBRY^P;`b zRBPk4-OCG_IcQ1z70WX1Gc$HLkn!3co>wgd3LfsHw_v1h^<7|Be+J7>jX!DB)iSY!vOCxXDtt-sx{X0v`NfAi+ijq32L4IXfduGDPn z&hvR?3V65#FHmvS(i_8=KE4fH^=doaXm3tMxqmH(&qK*)dPFgP?_MQ!_mTZ_AIfF@2(L9!D%LK)yy}&B(HVCy_MN+)+!896viDGj%$YJFa3% zi9mw{!?N)hHqczKs{qu{2njLLkIc2ND_a98ONtPdOuN=cluOzCh5KF|1*kswRSJIV zLczeltX4gsZ2{4+H^kF`nUS*Ym}yoBcnLM+$SPsNo{+=NdQRt>f|Q?8D3{e>w{uSe{cA3z?fxCe2zs)2P4G^jLEa z-jN#o+4y?cJxbE+dfIdcJ4?U9DGUSIh_PJ*L zTtrU*5<|PpN~<)ugC{XZ&W~ohk_=%AL{%MCHTjB-m%WI1t44{dmN%e{pn$^!P3LlR zm}UDFASv8s!z~SuU$VLyE7ZG z7yW@mtnbkWgx8@}$Q8j`vO2GF=IhYREAE^@hCKOi#+WzbyX^V$OW4w~aJ>CVIaZTU z6SnmGUB=#B%Oju2Q`iKEMdqVl6VAxBHx9S+7+Q`ZIc95Y+gQ`dvyZW^&(9(CgWN(_ zM4nqo`T^OJrvKoI1fnsf+_p`*tCuY!S9*9KAF23;VaF5HEFAkt!5MP%K-4&KORo zw@M&vH6PMAi_RuPaLfqx@ayefgsGI38|L0AhzAY=v|wrN@US;zi#d(EB{2oU2da^2 z=@pifSupsQPOje%_t86qQ44@3uKWnln2Fsw7h3LSOCPy;Pl;h+y8aF{6?(L|`Mj9l3!;ltcytG69HLLH zQ215tYr|S>>$uz69AJ42GKn8Lmz!|IZah8d!#%6pt^Skg_=-HhPMH~Bv#wB~F8)Uj zyBG78FPGuPF}b_h@VYks&0ywm3{Pp>00zv1#nL=D72Hy6MxTn`>jT5Ka)2phC$^L5 zL<+jSi9&fCNZ|~XiY#4R8yhR(hkdM2{Dbq1^Udn5z5l?iuX?@=K;#0qjxQ>`<4Se# zA_@0v>z3f}^mmTl*m&6)VL~y&?qr|@)QxrCrE5HgdqHOQE%LDE+Zb>cj*&=l*hHw4 zVmz0wU(ifM+-D-``uERC^;15m4^#^Hsm~>1TP}M#X;Ze`RA`5@$NbgM?42zeOi?E0 z!ckeakPv0Am+NKdNV?RvjxH~4=U&ohXh`4zDJYNZ>i^P9ZPI6%A?{{^ARZ>9b>cp5 zD=>%6^{VqOr{{FYEBIB%#I_L=o(Pkt&O_hQfxgt=e znyP3W*moa2KeL=~YoH@!?-__jS>?b%GY2>elyuRt^O~F-Y~YrKyv@I^@JqFW&IYc0 zbm*6emusN(mF=iqZXNU_Rrj*1=OppJ0`it$gP#AqRp*{Ek@2Q3^dcE)i(9W?eLK2N zR7Hy|i}XDsQXnSP(f8WhdS&E5Jo}?GTQo(-@~T`*H)T(maxD!mIO1v#JBh;6whCIN znvC7cJ3(M0a+7z4V*1sqB&87bLw^CI=nW-wgnwWLVXM_igc5T6rPlOr&B^05_Qz4a zVx)xg7Tsh|r}I=$NAW1go#Z3UY9op%-`GVoOAr*MlsdlQhHZL9CU zeq9YBnbPgJ=U`B?8@cjct;XiL`kK?6;{6!cNwx1&3WIZaS09R-B2XNLr(2-M8@ei8 zL7Q@el5d#~_)g&2Wk~PkYi|c;q~m>#jfA}PA~$vMuuLh)xSfv9+x?i0=c8on71=y~ zRki(kafRW%CUlU1N~E@k0{jt(qBGdm-cpDMmx&~pl82vI}WBPlP6M zkCHxcJLhfiTS;o-Xi%Bmq4cc#DHq>*p8tX2VH>p1z+rg9|GpF14Oyr5@GxnTgWOsRvl`g+RBn&Rj!`8A?BL%~v1ENR1s`IN3Vu;-H0WJ+bC< zn|*?}i?qGAjdcPV%t)uRn@GyA8ynrbHvY&*-Ef$|)tM}MI9SwSz3A>#lMP6&ZQuj1 zGBCaDScyOfS9k4h5iz=sood;GRk#G62RUQR)+~&1QCd|-O0h}q1fMFkRSMAvtGF^+MORFX%qcKs zktnIST(&dU>zF82f<-DT6;ZvF5tzJ4DQnAck1&*7ij20XCMFbD3bTG2{E>o)UTBhE zu4+9H{>?@H1+V{6E&}7kjfZaQ1j@C)4q6&S+n;&S8vBMPLVSdi3zw>$Zo@raKI zq6)hrh!OY32;Oiq!qpzOh4w7?IaGxJjFQsv6+=_qumW)}6e~5J$hb6f67^MX$q-}g z3?|qDGJ}w!@(EY~YlqW+XBuJY19RNOJGXb^r;IZ7+QczLxnHG}I*c63(Q`y+mqHYC z(0elk#rv7e7)d2Z@gDio8<2I_e>hD^j4#qD=HGI?23gQRJOJ8<2mY~+UouZk&(1vt z7eAgjgM381^sD}^FqyOSku8&fx5ZFn@0%ND>$;DEd*vb=lYmre^oG zb9}23FvOr7q=_XWr&=O2##P>oa9K72@dqI^RCth|^gtBxg&!Av8;xVFSv$fvQ39K< z%;nmtpvYDxM#_~F`94R86(l0=XRDJOW&AU_1gCJKtR-E?+UneT@m5gj53kGG85S{t?bvx zzKV1$HW4xGzk6`an+*to`d=BsdOyU>%?MZK%uSJ29`?>W$y|_!{A__j4;j_8Gi^b~ z@Gmc_se(Ox^kDQ^e+u2->X)K5f2NT`H3=Kb4nwHn4}*btGQiZe-^8ffkk@yCWZm2V zhiwsART|>QXNHoeXNJ6&m0#hhD|aPjUF$lEE{s66-J??^SPPb zF=a>Tlw0$$58BLskYOPf=7O&F2f|u~-^%-FJS}PW-Hn_k3s&s&n8~5yZlFc-I^wMd zpC7*+>U+#CT*%7S3;-ttWzT!{L|uCKH)MD~X=`GxL+AJ{E+Ns= zi6J6R&{z2bG3A$RkfKx_5KQAo8Ee9<#rE<8IQv!z)1E4l`q#GcJwUCHb_?z^I3EQe zkTc0A#rNeL_IR3M!J8Kc6l}(<*b)&w3!+qma@WJ36G(oQ1kQZnv|OF=IXV z+Pwj4ClX9>_z|a^xkOFz_{p@0(F%F*Z3t`X7A=v5(Ja*lL{tZ=fnnB7X!(WpRh#2e zk_+oR3Rg25h3oZxzAaEh&_0JoU?W{-hwf)d%1Z@N16xsmx$H%<+;>^_ zZ98(46hhq&Olyi2o<+e^exgpcZPZIY0DH6%&cuS;RW9GDVxL78z&LLKmLD_cTfXLC z!6{|zWD`wnXO45RRJ5s^g+}SNZ2vm`Bp3W75=ot5VjERdMiLzhfo5`ZvG>^^;{&~z z2fNKJ2t)kBK1K0vsY!<~X0Dz$9%Ez=!w85_Y$JNFx{oXEbT`d`WYt6)dun^m$6FF9 z5RjVia4XKqK>xn23;~QNbmMGY z=klG|3G}tkE-&MBszeKVfoVJ-Rl~79w0!YAOwHoBy}fV88n0Uz^7k$M{oS8!>yE5P zf(Ovz@-_GqYLgcUNr|bfVVvTg!%uvvm;^Bx=_-&iL*yMrbtk=_Ak*}tHpky9?!c}% zzUzjgC3krEg#HSuZvL{~nr6?UTx|MYM*(sFg9-)bEx+OdnZRU!WCA%Xa)o)zC{k)9 z&YCBV_r}tnwKn&B)6RJ3ugLZw$+Z!~nc#DyRCJ_txDyVPfQTwS;6*AvRuY2E+x^3& zRNYs8!J;`bY#IYF9ROlG2f*ca9IR$c+ge{V)RI}H&w1%nWw>J@;zU#C`^tizz`YCO z?O#{;SZ)0kn~N?QH~X}m$Gu$R^y1fcw|e)Mu5xwROAE58h=a-7iano(K-r7a8B@H3 zs56GU(>zZS)S@68@gpEm{=9OQ-+SGf8g z74i#2m{iHzU{X`^6FD)LGX(I0B9F|yRjs+K$MO-8$s|exDKBJeP5b+;@yQOrvN!N05ZL11q)Kye^Z33HjWh<96#{z#l5h?7rE5i#3d^pAlQ27XMzCl zb9seeKoh*eIJ=OJb@hG^493EvG{Plr#xDya<7%>NzFKpvI;N`)E8w3`ec?wQMr%Z;I%0%B`L&zea{B!L)pRT~l z{38m*m6Q1Iz%3Bbi9qO!KO}IJr1^JVB3(E*Jt(pdkteqmMj_`$RRHe{5L~mv48zJW zf7jJL%U)-D*h+9TxY9Ujcm7oVp8vWRz}W00+tupQGS0q~!=KkQH9_V;-s<8TFaO~a zLhtjeJ|s}~GkmmUn5R}pd5MGF<@pO%0|aXLkot$J2rom(kyR=qj7bkm~y zR5w@Dp8-!F_upLNgDG*Q!7$+uN67G;14{xiV)t-4-4Ne^4)P8{X~=A*W!cF|+>vd? zzP`tV&SKx`Cka8*`HPR(*`<}7f+Lq zl+qYi-IP%p5Rr^3g43A&CF{;*EjeX_VnUbaWGxA*BtcXOXUZ_bDh#o3mmvF&yxF%( zPDMWnG4HUFgy+FH?P?)zmP<@&Wt21ROvP1$Z`4r|Ixi!yP(6tcy&{;oFrS{0VGqN@ zrBvs%OBPQ$dDVqt60#S|Z-4y55-dSQ77Jc;7ze3PC|oGPW%x*^SICugE}zSo;T9@p za%%vxe8X?=18LZ&4SowwhHkt#2)K%uOa7Nds32`n|F!guhiliQM{K41zUpwD@7a&j zgRhc0{FnU1aUC|QU8(O8e#4}ML;SweAb~SdMPs-%C7x2;=T3!dg#fxaxT_J_-v1ak zRT@{JFr)#2uZLFq!2w7vHxJK_4BP&t$)eUFZq!4aNJ79h6(Q;-y;X0fwvRP+k*B}ejjTJ2v_dkT>9 zlOv8!GV<{8r(1%Wo*~op8lHGb-;Oo}>6r#+Ngz4FlEmJoaqB*wm%_9?9Y?JQ;a`%9 ztDd!XM4rU%l~z+zSom}U#*}liVRB+ z>hnGOJy`!>WNSywXZVOA1;P--tg${)x1(s z)^InHSTPL+^EbrJpT?^7wl*@4qfpz6v#!bHy^H-93^k?kfJtm0g?;@cC(>zMeSB5m zF7Mxu0@QdxVeA}?Ey)QrURJ!eLEpuk_NQLD@WNQqdyt<+3*ML(VPBD=jpg+&@%m;FG_l~$EA@b$l1;GgUc zGS}*aV?y}@uMF*dld)+Gc5@E6{)*_^ryWzomZ`9Z2qe`%!WGoR!y-*)eV5nM3%^EA zCb5V4lYAnh90`G@axr3RsT_n{hb$+G3M2A_&qJgF>$veNlw1eT!`{HT<Mc>vsky_9Qc~X9 z+{q;OEkiyj3rwjN6u1iRBwSST;6TURm2f-( zuDkkm<7?6UX&vjmO$)1D;OL=CO(kB@{M!Pb`NVY9NL^z|myJ&z7!-Ire${G+?&Y4( zb{8QJ{hnYt0i3q&Dvbp3K%4HO@H6xQ8xDqaOb#~rnBMKdr5k1-{1qJnSLq9)>bXFE zoSoPnpk~tOEmG)F=!;DG2oifofkdu{x^zQ@7pAvQhRK=~{zv#I-Aip)X^dnxJ)#bs zh28Z_M@FoDJBaPz+Mdykm1qAx2YnoY>Qe>efWv%be^Pfz<&DUk9){-VlgOz?De^H| zIMrJKA;CE*!a+do=nwsa&2rU#=_N3WZeLKo6gmL4?y^<%g$DZM%DcC^A&1umX`Qp6 z>`e%fzx-a${2WF1Cu971w=IjFQfKNb+O1?r#w1R`EN?9gvY`A<1Vr}Y&T^FYS-l6@ zg+XjwFUnPQf}IgM}2Dfc0Br{3_guf)JtrSw(Pc6HCF2_52xBjkU~T; z#tJ)-=6ps4)^#MA`*z!+7Yu9)Q6Cmg1v9GZI%q99m5GID@s*;c{H7fLTHfrXkt`B> ztP37%Z-95gF@_YA{pGXkZ%+=XO|XK(^+IB|SB|iKW_H0`4OnPR?ee8(A=TODU-E@j z&%cjbvY&HpO?qDct?C>-)Xm=OoxqXzPwUqPIn*xc%bM!UeI4Q3MnA2 z?Nrg}cM>8w#FSV;1zgctNg;6~!v~Pz0|2i~SU!YN*^7iWUEKD&g-w9{|@P?h^u&7ncKBlR2f9-JFB5@)yWlQ(19Ean)n z0mXoduZb<2qI^&-c^U#(p-aKL)XJ1XY=AO12A4K-teu}+4Chz{WcV^ zO{@6=&;+umO53!{;?%+l_fzY{l^=T;7w4*;NTRvC$@rF{yQl7!38ibf79E8tys-vT z(ta{+2l6<3iL3mYK{f%XJ(&8P&GA3+thnKwW_#K;whlIUp}zj@HjdLmF~#R;i#88K;I{gNn}>C92LvMzg z^cnccdf~)(kE!&K>>#aBzbJmroq2l%fP6du5DcQFDu5%zwzT0;&M=Q2)-XO&$rIY; zC}OOsMg+ou9b@KMMe$ZCwEUU0ohJ#co0{@v{Zq~;vUsOM!Tqr26R`ZGJN{4eRO@T& zsYkQjIvyU)%h6lAFTThSf#~yO$D0~gpPvpkUkU&P*$5DBL{B_cHO^w=h=9JR_WOa* zpUm2s!`y_X)s{Bsr>?sXs0*FlBQTa{uotWcm3~!GKcC|6DV54cF+Z1;Ns9MNQzeIl z7t%s#Fy*S4O?2ps-#FpAXWT(1r}cgsG3i)G z^AF0O1rk~wiN^tBfTqD;lM zg>ck*Y75;7`LS2OeN<+`eWPHM0XNxTqz;8Ub5h~tvKjT%26`E7fQa#Di9}s+l&e2{ z?kI3lnLnHgFb0eqjs#*xW(yj4f>1@Z949n^VOyGszEwlr%x-LLJXF(n0*3*rTnVIU zYLwBaVkrMfYPxh)^Z|hIzZntn<*{qhw!5qT^gO%iMM7fw)t?qG#{y=^m73X2{$5=j z{3H+1ls$CZeSxh5p0Eot_q#{LicTg?W#}QF?BA5wUF56W`X>1;ZUdoWdT%=^E@|)R zTOw4|`W?AR&mR~F?tl1zIDnCJ-tzp)zeSA9slp^Or<}x&SC>Wj+YD~jNc$UCJlnf~ z9mUuxt;gOI*j6&98dSNA^8Ga21Nn%14;~;{{Eq_KP1b;gxxG@sI^Cm66P_PdMH`M;wgQ&!is>2lu$Kxkx2|MT!?CBSfswiBoU+}Fn+zgr&~9lRo9r>^7AT^CQ+ZW33V7F0~|AcN}R>KRJV{)Zf8 zp%9>^#M5tiHkI;&9!Qr zc#Kn?l#0nONL2bA#-tsjTu#g=MDLnT%ed3Awd?dPJdw+<2l((i6Y2xXtgx6orY75a zGK_5{2X63j)o%otdR69PyDL;YEFGSsw2v5g439y7{SOSb&QibmJ29^Qv_$e-y zfW9<7{Q8fOm&Tf@x?7lpv6;5j@fLolIrmKB%4B00eY_|cAFB@J)&(i z9&Eeuck$VNg139>J;y>;Y#XeUXX(>FWaPRz+p&+c7;k!l2M5^{YZqX^SfH?{JBgfW zt$n!gKK9fJ|7&4J4EcOR!z+}T;X`e)`5xTrG7`U@D*X8TY5%dD814@{C+OFs4Ru z4?^3)t+P=iMJTNdjg7ExHpbjua09COPIrSdfV^*K{l0idftodX`{M6LmImJLxNyU9 z0td)4&0ampdvU8;uzb>w6RUn+$3fi)Hty=mKanFL07Wb>QQo53b)AztG^JC}ZijsI zcxqj(m%oTlY-24-qh36rlxDZy4_Hr@zE<;mj}5WaY;PV+=;8qH<*suR`3)eC1kZ%E z*fRhjn8=Es+|2s+u8FWk)yfF5kpP3lqu}G=JGq<;X^$;CRMPEPM5ZG}Cs}viBFJ#* zKr_Llj#vT614!r?XU({gDNCd(+{?0nI8jANX**|j00QGhVP$B)T2BE7yHLge2r?1# z6iVGG!_T|HztIC5PCX2~5%JO(yC^i*=LM)^l57ZwO7`9lmw52d+hl8f-Z?7hP?}$rAzox0qP?pR%K;7K@Yg%N z_ydp{cH;((K6*o=uqX%{_Ws2tW%vHz)O4Ua+xo!^nk~3&W4RzS=T*8Tvw0njh*#%V{*gY#V zMRgF)aal;zMPUb-fxx#iEA#xYJSzcOLS?|2d#56lSm8QYyA`0(8=b0XX7HC^L1(82 z0VKeY|3}!{$1}bE|Kp{U6m@c$%Q{`)ND7myQJgp@mFtU!O{uUYq=n5$g)UrlDje5I znyXpGTyIm#Qn|Lxgv*eO3{J8M4Z3yl>gt&T0J=;wN6eyd2M0wRXLT_f>}5}{Q*y1ZXj&c+Jrq47q%wwl37&^L@_^w?H41xbSQ_Q6Tpu|kHLdQkJ zN!AkVnbL)}e8GgeD5Wz5L*WAbkX2kN4h38QMi0G)8a`kexE+?Hj*3FT6~MUgg74YE z2OuqO_bL6q`d~*!Wu;>UKYsY_+8d}d*3EQVwF;2>=%pM@xQSAOOj=>_a*lhP@9%tl z0)v{uRbc|xDH!ic{xAfx=1nO$FaayccS4y=^?}dfG%l5&a7AICT;`;_6a+PD=)sNw_cx5@iD$coFvpackUHGG4T1T?-PB z4okdiL6HJ4EDp55y%ajbU-<69SptgqtepCPY#;uWnZ)aFTNK(XwFG419-YGey6x;f z--8OS@YNaRb$d6FO!}*pWPJ>|||1{+^HeoV#Y6br#5 z2z}`QTdOi+ypPEUZ@?1KROAI4G@6?|Ua!)On5M-Z@LEoM(=*Q$)$x*RIv2As20V3y zkQAqm5Evl`oL+hDs5*s4RCZ5U_>c*SApGjcvM;cyi&@dsg(P zTZxK_2k>azck`KYN$^RiLUX?*?kSLZd9EoM%dD9Dqf;s;=Qt(p45?M2eNpo%VAfvW zS&CS^pqFeUtl<5c*4+>R#2^GhLU`m1ALOpeeJvp)uQv}a1^J)w%BB>m?q-WLVFF-9bdqfD=%gPt_(uv86+p4TI&Kl zuCViMg%@h#TvPE*=HzlL+4>LF$YByjR5!Y+RqMUDs&B!C0juX53!?w2i8MP47}eHL z*75&4QXlm3CKJgj?gaUd|~uP_nQerZx%`BR#BOa{5ir)JgB!Lr1UbJ(SVsq5&=QlY(BGn>Wf7L*@2WqUNSM21})+Qo` zQB*4d<)J4|_JDs{`yd?VbMzi<@2v-d9YsLXhUH_uDJU7Ma~|h2{}g5~3Y8A)ICQ*| zrgQCu%HX?YxlDs;Lt-m&d386MecLswu_$9QkK`x-rctpG*vJChoEEZ!MEc=p;cU*# zc`~=@G`&XypmFSZ4cLY@>!!L)U|Svv3=II&d3fM>x(=bU1&(KmE5hGf(DPC)k~^n^ zPsZV2_6%b|a@Nz+rGntedPFCqU}2b(iC|aPCT7()CcY;z!ON65E~PcCiQM{!dTSxp zZv+soQhoOBDzGIu=kK*C==`2*>2W~Q&2n}2YAZ=F(Jg&*RBL2*SpGw&lH0edlwNuO zzUkShe@@*01@UKkU5`Fo=zfz4{p~~bt=oR3HGo2e=c^^q>5A_2^rbbk{6k|Gl~B&Ut%&O<&{%?)dOzI^x19|98wbh<{)c>@=N`5AU0$LS;(CUu$2~2&VQH z0n=4(Ie|L^VmeZ_eV0o<4}vd_ELuQ~V@V4ng3nPLZA!Xe&<2g4?3wRwuv)jn=Ovyt zK{(~p`76tsg5W|^5wk0E$^hA4RUXIz7QEi2=|IJecr9Rap?0g`N{@}%hMLQ`ZM2ew zSuRDibG=|TmF3&EtYJX-#sSV=m1&Yp2bJl3ueA8*H#636ztpcD(0s1E23dL=Gn`c2{5!>|!(fMQu z{krYxfK~uM-gyjaCi;9H8f%e)-ocElET;3@BN4lP?|U7K;0_3)FrrKKsTc~{yfUH* zhK)!@Ml)lh$-FI}M9F=D?cpQkp{p^X5z6Roa_Y1W6jvgAg_R&I9>c#-87wIt^%B*< zW>`_3Lh&R5#a|j8jGJ*{*Gy=kvDN{cdNr0c;Ub=i5n ze&aps1qCaiP_yN9$@FH*rX`yuk@ z8WUXIR|{GUG6ZN`#X$mM*R7i#+b{fX-mdd9G4;JLZGrxV-q)E3vXBf~P&T2?cF;(! z%$=tgSc&S_phtU{A-F^FY2o>5A1Z(y{tD@YuOsGSNC9sq!KWwEst_Jb93RHP%HB6= zmfk(zn(Y%Z?~9uTV_G?u5+Ea1XFYoA6*RYYy*a7tv_kyl=|p-k&b zK?22cdW#nz1r)ckW+&a`W;(RBWmB%gbLYB;(HmrM&Yh1^?(9{gopF5~RN@2JK5e1TY-$^9IX@E*ZW| zyy>*s1KD`yDWUiLWj_x7r;2XExprgDG~)#$(WQZL$oa+jEO7GCoB{B8T$WA17L2Bi zYHT$V^7}bmFeyaLm~%>h#`)Fv~|Ctna+CpEH&2Q+^&fXQp`xpi;$7acY@p+a4pi$OQyFfacYvqFwnDTcZ&|GWwAg?k3-ct@jV>mtLG}`ah zBaVzngJ`k|{2}-9)*4~WRfJ7m5y7!@Of_}T2PXQAjNwx{GZ}{BkX%|ZjF$Pl7CRF- zT@QGfhF;+Y!%Qh37JqrCcXGnQu*!&6Yvfr;C6m|w*y>W5RY z{-5t!N~R>d3@+xGLDwkc#zh2m6AUxSNot9!zSlcVm?qF#Sea7DvY~81a)~T?r>z}3 zs}L$Czv2z@A)K*@Z`so`omrZMS|6G4&k?_GkPmw!wMo^){{^+-=*^q^XxD!%{q^1+ z2_I#;eBb$)t$wth?2#)c^Wp&NrDz?DpCe%A2L?%B{n!oI;)BPn5oRGcCyZK5=}Shw zMck=^!OLuD4S-?MJWDWxW2WHcrWdLz$z&FR_#$&o>|M1GyYJQm%yh!JQ|nHkJNxRe zApD1%-^8@5(~PGF&~5!fb#rAPXt}@*1vU4Sqlk4tRp?@8)789aay6}C|455(l}S$o zXGs-*S~#Q%B>%LFBfZdGB3BHKWFE>M@Zt1EPG{39Fia31#9LL)OeI@m#*6;gJB*Rc z-4(R*RD$lEL+VJ_k_fYysgIw^6J|Z|!D*f18!)y5RA>DKyD$lpexSw6e2E<=eC?y2 zwyRDe!VgCy;+?c)K4eSAxUpPSlJJ2IFTQ1?^#W-lUC$f@A_#lckCXy_ox* zlx`5K3b)OjZS&+*(A7|IpL`g%!O`*@yq5LA0+lEgzL(=_Az=7RkAaLFM+?gK-A zl?3!mIU4Azc}o|Aai_S{*I)^Are1A~G}BX3AAT>_}^5xZBia=86GcA(ssJ?pDf zvH`&H2Z+6g1N34pFP~jJG^0ddX)?M)&6w$VuQY;BSTUwQ;V*;&{RRt*=@B4vxHp^;EFcOrBITl| z<8*3f;s9Mxw3C=-L0KY_p;B^=pZ639=pl?7X%{>d#+tYo)y9cFOPm=H{Ot|L3Y%C}ZY5 zhCP&iA-wq*g<`8|-5I|Db4glR!`j7rxB4qx>AhBDDAlW|ri0|q53Iwx2DKG#pVlKq zYFpM^H(^w+9>#17g@l9)*9DW~B;`qi$D(6V{m1V{IA;yFQa#1%0prjj*R4u?BA zM9l7ka_f`@J4j`%^!N1LwEdly=;U`7tEHUF=+NOR#h$Mk(4vkuU;6zt(GvAc@$-C! z+oW29LrmV~RD_S$e?Qv-ZEg;t2}COS`UST(SMj%|Ac>^D2ImR4l~N zp#RZT`NPQx7FK!Jw`<|H-Fzi0`!ML-^}TLVVEN|i>zdh?RfZe&6<^JY>TKe$J0037 zXnsRt0U^Jq%fj*K4+TZBYBOA}BOY3tN)d12c(D9k^n5?6wc`X?2sM@=)pEVtP~*kN zXy5l?hAUI`ju&ijT6ojVd9WCn#FxqK_#AiqJ;d!Aq5md5? zZ;|**K76FStTs3#Rx;$4V_DgTL5e9em?YtqQ?2g>j;(r?Hw3I(*Is!!S!ZpAVW!V5 z{-n@n;SgmCSC@qaSHJQqn{#`ykpKMh+624_WK5rGHP>7vUcD9nS8D{!tr@;v#vf;& z7V*Ph@%$68|6chYlgjeHp=LRJP?7J!KB9Q7TeaLPW!>dsOxr$2W+U*E*=Tx>m~My;*HO@Tf_gogGjQ7IOE4I6ze zCyX!RAb!~A&RMBTwJw{D-sF?)v%qf#UE}aaIXU?AE9ah@1;%iNv}H!*KHcJ#nQ}p~ z=(Qk^yOv68J-EJPu;s3U(5({MPr6c;yNrek?;j3KY0d|ug@gmZ5B~I6c_^llb7>Ku zN!BO2O@Gc59L2ez9BLla(v$o>4|HFSh{%q;1BhMpD7Pnhk4m)o?T4H~6dy1#bkF=UsI&9Xn zpY?PL)#T_@nmImK;(^ht_-L{5n#QZJx3AVJXnQ?+Vu}|`ZA-DKBr~o3eLF}!r*#Tq2yv1X5`{rTvxtj1+f_He)QdsR2GPZz^oI=GMK4azc#Y}X zhJ_eVLjqgJno(=p=P$5NW^}T9pJGS;OP8TrqBevJpcuL*g{$G#r?SuKR z*XZ|No21m5UoxIvXJ05Qtqk8KL;dN>{Y?cWWOU~B?Jb}_+T>t=ZX=cvy*pn;b{< z2|qS8;azQbms1#-Y!kc)o(lP;OI)pjtGco^$U4Ov@9#6M^(-zpZ$5RNTuH^|)fQ;Y z%z@xILw@e%j{eZy{2tPXljwk-Tfw-v`a~6S1`&rjfIEtwBcI=ak0M9q467c?@i&gUDur1ZnlfAL!kFfV3}T9}#(q`W zRk!xi*m&3S041&~313;a^!U||_b|X`kLke9kS?}ZehXrJ`sQuTkb2F)AO9_=tzVm< ztava;E=e7}tN6^LLS^qToAx2p*EF~{@Fpl34pt*X4A&~{f6?vHShFyuKV7ce+}re`FYN943Mk0h`l{39 z;z3oVJkh2Nan(9pVs%GU9*?AS|FX?VP)*0n?ZTz=Wkl-CT2Ur#hX|D?yfjt&5%KVJ zfrzbvGmbcAGQZT~1ylE{8pL4dn*^)=qa5aY7wKiAJvYe^%>!~1Raaw;2VAcLcMk&OC<{KFyfs@L5X*@k5E zxUbZ$A*BW7IxtKmKTc>CQw+BaLmF$ROaOrnaMpRRyj1*_q%xC9e>j!{NqG00OS($1 zzkz&0apyJA56PFmeI)-hZttvXQfp)n_a(N-m6awuTwY(|`siiEYv(ssqhUfflyA8g zJy2-j?&5<*Q!rGigR+mz!ipnI&W71CgSFKN(Me+}8(ms$9WYf7;+g!KNZ!WAtvi$TkPT|e;sPk!Q(bC~afyP=B% zN9Pqkx`hfZfpPSXt_Dt=P6ScIy_A=FK-QE%D^pt3NWf01(^+Z*8NyP^Y^ggWw6s7} z{LRxvpT&Rhg<(5}+oh~-h zK6V6PJ$M(eOO1D$UeWMz;0QUScl_#*ki2j?6{z5ZWG;JTXq?_;<0!3YA0EcmJzjL*FqKT( ztQ--f?}lmM<}y)V(_eW3Zx7ozkuj?DEt>uYtK1~vNANdX@7wSEzqm@3l0Vufr7OGh z+`7%O4|V%T{T=icKP3dN(=F?DUVK7zD@=iSzlpH)E_`&0Nx7FG7|HfRs>$aD;c)ky z<6cdrE&CkZ#(NOxo71U{<~2~RTRdlAcz&g|=*g(7Fl&bmdpFV_@@Ga*ZXr(Qy4r9Z zLAcqITeh1odo^H(oy6l1E9R^RPYamZ1XG;|Y5{cjh*mH!A;rdmGFIEe7Yl*h^m^;=);s^jK*%fpuRPTw5$yeju%qSJciM)555m>20$!It^-tO(`5 zg}&4NOOnqH^@+RxlU@9c0{;1DbAzm|@94ARdN;?`TvzUq-BsdQM6EK+7KamWIIhGl zs?i6^sP*#cDG46KK448qH#))otm^BvJE?)95>F!Jr8+g(%-t=5#~2@+2uzdkpl}2& zYn65}OuOc$NHM8W2l-Tu@H%^D%CsLP>7{NvjLia8VMTQUeM$BM4i(0F7`r<>7Ux2T z6(kA!!@Q>3rq)HKaVZE^z%MCN)GP&oB^qgXG!Yo8?=?*=1y`l6FuvSekZIey*9 ziddAn&3!x~lk}+eUvnPf6tb zVXsKxr8=%RYg^y*je%)OSt-fn5qBHq%eT)cHI|yncXF1DDuJ90C5Nw?c>r1SPF!7? zOy+nDHQ06D^v&PQkC1C5b(!c;bX{_%l0yS&@I~x55UC_r%SWibq5;FL82<6^`x86WWZC&iUrdt=p`8jlTBdwGOvdhR=Xy)4IDMJ1h7(ThI8>6U2BTXL4rJE# zn*7_M`3I5k`G&N0YNo22<6bZE7^1IgiIjkoA#!t8cN{$&pat_kxgl+ zD(l9PEoXztD0X`5%t)i9G~Pr8%G-ii(HW%Z7$Zj}51HA9AzEOjHbk2VY|gCQw}-x& zEe*I%#y$CeJM+(^;xE=6MdA)Vcp6E*=>3K#d;E1*Z?^Sc@}D;zLmf`=x0nd zm)0Z?)y@4_PkE&so`T{ILbkn*zkg!QYq3&Bcedk8tXU)n5}YEd6;OcX7Dbfw!)X$fUQD?8@%XJjpz{^T41389d-NVkWBajF}aCL zQBLbSZhkdYc5(6u>kGf*OUN7Nd&F7(;=xLOV#$ZnCDtYC+O-xXPU#^4i_r{W} zt!oTB)zjW^K)zTT%NVyobW=xmkrL@Lc@Xq<808T+V#pZkXLtlkXhdxUFZQBlcEl8# znkL6|v+%rpht*bkH7&}p;czNA>ah(7THEi)^9nds!D39Yk;{rR$w{^J|f3L0)YrN0cIV$xCd}$hvn=R7Yb8MT=}K z@7#4G3nz|C^OFloClM>`;%ly3VChoFpjDmSEZ0zK-warA>%bf;x^k=>$fD3*FlG+a zuARy+#%y6l#$jNVlof)S;A%)|py|#}u_$Co#V17srSh27Ozhy@S6&UUk4SYI$_A$s zx+JV-pN-3N+w4V1r9e>y1P>U~t3^vVg*`Lv-EV#MckcHohd(ejRI$nX;on&J-^d|Q zRT-4=1#ezo{c5}HyK}bN&2`FlylyG4lKB} zdRgcNr@^?Vo*>-8bX0T{*g!vrMuV?Q=+scukCtcAS*|wSZm#I5mxqfMmUr| zN0uceR;CL)H}9;61?T0{@gb+(e)^9VzziN;3Tzk=@eGn@0B1IrBx%YREaw*&B5WoU zJ3Zp2b)a{ne7RI7qRE0!3AF=}%9rH#v12gREQp`YWSb47e8O;~7dO6|?}UVyG;JvE zN^h#R+3>Ba`A>kWH~~=Ncf)3z|LsV5?&^{~Y_{B8cK1xWCoZF0==sYVWbEUU`uiWF zYzDB(2VDGLMOa$>K(7e<`DvQnkMeV`Nf5`763+nD!CS173-1^n+%?DBe}b0xESug7 zFx`J{Yr=&obwQB;TEuXOz|dyHbA*<}3>rH-J6&jTPDS8(*_9|fzwXEkp74-g8K5?R zW)g~#{qPUZ3xYnS;W2to6rI`@1ki<>CL`#%hBwey?vf9J&&7MN0E!k2v$j?~8F zkxMFo@Yp`G3m+`oB^FnTop48f>x>9`>dpm~#ZKja4cCk~^2U>ZE@`Y?j@E5&v;ZYrjDBdc}Mi@t{+dONfY>rSO=Q=7Y;t}%0!{+TE$|_qz5{D&C{9<0*``hGF!1?u7r0b#m=~RTQXat>;6(d z_%nF%ciI@|b95q};noI^${RBN$31^>oQHpD zF#aEYBxE(SN!ji9{g40Tl#hOVlg*zcT7CN1*DbwKGF8eKmnpb~d!;cJz{PFL=TE9;uIC}+^y5o!1tEH_bLUvZbK`BU$T zBif6Irt`;1Wk~A6k&g!L*xn*bF=Rjz6n35zJ~Zy#n|( zJ?2@Q`j`~iU4Z!JHNa^kGI4|bT_nG4joMG6eKbbeYB5D z2rzZG4;HdDuSisOj>~kNKnaDlfXwYjvNqH$1|0L1Ie@ru6eOm6%R2sJ7V(51>ViTB zyQIH|);IUO>#yZMb>D1UKaWaOH*wgv|6^Ao&8Ivv=ve&i=htqFswa?n`8%KPw4Xv` z_8=zoJRmvtdd@~uINs~gp4mEU%Y%;vgs|I!r)o(FS5Ad2YEU2A9w-Y=z+Z{oGI!3` zPodLIJ1r`Yupuci#)>3_up6^otQ^4S4a<8N9-$IQim1U;FyD_%L4P$T(4#O$z{Z`s z5z>a&Bg!rXM)9jpjMRFl2-p$T8P9hTOHx$yq=-~CJlFcnUNFPN!( zgUfet^)z_oeB+gotSYtjVb~2}^(&)%N2J=ovsG&{40g-o>blX z0jMoH&0s~6{u??OGY@_%72#i?BbkFdjgDOs$m~f9qL6%stS7OiRz^;7K(Dz`aj~*f zOtK_*WU;>KFs!;=W;*&3ly3h>>9O0r*$zvtZ%*5n>)$%uXJ-K7Ynq&W`#&c3ztHm6 z9d*93&c0=QWy=4k>8*R4`buio?aO+0%TC<_UjyYOAJrh!qqXFTfMtyyKbp}`#~jCh zRm(X{8?r{Qg`Um_WfC?;05I#+h zsGj*v?~EqT-TBv|YQf6rl#^+H5sE1ZfMy#jBri0{+e}7Ge1xp`HBo5O>`-vG7 zDZH}q+0xHX=6zwCl}ZmZ9;OD{;IQVUSce5|pDKEZ5Mask!H#(Ous7f@xf$i6*7=O= z4#t-{d_->1)wC1!Tr2X8Ujonq6ktr%@+pOxto_pi^Ka-h-fhB+_KmQ)ItFFu0I?Q4 zyfNeW(=q|kYu;t#>jTOuvcr`avzU{PWH0Vs5!dZ%-bzX}zlaByBW;wdonw?e&6UT z)l0fyJqiJpa`fh+c$O&2uvKb7CGCUr!e@tS9xlL1o(p zdtdzO3`1Ej1A_5j&3foBf5H%!F*c>p!oo2Nq;GtF)Co>l(3|-!Frpv|S}_-nwlp!x zWwPo>=Rcvij=U@ayGbuzle$$f0jjC2?cmMw+NR!%L7~AV1#yTuOequ$f=KL3041`q zd`AqAh@bquT<{oEJKIThv%JCKmo$zU}z6W52!}Q_$45E)1N~*!TnFs0^JjA=dN{g6zdPIqupaH;|f8pz(O_ z5WeWbC^7SuMNM_2zV4@3gQD2oI+7@u@b}XQTq}kc$?P@l4#z(SB6ucL6aH@M{e@!A zlUT?NZptE*(%Skpo8wgNI?|s{K?xfNK|H9t zN4NX~ZkwM(U6S6n99t(J$IPY|%1c|soZ5Hmc?=_JP1~jJK4v-;92dg1pd(Q*pK{KX zY0!(62AIilqS8;@y0{3{$8#3P#eMHV#gFbaou5RzMfA-C3*O&+XhKdD6blH~($HJE z=6Q!T$~>H%1HLG63qKHCw({nAr8pOYnvr^krHzrKJXY;cjJJ>DFnY`V_C^B<(l2D@ zGkdAAE@SKvEDpm(`+FD93%jwxP)Zi~-MBxwWZ=q`+L7aF!Zx=jM3g9L*H%h?4cbvBs1ZCRu z+RFK`)FpPN1a%7YV3S_yDKNO27G<-9CYBvge-)1FxVul7OH6X-6voO&P-$OKBXJ-% zK`(Lvf*`BVt5;HPAp-=r=qfN`M$av75r0Y?CKxf(>sMG_> z(V!UdYE#zmExntygeJFgOmbU5vAkU2u+dlZnMvl6rUKt0;ElckI9lPOj5u@)aZ(HJ z$L)8D(2t=)iaK4y2NymEppZ6VQzC7U`vz2u;4<8j6Il;u-2CFW=r3QiV%s|kaBfUJ z0@T6u!{lo9`rwI>wzA%m1dA9F{n3%* z1{F%I0|{4OIKAu6l`nZsg1Sw55Ula;K&SZCPu9#crGpI3Yu)C!BM)d_Mx9#Y`qBG8 zeFUf5+F4k~^my(op}k0>2Vay|7g96aUU@-kSIYei)q%e21|mumWz$qSSS59=h3jEy z+7zn!{sUL|oKvD@VI6meUgMVk+Cw%UPzq({2&4N;uB6o+APIAOFl7YbRk|}&qj$96 zyqb4Ylh5Dm7Sn1j4Gb=)*so)Je;T5>4oDcoRY=FahtOZ{#eT(8K^F8L%}vbd0p>Vm}eHM7xOsUo%QSWdb`Lc;N)tt%=ZBz9t7Z4nFHF9Fm!pSpSEq6qsq`Of1l)Fk{k}b! zrQI5&wBr#J4=m5t;b7Bq_%|x;WUgrE&7=kKz~aaKZI+czh8~B zwEGw_O{mB2HFHi0%_Y!s4An=+nbq76?+^4|4b;9M-`1qwc`@eJwIu=~qX{?kRqHvx9h zq?G=BU-D;weRfI;RhH5^y9pf7tGC4kU6$Gz4j9HncpfYus7Ue1*J0V#utiV;V*2t3mud8{smas00KJluRQ>Zy>2vHagMl{4!;;8h}YEMCLxnPE)M7KnyFlt4$6OocZqDsNJ z&jcF3GW$6*hyKZnnle2=p+G-F8lWo%dd}3rT9k^Hh3Dv@8Aj4;6zPtJ-^;F zx9q_fVpGysxxl@lph;!p?ZmZsS8|V95>GgVAxK%9eucE6mMpn`1xbjx{3NDm7}zm$ znvWf7J)S|T$`Vq#=&moPQutHo76eW@!wAGC-zUByyaox0g8WTMWtp*#IK&g=1$K4f z=Ox(XLGoefct0nB4(>%JHC&pyeItMulhllo-GAV~RX|&hgY75BNqXGTf|+S1Gn$p> z$egHG{k0O%Z`4>*nh-tg zhM)1-{G`tQs3uW+Gd~K-yO;p#vsDl7EAWb#XZZy+wfSK6QP4eJ2J0oIq-b7O=cIY|c7=}>U+s|zmFSDU;Vq|PEZpC0y zD|p&xJ}~*Cg%21Uchza5O-u_X@Zhrt6584fx@; znp-wsbE3DY#Qtvw&>s*ctk8q>{j2B6-t>B>t=^ikHVqAzZG}n@=QVp26kWL=)U0Z4 zRm%($j%rxJ6F|duyA1Mr%jgz_M9*h$L@qLW6#8cwK4&!OrEvy(o{Y;USEI^2ck?q^ z`hoQ*CyUB4^jol_*qwubG!|5=Se9Y!>@4EUtOBnxEnuuK(4G<;m54x0=X_YH^W>;9 zu%smSp@!5MopsYbyIy`+?*0G_tI@E9h|h?7189Rs#j%W9d?pDYSK!9Rjgz<#$wZyk zdtKU6tO3p~Bijb(+eMaYGj~7hmg2^+EG=!O;hp(lO!IMsH}C_w^U79D)_;<`ZlnAa zZOI}fGj;w#_L}eFwp#lvJ=^#FK=&6jE0fkBz1^_SXl%E^@j~eonnH>Dc(W}+E?)$c z1cW=^dHdTDOQAqXcxl23Dmn%P*Brb~Y1gBBOv%Wp?SX$s}G;NxcsmjvUq=V=T?idM}m>$w*_Gq)*F zoiG!}L>HrXR~{8Xe_h1&4I$d=0udO4Dzei^gj#@v@tPt(;Em z{U$91!Xt6b+WnW-|6A1PcU~)*Irx0*qNd>?68ENu0)S~m|KpX5^Z7o~! z=X{9*P%=6w0d*xGKiQvaQ5mgS_Z27+3WHxG7hsAjQ6m<0vtD5(?@DFX+A zy+85Ghm342uW=r7v8!Pt)kkt@o433IJ15Z$R8btfHoo8tkDxaao1o1n z4n;1O4WWiy13qNxM&0^&qKsoVG4 zrs)%tr69Q3L08$zb0~TIZM;`Ibv^?=h1>ZXlYSjs_M6MJ@Iz$NvKNP@J3W#vOFloN zPepS{&^c>AhJ&T^C#bC7B!iC7bN`HNh%4=&_t=-wb^X?9_9Ed+C8vl_7tsdA4D0-G zFPucPbov;_SKjDSFXdxb>_w(*N&RiPYHZX9W*A@AOa2W&ODBzTKn=N1ZBP&chfXO7p z=h>9&wsV*^XvBreG#0&_OUxj%Kfxe~sYxP-q{dG{R1DM=aCp(A+N=J*APJ)daWasN@Fe5Pr><7|0Rr#7%NQVZobTpE)0%JZy!_N>cB+HvDdJ=YDZ$+~+!z9L%i zp_T=VGA^+{W*QzgTRM=dY;JduD-ra z{@p_u+4Rovv_?s%)1v-AlWqTwZsSO2eh?DGPZqzA*TR+g9%iUnpd*^@F zPqKgjhL~$a=A3tKgyQlov|2n@0NjAZ1DxWi`TTJ-4sB+F71qZ_#Y_Ry4h&<22e$_@ z!UQAJ>=iWW;7`rn9YYkmz!?P&2rr;zGYj*3h*%yFThI;s98V(OWxg9i9njfNs zg6_duKw~5I#Az%(oj*RB(}9;$FM0a8I9t!NEO6&D*LZ-lEMR8_s<8cMB?3}k68=)u z81jvpW*vJXlScvGBL%}*GPhp%V3~xk`5@(;{yc2_jo9Ub{x2GTTd*>hDEP*;XXz^7WrRfqtG7*M?tmj6?i5%*L~Dd&K2tP*h(-w+54gja=W z{Il=L2I`}1YmI-eb8L&c7_KE#LV$xYZ&^Q$9*jGCQZ#8jaPX$x-t3lmjZT(Vk~Vxy zf!-RMh00?!dm2=aiAzYz);-~t)ay=V^CSpzru0Y_9KmtN*a}xu=8wiZ0XWgL zDUWf4CbQEkRtQTvZzR&10jnH=CtYibJUw~%VAIC4qqXYKsY5}2q!IJBa78rA8Y04y^BJ|G>w{BS=p@b7urwIeP-?ENkA-}*5-pSeik z{FCU1_C6Z}2Cp1|M!~i_)|uB82G-mX>3RRQi4b&Z8Wx~hGD)^tqKwI7DiYGJ45R7j zjhPFv7xyPwu;Xj`t7=Yl{&qTa>NEsPV>RDKwRYy~lTeI7B)v&qc9Lj6*}(a|#fYFr zU8Oox1d(?PPz=UBX9yl$Kh}aiA}WziJ*!RhB686_TM!=u>6u%u_+ny0t3SAN*SPCF z`n+Vk3Kod3<-=7Jc;e0QagL_9C?C<&6uz4z1gYMH8W+syhe8$QHCE<`icD}|BcS!xcPUH;(x-uM&W8}X%wjdeZ(R-VAz25n= z{1^T`MUN2A$(SvB)!4F6lnBb(%7k3VGzfyV@h7JxFN6d**>h-0-+D zUIuC_I@Qk&XSm^1pK+{K<;zn9HIeR~aSlkiCp?rI)Cc$FcyWm}@_3NQXFb%<$&dKfKr1MKG|)-G#I<%r}xZyFn0TEi3FLUb(9cx`oosOXtRpRbVt zc($H;${^W%&R4yHq6H-d5bIO)fBH7D{LI`6R z42}>(vW#tvL7@$mEn*NuX3PvR_AQDMDa=fEvc+J8G4}ag&i&lab07CPzrOz^US+Q9 zv%KG*<$akjr*L>Nl6-`Jgniy%%JQikczy%p>PkLl%H;|nxA%|(@;;gpUVNCB(85|T z{_Gs^uDVaxy?;6r#=6F0ZzIrGr)>$3vdjC;D8DeiVm)VLn2{^@6@yu2`kea!CM_q2 zlVhS27lsfBXHp_kk&>oCUv3!>ZhQZgc)BphB8Rhko07I25S3Xg+Nht{#(csBf{Aic zVimdUFjyd5jmukG;c(KxPjuJJI$PV_>QkSaav7ARf;A%9=EkU{&100IQd&H+t}{n| z3PlXG-h>&HzPohaz)t%SY-bLXi6c-*Zw4d5U}@-aP)$)a6GA7}$q-EK-U2I3UAdm? zelZ}p%4xS*4yuU$86uq<8Nng8o_$P{IJ{b5+JXQl_t}^ydq1JjnbdE$E4PZ7GP2g1 zc0h4bNVEGgXZ-WgzQ!a}ppmLaKfR}hKCqk|dtJP`O$F2YO=BLxL}79p#{8$7)d}9# z6wV2LS~IJqxB?5b&y#iQ=ZfiN)`^8NRcr(!?>vYb;;Q;5f?{gR`4?RbAhS|ID6T2C z8|2IuoGTP;Vy=D6>jTS|DsQ2xF%BP%Y9}@pvBYbf*N2IZY!PE$Vkx+_M*K|t!?ToF zPvOmn1xB8~Y5d#{Meo?9i`@QP_d0YBP33DF#827!2ET)HPaDMiZ2(&XG+%0HRvQDp zqw{dKM!V@?rg>_h;}{I&0NV{HqOG=^ccUVE+L9kK-uD?t>7oK9C^M@-_|s)7FMRD~ zokUu@R}@mNg;h+54{xwoY~pyj_ZK8`hWpLjgZoXpjB-C^mn-#Qr`A2xGu%q)33z6e zIp(ybKZeSj4qo@m>iCEPz&MAI6MPWO_VcKs*lcA)spm2$!cEhZUmMx(p~w?7l?Z)nTicixgR^#NoZeI$7U;8^-qkW>y1`!V0Jr1l{Mf%Vc7Ms z-7HFu7PyCsb|@8~WS~galI?A)(^GJgm4m;8Omo)LaS=`&=5k7E#N0ycr}f|4zxFpq zQkHwA4Zbd8%3%xDrkF}@qqTcdfeA@F#-XZGo>GWvsnTe{fD&!)KqgRlOTNp&E8euO z@#{TaNDd667{X z6={2z4JWQ727_Xv$L}>|^v^w1^iBWjQM#1n*VjI33v<-w4v@eVVIEe2IFt02#;M#^ zD$1WkK$0#|5tNC8JM$7laPNn;KCAZqh(4`hDg7y6e#t?9P*(GV{-vX-nHfja)?^;W z_dB;#I9FF4u2xx!JJtM=e0cMmn%%|>>~l!}rYY*p78g=mzEe$wpSR;OAFgANS98~g z{+O!rVv%X%)Or&tn2SGWH1VSnZt7%2Fx`21@EP2$F7#gs+cdAx2p zJ=yc-Ylb;X%ss1!b0FFaGqo(ws2ncJ(LQLPC~#&RBO4^l7&roku}H2QTPG5(am(QV z*WFbeAHlPt)Q+XT*gj;@ryH0S>10CMRvmnGAA0ISj)qvW>Rk5=n7n2Sd0_=fYOSjL z1ea&HrixLK&QN9UN)~+qHu}NG04ZjX?T(Fc@p4f!~C7V2*%%tfTctIx-*B zST2awm2-1z{`BWhgR*?oowg)vb%#FjnjGe742F@q8?2Lhd3@TmLZgi3JFS8(`SfWs ztl4aacA0$SB=zuD!W}d4!>8YGQ+f>w;D;ef9k`CnnJ;HL2B_~~G?$J#nS#EIDEsu2 z^zT(^X6;=y3t9OOb>v=w0yC;;GbrcCuS6ogCy=F}PT z>eorWENYl89DBI9&5vk{2OE5Yx}}pB!300i%>G zym(aAxG*_X4{ISU)78`G^qAhQrJoE)V{&V%FRLfUGqAEg62%c5vx;g2F_1Q`D$Y?) z{sB|p8cN+a(@qqwoy7_`x}(?Bep}3y7oQAf*M{B(FK6qw&khq$JpAC{I?8{+rj28@ zI&15cQ$#1ys zng7RuR}T~eC02^-#05gAmk)H7Dc0PDle7|D6%X$x@S&|uy=d$&#xo~`4tQ!2_nUY| zxpinpL@a;Uj{PRKJ@TPMBB?1Dm<>eMl$FvaJ&eX)l{|JOlh*uivxxMM)K2w)+iTH2-0o&f1Ava=vTN&VHLma) zwE8Z6Z|7B7OlU2fUutg_cRuSxWWJvFB%rofS9)FD<=lHLA>QYbPo)0KRYd^rG7oLc z-5m5bJ8Ql#ufXM-yjwfr43~IPwLnN5?}@(LyZe%ABR+XnjmCtEapsVDMAET{Gq2Ub zBUT&tJR!cH7_M(^`kI0GLs>5)+k`aQ!S+cf?!W9cGYo3^hU1hY^=GZ?XBNLYM|Qa) zX#w!a>DVrKuPxKqu8Ct`3Hyezo{O%UUnkgDnQS@QZ;H=3-`UI2`PjS$*MJYjby)y# z0^>AWI8Ns%dmxQ)RtcQGg*#ZyMr1Cce$acqDhg&I)Oth9orArl>3IfMkKRVtJ{l3$ z!L6Olg^6u!-V0Z@9rdRv}?Blbd37-)5WZFmPF_u7MizfoyQW*1q6=9*Bvi5 z*_l-&8Ts(=vm32+T#z~@di0=xtv*J6#pKo1#wkl?PPsCeN3|u>RL<)iJH{A?sw)cX z916U+nl+Tf%tQ!K9eK45Zsqp*c)VSQx8GST7vP-DYaf%Q5;$;nwNJ|*t@Nr%q}c>M z;CxS^NV!ky@{&g0nhs5MI@brIpg19-izF2!aCW-g3%Pvhag<1HkV93{p1&r%fgaJ) zfu8>3s+&U9=7_`-s&HjsB77=S|3id`k5{pIM!12o4oa*&Y+v`Xw=4E=@-4m_uTcVi zw?3I0OLv=AWp|YKk?|q=*<)uq+Ph-OXeR{wOKh5OHGvOVsY8of$XII_^oBEldQc0i zdLp+rZvBwLo$q;yIm-bU2*xp3vO33gdqKoZuv*NDzyi&q2R+ng2{~}<=H&yBy9T?I;x6yO^$bRInLe{7YQ^Zt z<$Z&5z%0zJJmqa|6XV9$iH67)vUgK;PhoM6{VUxyB*Ru+JVBb4>$JWt*IoZuib;MG zGVVrh19}0emQ%Ey)<7)|Y{aSfeT5m4Fif>Qwtl^76Wzg1~=!%=42oVG_*^MW+ANKK^eMnawYy zmd+Kf7CQuSae15LQ1)p{BzRk0uk>4A_Yrc0#J8TnWvHCgAbf|t>z3abuEBfw%;AFq zEotN9dksEbn{(n&x*iibrXsUKDzY3alR8wqn3@m2*ul?|h{aDNwvsBlja3*~{sAlsaF%uyTkDV>{G) zGXj??-Q4rqSw%jq3JO92LHJUW2>W!|rhk2Uki5UtC%jOz5A=|b?g#gvC+6}g=y8h{T`>G6&%~VM$yygXA=%0*D`Y2C38Fdt zxvK@3sUvqtRc938Ys)Fm*q^{^ibd&MAO(ZYYXBg>T3K58ZT^unPW{`j%sQ3 z+nh@Nl|u5*g+4O(6?6U>IOgG%6;N zY1B9?bHu-O&}{MA;KQzVY>)h*x;q%9p3G`IPXv0LHpj%so*9PoX~FB~fE!H+Z9R#v zjh?9VZm{F_HO)y;Bq#w?j}9-iBZs-W|5M!&iOx=n~-mV;LGuDY4doJOAzDs4*^xe6W78=9~@L(|qt!s=VM z{!;+nnE7g&QWpBRm;G1f`JZo?OAUlX8p6`GxK*C}&)hjWxLzTMYwJv76V4<=OsF4r zbqSXh4td3A++#!$;Gu?{AfWP(3x#-EWYQV-B5Yh+p@^GjzqwHhanRTbNh0mNe^&-n z28dETL;200x=Yy117KcGDhNylWOp)!#(rF+QVD%JB(xC~CAFPMa%?}Je{-LwU)7FxJxm&@6GWlN?8M&P<;OIXj2cT947umhTYETs#$f6 zJ8N8pTjsnqZLIjU-#>e04#|=$jxpkS97^;Li#msl8vF4NMJ4l-pqb-%J7x2zz<)U< zaqofcMm;hS1?y?t1xh*6X;2`NVm;IvjG3}RDDwLUQC9qH7k2^W6_qp3cG)ZE-{8mk zKl@rz1ll`f5&9jAD`U4v$S(??`X|7E#la4oMpkk&&MY(i;boPH&aN8XGh%?h=7h;l zbVEb}qNu@Z_XcBHT}Y@2%uDIR=7$h4j%!Qqk-nPg);|~QC)VDlpZ`Yn1UgP5L!C1* zTm4Dt31CFsiNRbhfRU&-HS+x2-4UsoD*f2%lE|w0&d|2{M{|-47Z)yY(mRC>XnQT`GL_T0tuzp zI;rA5aK>0JWIs9~Op);A&LUyI`Hj?3_i#rw%H?yf3E_^`vYcf6=)(Q6xCK0%=0c(_ zIT-Qzo^P1m!Vyy)JrWEh1&(*U{Tvqqw4o$Og*Vg1Y!CIrfVcU}ri`H$5d_O}I+8Zl zrTB=+-pK5UsFu|P2P4}Ztv0)@%Z^sc_@F+1$XS&3Pu|rw?$b))DE=z`-_3y=9x z&Fx`_%|C3@=}<5ot2M|rKhl*Lb_Jg!QKn-b--VmZ)CgpSw0`bMQ}=LB?R!*rsgFB@ zc(8W(Rj!Q$gGob@N%j`7C6BUTlqwW-h=FCFhE;sYgM ztmfFn4y(-l!%TiBY^4B(yi`G^nW8$Gns_>=su?k>Y26X2ZG^6izaWB$3?<%q)sLPk z5k{_6I}MBdcVqNVW3ijshM2kAX;I<(Q&qh!+mv*)G9GwfICnIYw!)}M;tgKUkrt{a z?w!TcGYHh(k8gs3q}V-3e_LwKiLxBtU-9Z?I+%?u^&E2eBQ+0XYFEi+T}Vwwoi471 zCf~QU)B80N>cMdfrY!0Jue0bPpdKZWP-F zKYR*UfZ2;({jWS^jWfjrDH$T$Ja^{o4=cb2me3~xtkAI36&jH2`?5+6vFMjt{eWnn zgwwjzsRB{<7A=jaJgOgU5k7(Wc%GhEXNK!%bg${Ns}mKF)L1l99Y2V=Sr1FXnUZT* zv;xdZe_XQ=xu|9_j)-0CPy1fd1NU%WKdTTBk8@Ho|Hss<1jsPG;pyqW&)ok<`R+vQ z>M3E_CGIh-T#%zoz_`Wzy}dnGl+HZelTmtGFCV`@cy_z+^DhUsW5JqW;2sj#7gMIH zuxD@ah5xabUHs{*X;mXwxNAo?A9x40kc+od6unbB^f?c7q<4;@oR)LVGZglKfl&aIVKNZ#1UT|c&nXxh?Xrqc1*AORaDz9YL z{w^m4B=^kWTXys53%lL8nj=zoUtJx{4~L5F(APsP_hG1%u2_^=PEG)ALXEQQYTJbc z!Z?{jeso>M90JX;8+k>ygs%gHaQ z8b)b>=l?v3A8w0!Rn&&&>PmsC;%Tm%QAP3N^Es{}rO(?S<^49t`rYSv zqd3wt1?|~?YXJmH!nK*4EN@UU;f(x4YEXg!mgi2^Ed1FC)o$kpv9Eqa`7vtgsXJ!p zE0O2V_gQ=~y{~6<^&#_)g|v{{k5v-S{blEsa%cBo#x2B-aMvwy2FCB0u}^j6%!Zec z$jKM?G&OP@N3@{s!6i^5zIR|4wl&Ez2kp|g_*SMBh|9CO+SAN3Ggg!rBsP|_KjuN% zUM3cK>_eU;FnfmA@HoFXq*yd9nVO9O zC6YJ7L4Qx(L$B`x8^_mX41T((^+NuP;$yss>JyQ){E<}UGjY4}q(U-y^yE%uR+|PI z?h3iN@nMY$V52VA`p~4D1@h;ox^f-QEHhUYX=5GB1Uk~)MiNX?*TcXar;t8BMSn1v zbKOI-iv>&7nPPh)DLpk@ubIZRb4Xfl0Vkd2BGGHL!<9@xsj^fA@;y zkhMCxMGb$IiQ7kif6wEOR{Nd!;W$eoqjO>$ku(D>&RHPc2z(aDiJuw^ta57gv^i;( z*=4Y(H^;cFt1_+iVpM&;AQQR2+Ut2PRlYUNlIU8=X%bf|QwEa_5)><;pLC!iwJztAd_3e7oV|KNn5G&WXG`xPDD{VWvc zzu}eL<*+9$8h=7H@97@j#RpeNQ_H)w@H~cUnn07NDzG^0dY92*>QW&ZW+T@uJmzgI zCA9t}K;SrawGat4VoYFiDb4KPC~cmp%t2;mYZ5c9Y=K95@rW9S-?8imzhB7B^B1v;tSL17T@(0k1P;+TgkNQ;lL5H=+fUzTKx$q z1`Y$ULX-f)toY|55R&OU_t#a`R^(USijzj^e*It7RrrzLb5+4~$7OM_AiAuQ-wyCM zn|6z)a?g}s>7sFFx&bi7J&oiYSfn-q0HB96n(W1SJ8%K3bB?b3ALRcA+p{+wk z;w)`=U=uAaBr{OT2VWZ>k>)jmp-!u(3RTS~ZC3mJHyN@B;Ym2*|Fe;!E!)hc^Pinb z$nIz_IFDF~rA%)yZl^^ZD&&!y&Z3^Ttd#fU8y1Qr34hmRo-Wnh9-8cSwFxPuo}t&M zUbH>RJl#MM;o%~VaBg_d@B&h8CL71T(cIf2N1_&EDb%9G>Kltf=tzoge{stEJ=v)E z?PT9mZmfE90qb{H{B22S`Hj-`zgfv&>F@t*w~dDvc{Kl-7M(6?P_ z6TjJATgcT&N>4x+Usfi!C&T9|d`W>$D_7)GV>`Kebuj#$c*ezdR}%tF%;*Fs2~HkI z3@_;zK*CqeOGavvf-{N)cTKo}Wq;|7lW&O2u^!(AgRJ+etl0GU&tBL!cBs15U!ZkR z;qL?4(K8#I)gb&I#D@M&*}#Y2U}N8Z=ZfRjE}ZcBmg8{o@kp(~Xs$+QSEEEwId-<~ zjFiG}cQi5m$^BI1hVgF?FVZ0$6RMttBQs92+wZu8OUQAx(jh}P^nFPhc$rT&%tPoS z=$U(lvJn((V)A640&c=t(uT`VK|5Uum1W=fK9LZ6>FwB&G7DP~PP>HKQVzR=V)5~x zZX^AK77y{n+fB;cf4#AP5exBcAS$1db7<7}^MOs57oodZ(`0hfmgjH_-93lvEQ#rH zfq~Q1$y}`{TyRHM%qU|<49@N>|5_*voS4zsRpGjJ?__u~LaibjpJSFwXS%UbMyc%( z{ao`*$1Jhdv|X;^t1}sv>65nQ8>9qq0MAt15dasYd{FGV)hZ3-Iz8AdCj$*1EKDA& zeD)8UsUce6xq0(X4`lwq0RQiX?0UDEFZ-Mdhx#^OmVpr7=0DViCfQwIxOPc^S;VV_ ztAI*2yBS4d&#BWGY|>iA>2?Nn!ghk0Y@k^|Ow*A&6{i>lnn!r)OJ`ecyGo{Qqgq+2 z%H!j=*RB<6{*Z?K&{*)SaH_M_i2R4LtCpP{$o3@s$*RBimH+!yfidm17;i7Qjp0PC zO}4JSOE`Uqw{6#jwPjfX(U<3ByJ-fYOX#KA4JDrQj_$85Y6HvktD9S1? z0yIx=?IsQ4(rG%41RB|xg+(_KrxoXg0+uc3+-Ce;FH|;)C_1|3R8{NbHI&$E^@V{4 zr-eF_)85$r%XqVQ1x+DsnSI=Kj$cWSm%d-4xSxS)lIvehJ9OiN7^hzy+cM3fdIUfvwwWA7x+z=4?b3R;_j7y&;yj(qy{%`oBMqK$(r8pt zky`iXr<*FU+?o5QFDLE6Ifj``yx1di(I8t3=AL>G9ZufJ1JZ*UG!H=8H!MiQO=K%f zWpr@mH}v(55d)yFW)>=m^MAQAq`5vcO{HMm{pMOO7^$Z&&s`muu3$|EGfRSr{XbZ% zpIAPcY*S6tktrN$6UjFK(mHx)sXsM29$x2!kgaCHE!$sjP{E36g}H$QoOzMMLfmma z=fk**0Oj_^&BWJfg2|1J`Fkg$UIed+tRcr^!i>ZqvW_}RW_X?_Ii! z_+d*f-IM^jqk5z@WK^=1(FZ~}rNg@!5h}YUoSeO#?9+Oya@A3_e8Y!o%&AO|yL8-g z_Uc9t+gy6)jbL#8%2faQRP{%X^-&LB&6PsUil*q4U^2w|&Ko{`tCeq}HCK6%*Y0kf z=+Oq?Qu~*G30M68_H~^LLhf1wLThz4at@6D>-XpD72N5i2DVRLqjLrh%Lz}rrKfDs z0a0|+hC(Eka!sjPF{0Va8^(H&;Ppy+ZXRD6(N^6<5eaHj5eYU)CAqG6+49B1QC%f< zFw&at3QcW9FN0`hsB!lCrK(AI<|TKd>g>vmOCpLaPC9E=bbK>STNd7ucVbUp$2MHh z?1wJ<&GLjPV;fXp?oi0oPbp5B?B29ncpIm!mTT<=PHgUYSB!74O2Q|b7i? zt1bvd>`T*=W+PfgCp)tG2r!;p_V03>7B$QVDpm2O>$g6d>|C6T<_SGEHT@ZfGys(a zs`T}d+;JtX(X#+->yWQ7aolVt0U4G1bBMh zdBJ{);ef2|k{;)QQP9hk+8r97d%Zr&^;-2W)2C^aU=-(9&Dmdx$mY(m_*%i>nY+`l z;1O7POz?bWFS>uFZ`y_x$T(QTZ9IaWj8i27vMCm?y)&N9so5yv4{U8(eWQYt!%U{!l=mW zsQx8l+x^f}w4UYOLo03e@|f{_laoRC92dvS*C&`fpMhE%LgJA_(<+#GBwjWVOBzT7 z1W6S#+!X_Bi!u)f8vr=eU%vFiZ^-PG2ZKL$+d_2 zW+JA`#&FEwDMm19rCxKje%gpTZ={OxYrlR0sAs%Y2v(rV$Z-3 zJqycSsYqKNPn#x}etW18_=7tm$i36FjF^8ZnmQYxt|2uKMot@6`Z(1m3fCVM4!4%& zN5&g#E)B=(=Zs$q-M!ha^ms+%1!mNLmJdKwUOF+bH{Mnu^i(AETYJH3775V(-|}YK z3oI>iMP7N4a3=U1d8&@@`T-dNAHK*iirzDQVYeO-NvckgyeYo?byfN_#9EqX;fG&v zAb=Bdvj4+q@JkAVvZ=9(P&o&0zT2gP5r*8g#Tc$XC#f|kdu2AUrU&!mk?mylJYF7^ zwmK|XQBpY~*W2B7CAQok^1>eWd@ z-f1g6PKGrny^VVB;&ANRt%5#4f{b;$dTCVAvR+5=fl=+lEvI@}%NIOZu)u_$@yl~?|?Ew5z#Xlb{e1-y*gkHEBHvKq$71=qiI~i z<)&(mOe{Va>Vonh?a~YRI*9qNd(-w8C2KGP zre}6pB-;3P1{z-IS^)t`gNZfjR0qb`|Ghp1a>^y>ySBuHW=vg5sBHtf%ZRU|4?8^; zOh$jth+Gz_VcM`J`he}>LW$1THs#(!(tr;AN=^Eyg6Ll5M|_9hwh?i4S={rh!g+&q z0dD)?^h|GLVs!uV^iphMPh%*-;MIpqxdx=mSCS<4T&e;Iadj{%kJ)ZpYM34*IAF38 z^P)exx}5bjwq_FT@04d}<-5@%K)?IwJHPTnh^#l}x{gg%56CO#43?1wy7O_ZFXTab z74?x18sx7?tXUoDZRM`E`gvfORu%P;&NUxbSgZ|)ofcdl5L~vSgJ9P*EgC#1x}0-J z`d+@{$AlJJHCg4MZP5~%!3`vr(xd88($Pe6xlC}+6ONJEkGGZ*wN2B5)hpA?!`1ty z<=4N-`*Qlw9)4g1+WkJ;Rq~?ex3?zz{j!k);&6!V_<*j)?9=Hz9{$=A7B(`Yn>C;V zB*6q%WyP@Dfxls4e@zP(RUvii0*g1dS_-(1Lyeruzr7~)iZI--t7EPi#hWY=L_(7H z$(UG7d_5I%et*bo?C^Tf7z!9~tRnsU66U6)i!($?RuTPf^KhtoVZ%eE$Lbt!@>!=#wlmBXVif{MO zVur+W#-gWoA9h66cKpvEVHCMMdZDHh!(HiT-F*U4nt0$-;XBp!1eNFXZ1MRkTr-MjTZixx4ZZ`Nyv#ysDNz~()`>#<%~>in{P&Afgu5y2)H8j0Ne4+q{r z!YOGR0Hl@qiTtx#B%oa)&&A#+YtmfS144oB9BZ2vpB&N!Ru0S;2*DAXH)(R za2i)%A3T=eEwFm3>vLhNM)%sBCfA!=R8%@Ci21Iay1v5gtY9tXHLY-m((;{mYS;C> z{^j?$ZK6ld{k~0RZwI`@oak;W?AL3d8#OdWxRMLK*&K*R%O$S-<&cfeakhP6#iG7yFK^$|(*Z`5~>M1^W+&wTwW9k^l)nEo$8 zXtLs02z&5CIuL1q3hm%J9#p4EpAx;8$;wM`}1CiAN; z6&Hw9G(&_vrO>M8Fg`v${p+K;19f^=ujXfE{RVrg3k?|kl!`2LYI%*5x3!4O_$0Jf~}U$DzxAGDM-pbV6as(u(aUiT;k$bs_HM@J$Zw_H^**zZ|Ta7fw$Z- zvxhHLkUwVt4@FL^@!x5BdC|+-C;2<%N+)m||Aybut zjhpxrNb374o30JWd>_Pdm&~ySoDhTH?_J{k(=D&{U?)qwl_Pl(2Z(3P>t>}Mipt+6 z@MccPyLT5;NmeCYS=VAGV~sQl{Q3eKPa5GS>r};6eC}lIgN9!-FqprsX>SF+ZNesd zjWl%vIB^^D?d`jFsrgI7aXYZwHO|6UhnTV%=wgcSHnFpR{QiDrgFP)e9>ZWH&6cih zAuqF;V_z*;j%&;8;@iC{-Be^EG|%D-!JOo(=-P3Ib$Zy|(nslb&#mracklRw%BCwj z1P<>kBOy%(!uP!f@w3KKHb1raCSyKCtp3EB4 z7`55^a<~+2oLg( z3+-2LDIpEvHL0EVV70Rmry%k9mDIrQi{7JO-#!p>e-p;1gkEqHO}KXT>YZbGAP*;N z#LL2pAtEyVCBw{xNAlJA-M1@RTxo<6-U|ffj@OR*u$JtbAD%6#Bo1Mo*IjrdDqpOQ zUMMA8dvmn2Rg>$6T^*9JQJy9c$*=X|WgM54`PzheRm@FyB6~_bvj^^t-ltm>81->NcoV~Vq~vhI&n-X(lqSY}eX z!7p@R-%%BIhbc)v@$~H~!sH0`kVoYgkogQLhdf`r2*Av|*%7R$2IWPzXJ5a1C9lAt zvHS`A?G7i;GZ*GQ{PF&N&YahtUnC9;4uBV6dO4RaRc2p*5n)|$^TWOh@JqPO47{NO zDXIH|(yWM$&*IyU;RN*F?>=<9^vyH?X-~WfN_ZJdM0+%a>H;6;77|cno1o_@gEqYUx;&)q+bmp%g{N?X%{I7p^RB&5%PK#gW5!olu z8i$v(i~HOl=5bBeUb)PMYu>aB&>PIB#56m0KW#EqLu3g{CYeiRdZ(I3E!nG@Yw^cd6tDebn%_5&o6l3Y`i|42!5iM zV`5@rjmr1y$1t2erHtFI{b-bkG<(pdY91~VUO4Oz(AeU!yh-!fA3M1H(e$Oop_oVs zfpMm4CgIW>ErS_ezP9s;13h=gex0ge)2)2zmd;mHMxw=AD}|q#n%=oubLdR~@ek{{ z(U#t&lvQQi((v%bKI#|i`UxCNZCXE z!7C&QrH-@EAk~pLu6er|w)3^)u}kt$mv+`8nVNBymA~l$6MI!fKHn-mEBVTq%ZvEa zDeR@OGzpaTj%epSzw8)3`d7NGh1&@p77{Yw{_}G4UVu*~x4r(f+Adu@aTPBEO?CBm ziFYu(CPM0{qs$RJH)1(l(ZMv^;dcnur8!<#BS(M|>liO%Q!#YanAkhhm85Z3Xi4@? z9#4YPvy_y{%X~|!L?P~ZzDRQt+);CN=5EXLv-QFYH=uccynC2L_#;>BsRrRP-p)K# z1vMMg2HX*49}U8PxSD=^s_@QsYqPTi@C5s@F{7K378yEe}s7)5#*?+SAzoTOFwd9XTebRW}>cuqKx zyMbEFU%!2&%_&77spFj9PTxY8|s^ZJO-)k$0)a8Db!Plg+8& zst^AH*uZOIaQ5B)pt&e02~GoyUk`56e1@Mi%dJ-)vWHZ`3U-ay&+z*O9(eSOr&Qj3!@@*Fll%7Y5*@vG*3=@3*Ua%P9~j!gl-74-8) zE}j+qk70tpa`ylG#(Tjn^7lvFMc$iS$z@erOlbD;-*jO@^B&4eu)<83!PdyM$H|kn zhkCwHT}PClsP0E6OZ>VW)}EsoaM5ZAe7x3UY zf6Xb|>K?uz;d>^pZZ*Y8FVJANWil4!hV}=bO)gzUxtjL~s=0ymA4MR=cV=Htg1u`0 z3W=0{lKJ^rogrlDw-YBU-#rwOJ8(_~7cjhl;4?#@rxZ{l8ub zeFE-VW!Qu-Uf957@g_abQYmo$7Ppf31SC_O$k!j$2lefKC?Y!v((ar*t0OnoDl<&_ zjS_02!;VpvHp63nX84tiC0v^Xr}kh7dcqPC9)RjY4(g^p5R&XlzjXIe{vHF7$$8fnz9{z{vS}F&~F2L1%Xz&7w*nL zED5*cEyHEyQ}9jZ2$U=|*W?P(G?MYb>k7c_Hqmvk&OI)UOLnJ~kMZ8iZ`MmVHLd}% zl(xe+SwMs~0(khnBIsAGjGfR+1QYzjR1mzfAqs@rEW5iJfDaXRQIQp{%ATH{oxy1Y zCuqjy@KY3+<8eO3eA0QBXVT4?wK^0;m4S$l@bJZ7rqZ~p&Tk2t$sd7}vy862iAGPC zH3E=y+s`0F2+B3NZ*cum%*@0k0HIRHg>uyaY@4pSqB17VZ+?n@yLshYm(0#=$fe#2 zzYEZ-riV0xE4wEm-E4z?T;Ufi`+kI^{#@^8Zo{qW0BFK${tCyP1XQ_?!oZzO-|L9vmS1&%4(y*>ny9ZnQT@X zM^#8obX=XU6-GhyvL;xGqe5<0#csQN*D!qBt%`09E^pMXBx<`#Ju|UUzTmo_Y#IOG zNc=B`Je&)zTW!Sg&i})8haDf_dtFK5wIoJwBRssIGLUUChP9WKzax_3cP1$Q^GPqL zAS=+uD)+kOuVy0;grnq+<#ExikV&b>8LgJisg4_|ICMAP0kcroPbn9(fzAAmmr;dy zNPo?D?^e%Rgo|)0-`WV2p#-2D#+?ooML|na=p+*^3MRmF41lV9#Kol)&Lav9_ekyWl zw?6t=RhSeIFce@F4yl6ZK26byKI|f>KL4b`8Y^E3K?EpDcpxrzj{5Ei)cNC&mlDsC z)zQA$Tk`A{_4-{{E8;0R0OO5*Xy=#a!%F&04S(Rm3T8nC%`1LkwB?29;f5*VY!@hz zG1rUlrR_FfdU#%tPqiw0?>ML~F;TtdgS9gY?|+TCm#4mnq4n3;J!%jO8!Fhe7WSkB zR19b7iC;FQ^#3R@{cm80ywaMQK;`+mrk=WMf>!l8$`1s*R;_rKG$cw?Q~E4p~rxGRs7I>e87GK@eseoJ$v@- zMaE077=noniU!qj{--rem8N`%NM1P=#Wd%NO-X6_F-av|;O*~yUOmi8 z?)u6)>j}XsK+fXndsQXJCgBQD2(Ka&i}9?;YAaU=Dfjpb#!E$c4>b?sKnkOdK^#iM zu9x|BVfw3h%n>b(SPKDPuZr#Jm$#gvjC%}%v^9nL^)_34gQ!e_EW&@0((fHkGy7r~ zs!EYA9pO6-Et5I$d;|0EIi%FQUmh{JUi_V-g*_x3Y;vWl8SekN?NEQkdRDCAtglV2 ze#RTS_%rf%#%wwOEUj3b{a8irZhFTLm3dt5&?vw`A0R2-Q+q7%SA?d81Kq|ZQ^fYa zzBlrBcfVKa6>ofZ+{bgq)U|F*@$A(rvf-4S<{}*zk}k|a%=a~o3kmlfyC*`(PQR-G znfneX?jcyg#OhqD-b6^+tloy~3$P~)6iNgy+Jjq6#eR=7HT<#Ff}n-h zb8v-F{I7?Swrt%F#*4MRy2C{BZ%Vj9vbM=82W&ZO(_W53ijv#c?+R+%0(chXour6N zz~k}ZW=mo!KCh(DXA*Q>0i!2ZX$*F_&cPz$leW=Zb` z0E;WIqqSyvf_OST%Ykq|hgAeq)ZZc-P5>BXM5b^Ec$+yY@V(sLNZ&^gOY+52X}}fjQy{O z+!eklZJzf${*+}~_&Kvk8c4pw=^lfW?4T=RkbxQ5|VH?#R((0JtI>#q1pN3p!=p(tr9 ziuB$bPv{l>c&e~3#Qcs5;I~!!HmJ-BR&_U9B)#n0L)lgDH{v8#!y%6%5R5g0dGyCb z2scFVcCYjssY7OAHWbUm8)$A}uLv&~=yGX4mb(O{ij zJ1eX1o-6C$2lf%Ay#HwA|v)4_->`#gBHji)f8!qm??alC= z{Q9NFd7w6QePvGYNawByLfGJ2fI|4>3p+Z@DnJTYbG1~-waehWOQ~8&2LCxKyXKcb z$qoEa0n7L};I{yhG56XW{O5Q7FxC=zErx)f*zWu(j{h0YvAorQLk}K2$d*yK=t5Y? z{QNXbmGD0Ej81UzemQt4K|OF3^s0s;TZCs9XX*uQ!ifMB>Et!wL!O4|yY3J?;$yY; zxr(4Cvb)mYg;I-|qu!g-P$Acz(gNvPFqv`Lf(WDda(2jCE^R&1dE{ha`+r}SZ?PHyxBL7)rH|>fyA~2c(g09fFBhtT}oGbjK^0DHb`<$!BF~JW`k^msj zK+Xk*s#f*eZL~!^G9S7j!cSpJ=GP7@I2pW6&uB3I+)Bjxe*b(7YZ3k~LQrCwLsjq^ zzRJNEe{SO6FLUNq6P%zm626dd%JSQ^sbbXtXO2jzh8#4i(S_X3l{Oab6HX8O@lMp{ z1Q}$bh_rjPZKvt}~UmYpvbGOPT(EU+DNR58Zo+ zw^eQKF-SjG++P!FGt+mfe0|CLYFdT;R8 z@WPF_Y|4J=UP*MWndTbqn#V>Oo9eIW z2Y7g#*A4RhCVjs3<$UUA<)^&9U!KEeIz6hFvU}02syEuVZAB_?myIN3$^?Yl9++CX z7n7E{4ysf3TH!B}&%69Va1wPRRQduxdg3RhmPV`Y!)Eozj<@Gju*yZrO|FOe(vSEL zY#~C42^D7ECgFLYlk}k~4Fu{Vif;j>HVC~guk72w*QuP&dfDfZ#R6^BV~1kK_WhNg zgi3)5=k=-I?e=^9CA^&H^YOSI*SO#B z*Y!wBN)k7#4yYi*Vl+j8URVRkfjg;jI)$(O0b#0s3)RkmAu*q!nF#!>LRJPqk9axr^6BU$Wkd)MnPH|mZCd~D?d`Gv{ zXflL)GZdI?+TYrm0V$Z)0lU|Cwlf+=D&x4Z9_r+2kn?)9yzri7N59&I4(I)`aU#$& zK^R&1@!9>`e`H(-gaep12|R@W9+7?v-_Apub9M4dEV~1ci3Pna;Kn^aym;n_IWIt_ zu5o}Lh*RLB!Y}mVpSNe!SL}=NbzDq?lp3$OZ|~kdd|~HZ4PGki!w)^ASvG(`)%>`| z#jmb2^lyH0UmG3NIvM`oq87E}eM&Jl9a#u(`J_vuEKP{Yn#VX!X(JkS2f6v-xyZraUTC{!2=x1L z%%bLf9ggJZ<)SqiqM&lH`SSX%bH{~{7rt6h4*0L!*h!4@h>=IXX5-CwFOW}r^4q4h zzxOGOoC-uDP0h`sM_3|X0Zrum^y%RJH#|_5XMU-?r`KN_R8DlcIPfTqGG9LK8^7!B z{riJ-DED%6)0NbQ^PbkyNY)fQ-FErF#_ay@U+iFrL1{Y27yR~KueE*iy^nz~8e5|2 zllI}9x;llCoI%K@pHh;)Hn>`96Q6N^q~M;P@S|T2-`~%X(97_Yzl8i3w}^0%uhU>d z+poL4W!+$KL>CHBNAzA3D7{ck0zjW&^`pYmt7MH{$sraZZoKjrd9`lkU={{XX&d%%fESatpszc#YI z|L&LExiU%v04!_C(qUzvoUpROe@cjtXE?*e==b*?Zg>IAc}AYIp-fdDi9YB5p7`&t za9Up%E`Kb^ew#ins|(oj^1p!Mka`FmHGGNM> zlQkIz;D}o~Ma81I4ioYBTE8Ti2V_8h*+AWw3Btx&M25!N)3A+PoUg#^6p6*?QXb2 z?Ei#o*)!nUZHAP&a+^OmK`Jq26+V{lHM#!2R?eFgw4>{ z#J^|Ho`JSksQ*7605DH#cz~zJ$y^Yq3l`jNby-=W%j_XaZ-tEP>NiLhh03z|-PIvK zi`CQ6)z$s1$IAcHMGJqO_n156k9oL8%)T z8CHxBi~n%;8+r=a4IK~?Dq)KOTr*vo8(%9@UO4ur09M!vw&*QQ2#tu~Gv;-V=lf3^ zP=;`txzHUd_6OeKG1=^B2i-xF*i(MX9^>7`wu6mNXFsT`E>t1hxq3?quue-==%#wO zUq<<9mJcPokg?+b*CDv4pbw^X;K;=a2(WUvZ8EHtN-fMB*77#s(((_@>TP z+7NSU8@3k5C79vZuaJyT*X{JScYr?HXI2en$_lkW4*1M1poXOxw%Y!bj$G|k<ADdVclVO zA46H8t*@`2H@>Y@B;VA)lQ1(|!yT>ttv=GWulk{#w%<(o*4^#XL(R5e`mHg?Hd}WU z=FW`w^Z-l9yfSDy3s->bb~HToc?{rg$73lIU&)d@frB@fa*r4J#BA~8JQT-sV_83n z=b_fIZA&vi?91l%XE?d%wG6HVQ5wS?Ri~Xnl`EMCIC44Yx{m4LC zS3Y_7xk+E1#2cO^8@WpVg6P;5937%zf8FuJ?6-@<(LknQYtf!@etYzfTK;rxkecs_ zYAy%9>Lb)WV}26d8DQ@|`iTL}a!dwhz=Y&UUKzgm{{tGEoC=Bu5mAXH@q zl_`t~Thsv&p%N}@R2;jT<$%deKW5Q4KF^DmG{llm`?I$us_vHh_T(~G{h795skmV+ zEA>Q`D$`z?>jXe@P7k~%@6*NAVi;Nd}D|}WiPd&}<`Z`$1U^p7Dl^zx8xv~)a z=#j~;W@~gPZ|~#g1hlH#1H1(Y(mXG>JcU+KZQED3wyiio<4G7LC**?fWEo3h8^cjf zs3` zP6(eyJfGR(^YB(m;UeR5W4*Wtdgnm(tMj!lHBA{o7(Bx0YYQnkr7?qVWd`xo#Wxhw z`FT(3oz$@YjdF5g{@F(t!w`)0WCb+(F{X$-(85)3R%?{!WC!{wPA%A5;fN(#uS_4! zlZE~r=%2`EyRz_)stwMwtO+I=(wCA+phr8!Li8Q)zI$@y$Mejz7X#r+o8@(HAJ?ct zH(Hj}M@k`j%E4~iiA^BmbnXJ*AC7kvv+Hb^O;s>~l6`Ke>SGNbkl)>)-1%51Tsm5z zUH${EJ(NWx8JBOn%?$S|dAAfo%apl9x*v-~;jk%!K6D%M%mU;&dR+T&1?Tb1{fs}6 zD1W6Wn6tDz`l~OVKbv@?z2~=~#DQuPmjB#U9_>kaT&P9#^;T@kxfZ;laPf!ToZ3NW zwq7xc1JREv$geX6^$Qwp#{p3as8vKi)_0$*%(HCX?>UMrFjR_t{%mlQEs#ax&ZF80 z56wFo=b$0t9{-|$uY3IM$9&jYPfdqJi-xG#*L14$$DZ{b8I!PVdMz90@E%0+><8TA z@5w9_W5)2_pXpEEJ~wC$#Qf|#iJpRC?eK`UobQmAhoiL08K2x}9q>snyfQNIrYW2LN%JmB z(|H9(HYsPPy_S}i$9Db9Tl^i+V!SA6%lLYQ?+g)-it9bWk?o;C&m~nnY@-ie0Oru( z@R{>Z01^!M7Q)|(NAMcgkV4z5J_2m+L-8S_Aa2b?5bmX%;+4e_gMGWP#Z%!G28KR+ z*zOHz?@9;R-N&0bh^onLdhk3bM>wmB>jB{%eIWIHqR{RdLhvpx{4fYr;ygfBSA+?4 zdCDitmi#VKO5)29u(utuGDubr<~}fowH>|1VmGmb=BN>9r`|0yI>5r}*%)(7=yiPN z_u@Ijmq3@C;M3*3pWv%gW!1gZDt2vl^hj$w=Qp4gwl^UPp(T$QxUdAwb->#otGQE# zFs&{HvJ=9{jDdEBF|J2;C@ygMy8Y=MRiDjbnrAP9lHV>2$}PF`FH(r1QzW?KP~Imu z7AY`#uU|!CXPrEhFk-ZdKCjlF2+sZ{tGZ`*k?SlVm2gD67#-78A|*KqpI_bnguus+ zZ$};r8GHoSfM!C2q-8nx`jT^-A1tYebXhD$JtI6s;&!hyWFF;d#e~^Uvx%~Vnza~W zh9*h+*aWQ%ERJ1Uce@)S()J^aqdd~N_B|!NocX&`NElO^;$IARfcKIKJ4Ns7f=mJ2{rmvct99AA7W32~kJND)Z2+x%gnGQj^p%h1)7 zlXrJ;R#ixqlLS<#A|1c;`w{}eQP|J6aLPX4P65^b$^rYxZk&-PLCmB@xGNS27b9p1 z%7-A(__7k3FGk0$5&rDQ5|X2|WK<+yOKSQ6K9w+%BYM8#W`M_4?m_w$L#NaM8IUDV z(vXpzFbC!&wVOK>;*J z1NmG}&zv*Tu;U4qS!`D=UR3*7Wefk=cTq>B`WEFmg$yR3S>d*Z-!N1mo;}c3LAZ{k zUIL{(VFjo`i%MUgm7Yb|lcpUzgMIoOXs$t#~|1}?!nv2z7lu?@ts0j;5| z=RIuhTtW$LidoasPM*vAmBZ(kCMPFXN}S;jGc^hGu|ZJ_!L~Q``OXX#;!C2;G3}Y; zm)jxw_XLX#8T*DZ8jvX3O<0a8h>f5a^J#n6su>jEEx6Md%|16oR)+? z3ptm*Y`CdqnQ>(rcm{JF!tliBS1Uz1bX2NIrI4N>N{-x-e6W*+B}iPKvK@9P8@J{@ zd`jeE+-1n{-tUh)x;*g;RHn%Rrh>XT6HDbJ^-!%ZfBo12LJ)bQAB1=I9;Tn$T$!#< z)o^!zKda~JY70bKcUma{Tr7cLd?DSKECqVE7TZpK>?rjtB|vD0uC;-M#2cJPu&e02 zUKg>eMLeV2+}un)7qU(fT^+O^6#d74@ag;>k~emKFURB~q7SMaXJFkvL8?HbWqZ2C z4cIcgw z8cE4M-dzbD%gO_W>I^Awbu&DkHE#dw785A-;hovs2%S)W6t3VS9QYLtXY z`io0Y*o+Bj@!$dau#;WB{iby%BIbiZDix7Y;5LeaaM#^+C5(3oavkUnUF3mnqHN9&zcOJR(FOS3;Q$lk%dx~IJDB|xumHU zM7^igUbyuACE;mKH?WI~1W0ukm&!$gscSnan|N{%Y2iZKt?}o4VQobjA&dB3*b;Zw zagEnwKwDaB8C}nn3VHNsha&;$CYX#eZV^m1YQZBlE#9G@?1R%+P(N&UEcJHDqOcFo zMwxtWd9QI)@dqMjcx8WyV2n}yocgSDn)<_YZ|vhzvJ|$TJQ?y&4o$!*D{CW6qJF=BBz_vSa>5-TZMVb+_luDw($`Mu z0)|#>AVrvNCOCxS<;e5^EmC60rho1VNBl|?#gXFO6F&O~j@FjCT8c<|e0Btcbi zPe2E4;U;=<^p*!%yRN`>I|*oQM6D6BfA3ziU9W3KGBqxpd71>lE*y2H!xDm<(<(=v$(5kkcth{R{}~H(7tSUyl?=DR`y+kV6eb>DqrN(EqJ$1u29e z0nmogm(^BRi_BP((`(=$sqTOp&=#@@O~6h+y||o+eXdpxya#HkkxPrim#>I7p_~(DS2#i{z#|VcZmV zW~~Vo7In0@{6^mSS6i3>DZoFbJZsxk*ut5F4?=0--5K6NCtI`*yhiMiy{1GyuVV*j z1@b}*$bYrhb-3@pYg?_i30wY*Q!RiDYBmEOJQBc~J72n_57F6xKk~JjS7=F>bRLzT z=ezS(<`>iIOU7inU*ARZ2h7ykr;~4&S5R#Nj6+0IAwmYD zS#?6$5ZiSD)TaJkPl!hwLx-x=+-3*vUtW^K;R(2#2Xq0Sehd4rEjsQZMOTMpqXPz;*!8(LqL{rsqK_>q}~Sv$T+|M~#LUN@i$@jk)OO&Rs* z;lqbjs-{;-AE?HF2)Z@fLKR7|{q19HIt?J#7->%Q$EF(|Wzo7!@#`NCkbuvfdJ}Z^ z58sZJ0!eyDnwy#&opv+rao{3_TGnDQO$J&8oS{PbRznY%u)q{RMZOSTx5}x@t?17ok%mU->$j!X)f_aTw82c9%LnN$BG_8 zy=S;{?_M@?458*U2SE4Chb_Y3b_h_rpJnmK{qqt3TP9p5n%=WlzHaPY=buU2$T?__ z;(!@xzt1dc1Azo=)@Hbw_6`FMU z^z+Y%c)A4`?!+&f>q=zxiJljqvd{yj`Xq2sl@Iuy*N?nV+7Rt>0RJqYZ3T?fQJMf{ z!Q!UZu*^EAPTjJ6iI6Lr5EFugjby66I&LO%cJz>K4*Dwdhucg;N?&x2ToLPmh!nS~ z8q1STsl;e!ngoZ2UPPZwdh`i4VHB$6BW^L?w_&K}Jp;F)c%ir07mgRCpyxH2S^_dy zy#j})w!qpU*&WOq$$^0ieHYz@hzM*P`Dk6{}m&DrsGuTZFaY6p^|wMS?zc$U4E@Jr=Sc(qJc1oxD;@z zACajW5SMe+RgP3@T)#(zsOZF$43$IFZp8?h-Y>55|kAP-(l$nYZEG zcklqV5qaqibK3EslDcHgRLOxrMWk)S2xOGp%_CpFeockhjjiBItUI#e!BKYsD`O7l z{~#Gt>C$(pxog?hZ9>V=QVF@dP;=&CB5a45R?JCf;XcVfvqKWxGK0LTYD3S~sY9ew zm>95lm`mZQ)nW@7T$s$U^L*I+T9sstq({kj;_=!r8POpq0~vX;7|9IqZ81RQ;ujk^ ztUo+|xR0A#58eqWNIICqt%GzK}+3!(!5DbJa#xK!; zWq^F|``q9s`_f#n>4<@|pqSuSPf#*HafygX@eXpsI2YyCKQu@>{tR#rOdK%WAreFo zy{}bnCGLSl0WH&#Zqx&Hs14pb-b*6`NsbY%H`8GaH9iUDay*=mLXlv+ou%x71MhuG z5LOs;;17r9tiTH*%_U6ex_6fgHp*p@py3+@xzDq?dL7~-(VuPfwHHQT|H&HVD-X4Q z8KtSO4w~m?ORAm(zd>OR(00upgc+?>c5FuBK)Cg@9lm@zS#eMv)xkpdR%qLP%*jN$ zYR=k$#v~YYelYDKTRB2J%-aYQu(5o>tNYu3$=h-As7tr~bL6=X6FKulgOh^^nZ^Nm zTlidVXD(?r91j{1Jg~bBa)MTUuJbvl0J7Go$U4FNlastpPaxSaFw*u&K;*O`HGH(H>z^o5zEk z17UonAR$YIG$}YBAm9hi@zQBThXd^}X;WAzg^+>=DfOAQsQaXVV<{Yle!RqtokWDF zcck3o%89x>A~}ZI1gS!it|92g)X>x0ZvZNKK5VLc4~*od}STt z80zCQe?KpN&1yisi4^VN_tN+pz$Q#QkEM{{c5Z(T6+43QgiF}l0cn!wzTo&N$1*Tg zJF^0T?7>EIA=!F)`9Rj*guB1&=?i)&DTO{4{JK3vT{_H;@JWXOJV>Dq1a8o7|2`1s z2Hf@Hdk^m89U{>4v zQ$(#k@MzFtVo1=V(1V1Kf}$4nPAwUT-2*U_4P9h<(BE$Z;yf~kRwau6#rwa%%G!e= zOBv<4^wY%AUu<#x4mnV*Rr4S?>#_hp+nS`#y#(E$qe%S#Gdc=YVFFv%BvOOAyDuL& z^_BjY#iZX04G;`*`Q`e5w7n!aCDLIK(A1}ZhY93Ad>^2kGsd9Xf=v9NI@wDQbRins z6rhfgp)n(!aO)|wpv_P=Jqe@y<-<=ha(FRx_{6S%rT=!a7~GDXpc;%%O#B4Yl^`!~ z66T#>4WI_Cs6D42>5ANlxv}ia#H*O01i#_D85FeZ z+lG`CSHTII5{IJGnncP3BHqIXZi3b1^eh)huSPEepN!-L5we~o$-cQDEVgdXc)FzT z+8>0JC%{1-e>hV!R`Dv?P)<}pj+Q^)-e+lpQQ+Dcz^Y%4Kll^*HUfClJqH{`ZxSkR z-}YJ^A(Y$+V;B+&wZ>s2XF`VMfWI!GcAwi;ox$B2kvQJAig0W~rcZzvQi zKPpu49#@`yg4U}}`e_&4%RiSY^whW=KBmlnxG#UL6^|qehO}mci&^VJhGsk3p1CGo z14;=|I5@);bhsCGECmojANJRjaeVGUj~_efk3gZ{UVU!Pp>;+kSJe1vUhp z|2td2nWB*?eW0O{TnoU!Pt3x_I=Yas*naPPF^fo&tv)VhS&Y=Y!*pNc^<&S!)W;#{r z121}v2rWkOAfOaRCiDe7muYOY7_Rlx|{mIW? z;5g|-VmzyN>0hBZ#47^jeGcf~HCNcK_Tq+suu=qlg=!%vWt(MJf5;}nl?nTbR|(^) z5w>|cfxNR-c1PNi!kSKDT3|6}w|V2g6nr~6?m7e`9aLAxN(uQn#^v5MaUKhGMhT!D zibHN47y`j&iVY~*K5-k_=j_0ppLB)lbofx@L2-p<>kZG>$b!=;uF*`}@XfUs{PjD{ z3?g2hg(kDyB1I@Y`{FB-P8#v;pX|hFta-BJll2hG+=PJK=I`{XFb?YPt&7cu%Nx%nfF$&s?SmWDl z&hm*-ZdS-FmgZ~SKcc3SjR<^x{{D%o9#erso`m$(ps4G!9h%vLi<4kNNqu3mbP>E| zqaE4J(0(YkWM4DsHg-R5Yt&HRljip2=@La$$~bJh_N2w9YLOdR7V3V43;h z?nbPhpd!@re(reQ^i)dEykC#J(PC4GIQdDWudWGX&x_MAAaTGzomAsGXiZhOvO1LY ziSuvxSP@ouvRko{Esy^2$#~6iA0gvzrr>@Ou(AZBb}bVlofV+f2;1*U%9LvGQk($S zzz7s;L}B?F4z}3fU|iU^!gbUA$wkrttPO46J&%!u-il`Mo1sF5gU`SNaQBDs`$O4f zSO^o2nqWRqQTc3AR3CHUTiwG!kJ~8zb_mFWtJ&c?`h}vJ{?IhNFj8cvPMvDPZDkKd z6ackf+e&70b0A#OXP=P6v?e7+g&R(IGsU&r(}p~sgQyuTGn?rj4^yY@vV1`_iIIeH zxZKg-f{*DM?icoSjlm&;e}%FRtRMeWSKNlZ za^nI&$vt{--~aKU>wbheLMr~rCVP5Vt5h=5v6eT;IM#D1-&4`z$)!`05agPHJ$>a! zpM|FvcCAt5(}N7uV5W9D>U-(x$`ITSq6|Vrqj#HSR~H%*;o(vSti?Aj$OOc`C_J8b z!`WT8h3lSW?L9v1n9ns@%5{3Z{d63gb2}$uLk1oT{X@`qAYVFPDcscFt_>et+yu|9 zKGImyGYd6xB1i|*&5t79WE!tk%!e?3K^+xV-_7=H_+Y9ca2hy!6NbG%f9|i37{NsCA?Uq&qO=fd`rr)YYRaCIOrB>u7}MowKkg>H-yNS{ zGLss(q@0@tUc6RlLDU{kD28h;*FAkd1FaXfJMZkl4eoip_5J(Xm>%+zjaX8BhRvzN z3^L<%{zv3TV3pEGOJIzk+wBe&FANhIwd7z-t&KLEugi;YNS_G39L=K7HbVfeArMPn z{^-=K{8BsIFWa}_{xb7H8JUtvYGpfD$aT(<8u#}@9;ULJFnGEd@rmbevjwZbtmvBF z6dL#5%|i_7S1WUFG;+0n_$i7Df_0XDQXTq(<8hSXIm6W;i)%HFEZWH;2}N+zFlOv|aMj0IN@uN9K%F{z2ZtITG` zy>dTqBul+EJBSkHPb}+T< za@3C})2;a>xlhD=-1>nqjK+_kynvhFlx2b(@j2LJv!6`G{XsUvEZOaO&BQhI`fyT9c5*r-XB>>! zKPABOBG;F{?p8(ei)N6n2Ex@x{&6sh|7ju_zlY)591fWE^mfbbG*CNFEx=WKkKy*O z{P?SH>tIs%LoF7Wfrd2wH@Bp&ZH$Eq!XjBKTfcg)m{F{m+kfQsYQlu~+|@1wih0b^i11 z<^xr%fdB(HwkT+Al{0|(bZq=5NL?y-WZPDjYER|W(yeaiXr#k^lbL&A3F)9JTm)ge zk7NWMN)dvxi*IinsGbi1WtKS7@7r((dOP>BK?87ey;SY>cVcZy>`bC2l~hLlvMcj_ zc}0#8XX1eiTv)K|s+v!^e^74Xc|&TUlHvO2#u!n&9ZqiI6Zq|pV7xG@r1%@@g$MRv z`?hbQ%7oI7OO9E9l>G6DUa@pn+59GeCA^3P#Ey~c%kRX<^rW}bx zy`F>%QT1P)FkJq^q%|D=!z$P{X=iq7;>t# zG5~q`%Bewgp#*J}-xcKc#2DMyq&(6hJ$Yr%JRVS=2`z~c1A!Y?P^4Cr%hQRnFc`o| z-M8Z@jO@Zq*SnO6Mmm4A%ys$R*!sodHT;)z`7WnZ58Tey48q_?NF(bszy$Gdca|XZ zIf}5k4L-F5+Qd@w-7<66y@uKZO3Tw5Ogc0Vyo+Vvk>8(B7^jPqE+~z_$scJj+(J*k zS)fp7t~)p93B|NO=B?Wm7=E9C%pW`osqGL9x>G&Pdg-K<`J3m?EeAbSCSUikP$CDZy1#S8ea?j)uCcjX%I z79+WU=NnJZ7R53u?HesoR)Aowdp=^1LRC;UICwGsR#&LO1%SvckYRO(-Sg)+vJ8oLw&~n z>MNEe=&p5-3ll@`ZTrtLzUn-SD79p)W>jr(6QHYh*9sugz<)hn3YBLCP`BzQyo~&hu za0TmH1-l3rMQU}}gKr87vo@S7Y-@SPk4op_5il>W7481e)j>&0?qtvl29Qpwe|@nz zS_LTw&XL~p=-s?q9*H1TbvvTfD;w@qXEck>34(0lprUc#zIi_S4To!|(k4;kZRRiW z48Pd}$YoZ?w$m^V*tiEOu^c7p|KkMK52d z>rxflP)c?^p44Qeu5?-P0)Y*J_~Y!(+}ID5VHsS-Dx|06*te7GQfZ7iWoSv+#2wpU z2%>bMV%e`)K{)^0Cu_9D?VQ+MrwDh;J+LT7>SXrq1tMMj&?6uHoilK$N=Zju@P*R$ zWn)fEj5`X(j9B^~ot1~N$Bu3@$wrZ&1qeb~#GODt;S`^7fKHP%iBXpSi>RPyTJE_! z^Q;dz8bj|2d#8W-aoGD9nNjYuVeR$D`>~O+^TNI7H;LKqIzP5%8eNRl<;u<5W_`o# zk=tyx`LdMLBDW19Ge*VR4GDWu;($hJ7Q4oNduS*CH#GP;XW&S1QIu~ZB!nSjK5q$Q zITfHANMjLli=dU|hE-_@0o-Gdv0FmW8o$bXSxv(|s=pxz!7o6yq%38OcYBk} zF`EJngWKH89s^hQV>^+ep?Q^1x++0{hV`@*S*SNGlT-5)AQ4sk2>3+PXve!Q)x+j7 zcS%x0UH5M_Q#QAPwBsDb>u(kNLiEptZQ@5|sT}@%6S;WsP^FSim+sUziz^?h{}T+B!>#-+nQz3V)~)OS^uje%_70cMPXOc{1j>KxNrY z^B>D_^76~mK)*NG_ak(2{|Y>*bKocQ!X2XRC)G}p#sQ4baEQ4IVlx?zJj1>1Wj8jA zFl=qKr0WK(VlzxfHwa@(rtHwly3>*I#c0L7PSm8osy(T|&>)Sx-nZf8rs$S~QFQAf znDJp!QsZr>rIPopdx8HjrWk${T-}?$|3{dFz2Qgb5#^4~zr4?za~GLnj;OFDJXLrZ zkTK`p;^%_%i}n%ziEeIi9b>e~t)d&$l03GBLkX!DMEqj+kn;y$)d%EcOSWD=bNZJV z@~Qj#WUNUIusqA_mlOAtk=9G>!@KmU|9Gl(OONv-bEG{^%C2PLinwQJ$Cq%eMlR)~ z$1z1}J7P}0^}YbdP?mJ96z?H)^(e1Fb^xz6L-(vY#cSUNUl41jX>rM%p0YU8CK|%I zVOgewAeNR`3HkiK2EL_ZoPayqb#Q4mzqI(6CfpjV$Q^y!q4o9all?uVt4pPcWBENY zYz}3cEN5+AY3+ZQLc7xer!3yFm(aGlXw(ezeWQd5dc76O2AH5Y&8cM6i$lKHY3EF&foip zG7`$7zJ0<;AM79lX~y@%b>Ic%E887rSV|{ZhVF~j%Y%t)H9;SbdYmgTGKmPR9JtgC z^wAKEQVMgo1K0L>dpkt=HHgDJ!nO&)e5T)GjKW41e70!}Tlt#oB3F#R-v zi~QAbud)dWRwQ$U8XJZ(s3^1+FIhbZxfuxry>ai+uX1=73j@`SAJODAIEXAkhqN z^y48JnJ8qc$~3fS%NPJDGz4dkjD6!rs-ZV+(vH8l#pl0PW!+96Vyxbz06o`MMW!r< zNT$l41l7v3ZAmSvT;^|U>tg+7!Vx%lIcKQK?&=|fol=l3J}Lu>IND<*-y`|pwQdB! zv;(&p+;mxF;|aPL0`AE9jKb~p(P!OH}dV_vbW?KwbUBru!yAQx+|;r5Gjo~v`Ws=7Ru#)kY?2L1QX zxvu<(mVMFvR)LzjuH57ut0`;~U>~ojrQ-67$7~T8NX|EG5_%NxJ3;E9x)645C7kys zZ=H?#k-u&Dn=YT!`Ul>Sg#2sGF%4veYz|HkMiZ+qL!n^_jRm~L5acAfT)#yQ!9){z zq^gTvnNDnm%NqKU{E@Z(lT4cTwJ1?FV@q7~T=E!A!#V2iVw$bYHs(wfg za}?0cd4B#Zv_pNIm@zjL*j2TX3g2EH{i|##K5SE@)rp|1N4|fbjJ|4=u-(jhuWnP7l+rLKR!9BnN*&qw3zm!(5ayVC}jn%^zScKPiA z#HFQB^{v!ul{^A9``&EhZO+-rY+zC0jG>k>s?E$FIx-D#DwTa-rAu2b6WqTWD&>C~ znS6O6VUuO$*C{vtW7;j_{?F`Fb;^9xB`*KCYxk+vA2~v92=O>)8$(L`$o{XK^CH3B zLb;Ndq~g4{(b!B@?!39PT()MRU86}MZZU!Scn(?YBKEzNr?3g`32WJ3L<-fHRql_S@F$oxcoJ=Jq zf?PLzm^I&g)5nZU`$g@=YQ?>Se6J+u+ppN39ho^?!r*^cTq#j>vrekx>pb(}bK~v0 z6VH<6G{#5n_eR@FsqdJQ}jxwJ;gr!d)EH@3y}iUff02o^81^-Qai~! z_8&hoe%KkV?n&Bfa<~=&KupwzU}M7B0c@(hgybn(*__ne-rqBPY`8KNk_f0&X43SO z%7V~ru|xHR33)Hf92vqWD3G)wq>G7Wia*@!iC|je9>@CRktY#lyyF$B?HHigsU6c& z7JOJ$aLZy}1jFk*ZT~7=ocI{Idu?{ssmSnc^nAhbErq8`ZshgN$$v-=;>Z|1x6Brp z|E@2yWloWLOWk;&^9S4gGcc+OtUgBwcK6B&-gQoR{f@5pNRoU3oBeg!mL-~x4=kHD z_Ri1G=L9Z52#P|=!>FasZz@ae?wCs?7qW523)=pqQ4Cv?`ft$}R3hEf99Z zr_9LIj6eM{B1#|x?&E66%*Zd>T3D? zdt9=J)8HZRD+`u>iB0VduJ|69+m6`d7mScuuzQtp~;POQW3MB?Fv7l4b z)xI2p#aiD z^80mkSia}1RGKs@7)GD*Q1AbG1@(^nb&rC8>#bDt&GgC5QFdh}Gm~p|7-j^Ot*3e% z{d*Sx&UQz&D7vpaHnP?N?_}C&MYYJn!V>YsHKy6+yfAeZgS!}((lxTofgI_c9eN)7 zKmqRTb|?g)3%{sw@l2{c+k@J*I!}R1jVd5rE6)A-?nFO?w?WVQ%`@8 zM|bg5Ot*c)vQ+xcfEFJ4x@#|)ZI=+&kPR8Ddo`P_UNk|f^bF%c^Q}s^rk^AA)Y1pZG&j9C zI*Fz``&JBp*w<;>i;WEzNkh;m;{cP`)z^xJX4>CAd<2<^S2xsYwW7OE#%LH0cXb^t zm1!}hq7v9*Sm{-^v)Wp%SHsbeTSR|NJmL10r6-h&d<1womBOi7{${{_j20yHcUUQc zEq0T?K4L^&$QPB^Z~XAEYKlal>XXxXS@b`w_pw#&7~G0;F7El^C-|&BGb)^VPKdH0#+goIMnI>AA_ah*620z4|p}ibfSG$$!SBMmI0&-w9|Bg6X+ZgN|#=iDk6yN8u*LQ@yrQ>BFLs) zJjJn9x6;+X(8aLr{*&wEkJUL=gZYEg79|W;^n|~gBH;6@y&NRgp}KkofAEk7JML^M z*z(z#LQkCZmcDVr(K#g`h0Ok{8z4$0z||>nkiFYfV>P1TZ;#R4xm+0PCuUx~9n&MD z#k-y7CFZ4Pvm8cCz=J!uxIf6B1n0KKz04TqY(&EUHTw-;#JH>lowr)hFcY9V9)7GI zq9m>iW=l0sTX@b|3_-)RDg+#Z!uc=v)K{p+ZhYbAf*|~nFW?3%%w;)yM#3EH$Beap z=g{1Ge;<$(I7!sxi$mh#;tN)!mj{@qNzIhAv;jWq12?G9jqzQNQmv2X|9dQ|bPP57 z9pMUr{Zav|E@@1zZ#`???`wd_sadNI_o4r*6%&QME)g&*)vHgzH?TD^tj})VV1l zcIS#i$T}oB9D-}XlHbmPjH)Pp;l|h*c;iQuhgfW|$-B%oQsYfnHle!ct4l9M)QE01 z3wx2yEC^AcQ4rix*P~z>;%~hEA&hsSTpe_&q(uj9b~!lj!x#^e!LZPQX+Uod6GB#} zjGDH}OyF{$PS{pVL4qP()Z*|?0ojUaVg zEB~jX=8CxAMaz2f2x^3r;8vKbzula%;%di7-5&#U3dCe@(y>!(QccCDK!FE!Z*OkW zS{X;&FaKi+n>j=zC3TZ3fzBiOaEbxG-DrRVl$R$GD*qv<^;O5O^(-MS<$V{b8V0?g zY?O zm5&zt@`70}>lQ+wPHS?=WRqQ<$=lYa5)(~!sudXb7pU5e!kP6e`!5)@^AJ*LD+KM| zzkf@r)i4yPxHL-8QTE8gfJT*%2>J#bU>LQ`d~aIFv0D(R9$=`t{bTSznrN^m;&n)k z0OwcDjaU<;#k@i$+DAW$6v>hV_=v39xm|yXbc-^=V75L({`x;V^dDbL=VGGdo~H9Z zoj?}xVVSkzk8mdho3u`Qu^)s*MvB|}3IQ(-dVbUQxECgj>`Nyy@F9pxv%wCV-nrcg z88EhaX=D>}Bx1 z-Tm4{S1;8KtyNyI$~Q18)P9?c0Wg7>`Q(VOa*OPlixt9#W@Cm)hmUk z-K6A)cxgoku);4^kCkC_??ZM%JGdLyUl*c<*5luqtbK1;XFT8aY^5abGib5Iifq#d zh&?J|%$PG|T-AI=^QqKn;VKXH*m4Z41KvALH7rpp9C>EoJN9s^zTDBSF3J47H66gk z+pxk>&}VVlv1#q8SA{3~0#y@W@>9+Z_bZ5Ax50ft6g1-;jcOb;jsB6(c9)H>-qJS& z>>Oi+?rT`=r`x1$gMqmGOC4e;XO_3aFV_GntBiXSBeTwEGvZ!-nOpljnjW% zGSRG5cc}U6v&iI6@!ez#Ceda)=|ePoSSoXX`-fcK7~O|6ay4!+!{VwMYp^tn@x|2NWJ!WYkYg@Kc3Fpq<_<)6lIw3$8$pNmc`DdgOJ)>unDTm-| z+6%>vZuoQXLjlwzi_Bc|$>_V2M=n@0_ai3aN%4B_%(P%tF7C?=Nom_sGFwnEC{#&} zbJf*#!LW83R=+$f!e!)S<#TO+KhHp$kiUF(3ns(;QFL9%*0#QHaxaVC z5!ppsob&Wix!SP;5*C6?=3q~f$m4<1tNwlQJ0EKmkwhD%&$U00DL47py+>}_2x9A= zYc3Jy=vQ^Ysqg|Xdqi7uT}Zy_TxO7S*coQuF|8uetd|6( z{QgnZpWNx=V=?V>bl0(|vzIjMD`ZpA|oqB z*<49NR@r+*$jFYY8%bnkWV^^-DO*NGBq7|CSw>b_*{k1qE341<_gDAPb>HuEUgvdQ z`}KVN&Q(t_;;BF`Z=QQ{0amV8`^)XAL={LEA+!z^^}++&b-j|~%>&JhANyN{z)&H_tb5z(Hw$g6!l z)Y!@Bd`#9sadHBA++dr^3RI4ST0A~Rxc(b3PSNTgPBrdWTKXd4Dj4U%tbtAd( zQ0#5}6=yLVYs+gB1`jTWs|V_hb9qvS7hBMz6h@y3>##F!%m3(Ea|FoKy168lM zVgHtUBy;{I$*;SW7{oJ`UfB(Z+bmIACsI131HW7xgf5^pineb4{uG$!rvOHw_-wqU zty8PgeQjU!^V5`Vrm9~95w4T8`qrlp6IYiXx7Pdii`wTJ8I|ul!bIWV^uJl(IuC40 zEO&flN*zWjcE3RJZJPObyZU0UstU{GuNH(YgR7&~>)*{hCW(Z1cEcr25o z$IlAA&FZ=SeAoZ+f3%in}!TXH}EDAng0(gs>NTpV?;(q#;RDtm+r4Zsxu42&w2SmfUI#aWD3M-G|0g6u{4IaeGTwu$>(_R$ zBl;~{adv)_tV_l+kKUoBO2oWiK61ijbiAm^yRF{vrUdHEEVH{lOfx^H`-4h(;dnBL z9S-_!Ln@ro5r^E5b=N&E=~YP!44dR)gtA+^AsI*Kf~x#x2+C#xWr~Y8Ai+cg17@`~ zyWc3W!yVab0IkYnPUbe`F;UuUbT=mLPAfRL806Nh3r-3*l`3RFp&f8X60U58_c&)DOG|h0ThkjwRR)1;1T)&=c8RTAo(0p>3kv@cxFzOs3H0k_+{K%>#M|?y zCq}5DN9C~3gY(P;MXhYR$`^_{B!+>~Pa)r(ExnpPV4Xd|b@Gj@OXq*{9+G}=F;=qS zTcVSBMtC33e6`eGcCqE`)Q93-%TJ(GT>?!@G64pqwM;=bMku&Gay`i;0Xu(494%c= zJKU>2qU9?zwtBh9+c833ysKl%^GKE-#4QkNTZ2Pwp_)q=qBk`c840$skzSV{g@$fV z1akBSd%G2`9J>&oISAPf!mxFYfWo1IA_wjQG{nR44WKR~B}{VVbLoc*{Z9dYusBp_ zs0jMc_;pWH@26T#)Zk_>%?*6t30LPAY0oOzEq`~%OnIvKHM9Xe`(dCx5YclkluAa@r)6xH|3QXh-q>-&$b9t=4;>V2TY@;gfJeRuhLO!z5odv`LCeWL=nnt zi2-k;i^pgocR=n)no%0r%=v=PJBu9D{d9q1+- zCFA`dhZgk-o<$jFk@e(L0i!Z7+VID|6oXQu%Xjl|LC+n}ZBTvtbT_>U3W}#dy}1(9 zHRajvoYGxSea#6bNf9(0;Fyt0*FhEXYDCfo{|nUpfgC1iTx-@@B>t7h^EL1ehBbw9 zvIuB{S` z#y?M6m39fTZv7fuZ~=RKG^R!2!9=t3t$aZqR~dcBo`cJ^Q%~C)9H=k~YGTQ6ySEHH z)^dtF%XjNn?@mD%yk1k*gJ9_r;`dNKQ=%!+Wz_`PEQs0ySg_@LB?B)lu;{g^H?&OL zhGPw0frA4zjZNZpV`v`Q)MKS3n)#!!g^KHQbDZv5ax8h`zZV(f#Eg(9_V4NL2b2_f zi{OnaQ-xHaI5P^a5@d!{O+1x29?PZQMzh^KDYO2(|EoH;KheXca0PYEj(C;aNW#sLD+W3J8`o<9m=gQiB}NXu@QpnPZ`va z3$joN;u8qGZRhhIwm|aV*bVPd#Bb-_AkGj0PIDO&zMcv~7E@6EAUnxPQ4F+wb%Q>Q z^F18gs%S2<`eu4u!=}2oYSc{!COXkQz%4RXa;{)wqcD$W3NFu@9{U$gTi7BLB~SDd z5IsQFsp%Eix$u6k5gQl3PGwy2kISFMMz*b4GKS(Ox9ZFtPFybvuJXdey9sH)B|UUJ zwr|y=jfv+0z>;s}i0CL2^N`Xm7vq8AWg9XOfO76Yz?dxahFYJ~dp@3-vy`7e&P$rg z_ymE=Rg=77(|bi?m(Q5~A^onu4Pt#VY+NvG*f<3jp1v}*m1h1Je^}48lF)|R1U<(a z^4KGec{iFbkc^hC{43nYu=!ck2OX&E`I?)X#RE%00PLN_vajnS-PdsZHZ{aOTC<0V zIhwvw7ZlG)b(l?QVBN}iSqsHJc_TVrB$-=~|JCaN6DnoI;1?LHq*A}EUmw#BS}!2^ z7@h5>coD4xt5VGyn&UHPuI?dwzF#OmSp{gFB8-9oDaC>ELS@9Clbev?J)05w^`T|s z4_T8+_#!&IlG5Xa(romjZQ>0ptq(r?0t(_>TU0Jy9*TrSefavX*Fu+rhW;p~#-3UK3K{(Nc@&F9`vnpBkafr`X>nEQaEJcJK?%}b>o z6h+B)~k%+w~sEF{{HPYrfg?G4}7WDpC|T{R}J3@bI`Tb0&8^ z*vDc;Bn8OIu%kwB!=`yN-XLPVgs*ZkVJ?C+oWgk{J$o;VCjsoRUUH6q{A+6JP1Fl;otz5iH%T)lq%t_V)sTQl>HI1|!^0tKrd)p(yf67}RgE+noD* zUHUgCRK92p(gJ+)ucGfkrpfNvH3-3-d@t3?E?w`5DoF-b5Prj=3i!1H-07L-K;${g zT<)RxbK0zU>5sz0C$gC!;TmG2dL-*&*+!)k3fIu$kA+-;+_@Ug&{S1&t*T#f z3O`y7E%N!@4{Zx;>T!rB<|wGj(9q?h+-|y}em${1UTQg4^}JLA5W0IWQRF4_rYvbJ z%hOYWn@!CWu$%WkgVLKT?tUATKq;gUWEnlvEKCo1S+pC)f5Sy49WnAt5YM~OEU>}U zi#A7aZ=+zx7BqCAnp#&vM*F-j5VU^CTwELPtmfrlou&6N)l*bq$PTy@R_v`r$y=}! z;N(Er_UfR!F$mhe=z>}}UBLW=%!>N$pMr|bG6#!nJr%Ke+9u|7k$w~1rR}6L7h8tQ zz`}d;d3yRP`YCLkBvSq5W%nIE8fnrij@s|igQ4`^Q;l1l@;hg#!a>~s?WH3Wm89a@ zv48Jva2Bg~u8iqIZ~>%3Wl@)Ls8|(V=?Dh6(xI@axqK|+wr!9Mx@ACkSfM~K=LM+e z!Wx7-cZUOcE`%6MbS`U({yk@?EHX$c%(6SXlRzEm!DuakGjR=sy$`B#dZLSNz zBHuIy_lg>fT3s2JgW>Q*Y|atu(SPR9It%OygRh`g=L*yfTMc?cx}Y9TA7EOD+m&Io zaExwYW^SS?S=x+vT3sq6$F%qXJ;!ZGGg$;mVM?*QU#vdm zk0krzyuII&=6yFu1XQ+AwJ_#}&MHPmU6^N6y7P~j44qYH0&@0AA~zk6lVj9V$WTKM z!=2_eqx(%VeQrsPgZO%b@A0hEBMbX*ib`nB(dh9RWnE#u0{59fEck|GzBXXt%+KD+%BViuTT%;A_3@jfhMTq zi%tUL8hPqFsc>Pp(tk=(A&zfwfu|eW=OvyxK_tqkzDY4@jFPZ=H|+JhJnb=HM+7{@ zYw_I@r|%-?RQT_;@NS0TwYM(7%ZkT+ub&|VTldPSMz%}m$w6fR&;`$8i)dz|zb?Xf zev>2}G6rllU0FNjP?aI(ym*uD zBB0RN1wEYw@-l#s9w-U5G_BO6y2|#=_bB8U++hE-?b_An>|@G6oqodg0o@*%py(yz z+$IL<8&Z*6?r{Nd+!ZO$s6cLt%v) z@f%l_yowut=-^%QR3)SH`V*G>Aa!=lt`N#ZJZJz|8q%3OMU|{jMVt}@4#7@019}us zfZ7p@TsRy|u@I#S+>?|h4Y-!1ceXV^ww4C$Scx@v1S^Ur$_+3d$W~oJ43|ktzn!QT zH;KtXQ5`y5R0@6%L&IZ2N^c7vaSLKdnPx3gTF4aQA#0{|+-PmsqfbE^nN_T$no>sLK=>#t(}n_d8W zMzbQY-ZOwU7Gk(1#chkcX<~dyV4*Z1@#)sil6}abfL{Gqc()JaDBnszAJC=V}w^soT5| zdchPJNv6t~bo-(w=y%zzhI8W(%q1xV!KTT}uHkk0AlMUH-v9Ru$+!}cjuNK)i9T(- z-%3(}J+XZK)@dFT2?3S(4jUo1HsifmsKqq!3VNf4UE5GVD;87H61{V*l*mc8{w_}Gd7k9x)?xQD4;HzA0Hg3jauxohl1H| zHi{ugX2M3&Wt}a7xzWoSF3<(KiXI}(GPsI^-1R5+xu}{?P11^eQTTRde<5O*=2qEA zL2t(e!wE4paZIKh@Y!~1sJt+r-$7;r9o4RN=Diq|dKCyq;VyM~#u%TuRUZenyeuYm z@*w9Bkb11NR#WJc1)*o?{6K)QQB;|iN2}m77xpNDFl$L#095wynJaitEz_Ms4UoSC zEt=Js#nD{;(8E67WCU`jkfL6m=<*!d^BcQIJ_8HI6S!o`ihm146y7mf)C>uoJ9!SZ zrr(D15TpeH3s&-=a;b0ByMb}3jgR43%wXDR%$H|Ipt*tMOkl!a9t9h+LUiUZck8^y zyKOYR?_*dgBW6D_;Pe@C-TLtH<1UdIQ4$U60$T1b$B$wKKwkpnnL!;Fvb#zNUxpE2eJ$xY?a@cQ)S48z{gZPttif|C}L2rXs>I z$NBZk7n)#`^}IGZXF|YHJ}(V$B8v+|dVOi6Rd$@OZr`-V>VxV#;EF5)ngUt_p~7{; z0}tz{K4g+XE7@b*KOelgJ16a*S1A?=LMEdgZOg0hkIj+*B+>I8$o7C&){(DaihW|B z4nCKyh34$ktuf9i01T)KJb7Dy-=#I&BO?Qm{0~zv0d1A5F>!`0U$60((Q(_wv}L6; zgb0USHyaHNi_KiETdkdfTsu%MIFEPi?boHr#)>~|;6olXR|t9(su#D*0OSPKuxk^0 z-HchyT*V)QZT~fYKXg+D?Ub!Ox%utkEk%sCBWQPwsjHsKv<0ur!wtr{&;N4%wIZ|o zvea(zW5d_ujiMMDecMPNmTFu^Lxr{OJ%dotN=aHSN6D>F5Y$ zx1}tM~#T;EY7E|Fl8r*S1=6{J~N#{A$! zO~DQ2YDf6*?`FLB4(TuTKwz-A1HuU1XvVAhhS`hMcFMy%)FQs-OW6p?0gDWI3>Snv zs;OSENP)J--870k&_)U3?)Jx=qlFHb^A%N))RRil3<4#*7v7tNS^yfT@LBchz#bMr zQ?xq*cbXV+TI)gwjG`ZINNsTtDA_+*qVxt0Z_8OH;h@4ImB0K7+`pDr+Ugqs0CT4x z;2Lx@k@~ohA=b*P1Np6epvAcRnDp8!2qCNdJn6&bCm@P+F8{7?Sz;Fy$oq2^T*uq^ zrAh#1$SUqg^D=PoKRm)U#M~&^SHAm#8DtU`O`-wj5vIUmR?s;rJwf6C4rU2y0#z&+ zjHQh3@iqO9Y7^F2ZnO=u(EGW0rz)?@{;GHEo8NShG8T4bf zwhr1JDhz+DZW6z)i+PaT>>=cNI^Y5c)2$C+s~~`_Y~Y)eKewkTsIzN7LH$xh$x;Qe zQF*xMYAJhH`})!t<`O^^A_DJ1uutvf)Z?mG2x33|{={@W7t}mBsx5;W*WXkI3cK?87Hqd|3qoo5 zV@yLBCQ#g@OGW%=PuZ454+F~6+X%$zdO9$P9Fbsv${sV61NYd5FyqobWm->x0JkP@ zVoe0i`9mBv8H%ix;ZjeKkbQ~HA1V#~Q~U^Gu4a}GM$&&5=9qd%;0AAZ4z2nRbWKB1 zs86`F-~e3!`ruO_Ao!pu4O5_%TYl!DVp=JX-#-mnCghU>lM;;qF2-rR z;$*Dg;8)zVH2& zadTW0jjL#^mce#0p02J}6SlTCjJ#|0k#BuzF1eT;X_yI_!Q_q0~K;_PJR;_XKPy8fW zA5mAc>->JW_m;F^5`Q(>D60AL!9o0G8*CUcZo-r=We_H!GsGLAcGBc0y(6_T|#v1(3>|0hd2kM2SQ;3Xwygs++b{-s#CC% z=K;-AyL#@u3+Z!dlqpS1w_N>d?VNJS4!|tX7BN$NP4~?1ZIn(-343|W!^jzB+|SM;66she)V{*{CzgQ zi!%C?&MOIfHD-(61XGi96n^Iglyg~=K|XA1VN~>(;kd-;8)30t;;weqKS3wtI_}7( z`0q;k6DPiW-IT1GVI$*NS=ct(Dc898ACjXM38*%Q7ySYaTJP|a@!Yxt^pQkMpW)-p zTud}%p^TFUq4S|)8UXW{6-Rsv1jdx_{wR_6IG%Irk8j{4S#<|sflyE2?<+9y)Jw(| zyiGoe-qGW?vVFKdLSBP3mBtAIV0^%efV&ZummM6o*!aM4>$VXn|0)jVH2$+Vv@&eL zpSy1ES3~%f3u5oQJ1PitcKzu+4e^IEvV3H{1Hs-7a*u$<8M!_KR|EwIq<1@FpFgC6 zN(Q}IcD)5dFzS6O#CXe051?u7VvK=dl-;xz+AY7SK#(ihi(<6-ZF4hcs{VeAnyUg? z5}vJdO?cq6jZQHqh*W@5PY|zU)4PRhi2JA^Kh5WE4iu`uCqf~&T7u{9zS5TM{G7M% zG6?XYrhNp3`Qm6&lmA#qB_ZxlLX}Cz*97zmD0|s4Ujr& zm;SxA1obUXff@5xA@2qP0=RjZFm3Lk>Xgn$$U>xUPdP;8bC#@ zKR}^_`s-2rgkZKv`8h%e04o&$GpuHUu+2HByXt*r1l0bwr|wwDwviB!<3Lb07lp7g zRNv?rwah{jC)wz#?Fy2RZn4_a4{z*bYM->Pn54PIs-Y9L^c@DLG zm`s!c>ZBJn!wCKJKN_f1-vol-*o_NA4!{!d1kOY78>h=b)8Zas$tLTNDN}i$IjJ&( z=x`iy2OYwnf49ObcwYXWF(c%p8yP zohw)IeR6trVY>h4oSTLxbfrQXpkCEoKt;at&#aIbfRNj!j`(U2l@(1ssbRD%4!k25 z83cbT?Gq4+cv@+aIv1?&_2H0qH3Ipyy5NRch2xHFQ_f}TjAX(8Nas#s6aYuh91RoaZyE()BC(6Sse>(d<9^}aoCpI!Y_&r{ z>~7l6W3n?(rkKCuAMiHlJ>eQ+DZ7()>eDZ|t2PF*PsIQ!t$)0sv~Bn*I-5c;r-|YfsBCcWoGI8%PO34XJ_rv>K&8D?AS|E& zfZvt%?aDMEK$LEPdhq^w3-)ORuZd=n7cVc($280o9K;fVSh)G zAq||vW;*T_<7F#j0>Bw;bIsqXU%52}y3{{D#tO^!RCdcA(UexbVsPn|Ic@_1^KiSBYgP?6zM8&Y^!84q2?Q_z9Wj{wUdcQP6TVMH#+$cd2z2=>| z8k*?E47$bFa}TZAj)-xqg$^vVpFBMf(<@U9`Fhu)bnW6hjabOWSk4(z|#_21-fNzCI^H^rz2_ zql&?Z^W__&qgw(+P1WyKPd(hN>4JC-b%BsqznmAOI&7R?#}CMV`EGlv_YZS0jq~Y) zYMVyVZw$t{%2EGc(i4D}p?b-DjU5Xe4Gfej+P@Ny3OJm7cGg#fU;>!^k`g$HiSZu}hH`RUSnI{PUEY+}Lhl-o+< zebe0BoFV$1`wrg8l)V?>yIeu6ic&{~pU`QNq>|fk_jdbDWBBfvF4_}Ip(wv0|4>-} zKta*rPA?}CXWSRt6UgXcoi{MAe$js*5SSYAq) z)W)O>n1Ys7zcW*>0-A*#JV84}BGhKr6^eupW{obCCZ8e(xV{-q#WW5r5KqS!+2~0CaI*26rYQnXo<% z46V>>piTzR_M-WAiHk0PJvixfWfvs}YNd1mfQ(Ops(!KP0GPpo&#B|R=AiF)%V9|e z3fw)JLX?S|_2%#;TWFL-FCJOCU5nzHQBxk;uZD zmCMV3gdY=;n)|A?kDF}ba|zvimzLJUU4?iI?z8eDHM3HGM|$gx^lo7t@R1!M2pGne z+}}|Jpnn;JIe$}M$Jx8}s5+!=QeXFmuE{pBSW z#jjAe*)S+3=?0*@k0GF56$>`QLsSyuQk&6xHq*cepF?(mes4c+g@bQv$$%R*h|rgA zzwmV)QZx|5so9{so(xT2d~0|YR2k(9-~Fdx>VPtTnZ}y~fuG+&UyxS`g=wypeYk$` z2Ka^s0&n&^dvP<`@E|_ofop?n+$1`1pVmkxem(CcyY?PbF#r;3tm=PnwDP@LFjZhzmU+@R~QTAd<0)u-C0 zB2>e6@2>CPUGM=NiF8bYl)K>>`vA>*y>OElG&x=&KDoCwx&>X9xAFInx4FnBEY5C% z-pu?9Ty?Jy47cy>%b|Tig*54dHKE9x{FiH~4+s7ZUflV}kM5CWv}b)dR>4rktA=Q_ zr{5fCi)`pomfJh1ZC(?BE7BDy&-Zt-zu;(HwAQdb+%$f$w5>49_JEXozXksj2bFM4 zC5om8Lhj{-1YH>{En=?3y}yv8$`Pi!0Y>pi^2R_lBnONCb6J-vkcB=KIgwwUZPXaX zfSI0qr+(;z94Nd1!Le>&;hZ%E;d*O;ve~WDfxW+;Wa=FE9ZP{LHwQESdl>}QU>iet z^`V%3_v(g0Xl_lPg{9~->LCk}VfVpaum$DD#jlq~yi`gr?*H$P365EX z(WJlo-X$6qTradCNJ}ZmOE0P+fPk{AtGzuUPn+Wv#otFASrro9Pte%gk##lx9$9|B zD*kuU{jX!m;zS(HSgR9+tgkKJovCyjHNn~c5)r}Tx5kYVpnQa~Xm$cbgU!1)U1ADT zxwVp#lG_v`1Z>+8oqH#Gl$4$K;fjXRGZ(3@o;X+=8jh{0kSyWh^srXwVEeNiertI( z;nFV-p9WH#9O;Kqk% z9sT*Mae^Dg#S3cli!VPD9{G-l4`mucc%k&PfF}<{i&T~NYaV+h>n1iil|8omOa8y? zD=TC2I)K2muhd#jRyF<{?t&&c`GtMfKDxjNRjdU~DO$Zf_Jht-jTrm4(Fa+gyfjT| zW^NNqxJ9#ESE3^ry=w|)MVSJ|W)zm@fFD`o{t8ff1UHxsp6+Ob7RRve{O<)0UIi}| zX7>J0a_|gL2fFd0kCZSOoH+$aZL>+PR&VNY9gO2^!h;h!O1eoW*cS%b0DsP4*L8(g zONX~`aOyU!5w&mME|(?Deiau3Q=UH znL;|KzdVAzj<0iD3JhO%QIu!)cImutOLXvPEZ7HDbRx&nm*@wG^C2RmdmNzV01GOf zR*t@Nc!QH{=;2~>*#PS9oZlPInsD=dby)Xh$?mZvt00AauH2y>jws-?xh?CI&Fu00 z*q;NNB!PWNyeKqt`g)eya-KMe^oxLfCgzhv6t(0raC4|6eZICHNwiNmE_r1A>1hr% zhSZx^@3HUyH{kDSS;S_%zWUv}cV{S*xa7bTX3YS9;=_l9MC-vI^bFSW;M8=Mv!6bf zn0wGWl#wm$oj8y-I##n|r_RZHS>L2UE!Ni&P~H1h%pV$~ z^9ERE$yYB6-%kQ#flR?9yyUZ@me>c3-suMS3jNSKFw&7f)aw?-YP$FH>OjAu{M>ul zQWao;4Yr}61IO`$UtysvDLNJxHBWsgA5+}gMc01*9COnACoMI_&8^?qM^uzy~MXtDcoP^n>cvT1ehVPJpFldImA zVOaT6C-wPTAvL$?s0i9Q%SV;Fo#Y<|?7s+klVbkKdEbZKytT95$)e7dxc@%%4`onq zDI!{Ipuq!p3+II&mK1@s+|$q*htP4=BQk04f&pn#Du!*$wzMpJW^ibFy~4={|C!$U zBoRWmRun=aTzYu(c$#o47`rO&XXUbYnIYNP(11TFn6u+G`XGH?&fbJ~>i=63FZM9+ zw_EmobFOpmS4?v?ce{ch%0H++94gB*%IfwrF!@cEC zo`Tg$ah;82$_XP5!-5sOT3}clst)o2I8=ew-C3Dclgdt@&`OPopZ)dm10mI*MmbMg zW%?!%cMvQ6jI%w%^Z*_6|8odGRT_HxcwcvUl@_@@EEV{l{0_E>7ALqJ`pl=<8{A*a znUj$T3SU9SmJkdP<)**JKJrwhac|Ag9qr76yNn)F?Y6>X$Qu3cz5Tq3>>XNSc+ULw z-CXleFQIpg|IY%+tk_|IL8q*GB9v0rY`}oOffhX+S%X7j^YE(=R5$d$F09zwDH1>c zA+TvkWv3h2+WsHq2e`zN64+$6BmO7PY&M>|e_cU@KZ%rrckq9|N9?g*QTFoE(y88# zoRhqO#8Pp2kp^NseqcRf{beQz$XP82KeWgwJ|O4zAoYI7_> z6vi&FOTF(iX;*J_55l0(x;U7jTqX??+&MmG>H2sHM`)3QbrsI{#Rei?) zE)HIE)X^)01fzTXFjCFjRUX6Xc!L73pY&c3yYsZK3vTd@OW!n4>syBj{K7I!=<~@Q zUxABk9T}R7YEu5|70-<%wj3(fd#Zb?bY|A$CWP#P^cg zXrvu_BKIwGo5%P`oufh{lr3U==iKKxmeBXnE&tEjt(alevoyou=;6V$|LlA4BH#uF zM(e>Jl5q?Z{Z@HTkE624BO=m)__H33q0dQZy@W9+r7z%ms=1T=@K@(q`W%e^gVcx= z>PgJws75LOcbP^PpXoW-KP|?!#<4g@CZp#j6?~RVCLwh!)BGH8O>K3`k8lDC4C`LY z!zF>8J}UF}7VmNr^>W4@XM3=I^s4~Nk@cc^@QL&do)yKNj)H8c2w5}@ww4wayLxCL zJ$8(@?nM(c+=e`(Nj0hKj+pO^27S8i8kMS7gjeF0W{S}Mz&Ky{&)?Al9 zo<6R%f4q>?I0s*}hkX;j`~D;o#bLbz3GqLo*6KR!%kWgL$Z*!N$E8PiB?DZl^ITpo zfmm=cl%IY3V+&j@oBRhRa*jZ#p?AQ#o{on~MXURR+dF z(L%rEXW%8=T2+XU-lvik9tsS#&w2ygpsbk!!1^Cp8p0($d~4(Svrd#7OT$fnH_>Zj z1P65Qg=|&8@ORd0SpY36MV_x0bIMJGcU>4Ep?|base5`d!ih)6~WgG1WLC~ z98qJka)?bs6QHsrknCb~_|I7B3ihQ3codmQOvW22e4z)(mCLszS)#ISg&s&|{JoWW z50NW^*usJ3T_0Hv`GP1nj)t2*-K0FnNKoG?S}2XGUdPC3wsI9YWlGW5Og-!~dXl<{ zXrv$}g2}2PcI=XdM+0~6L}2gJ0kB(w0^)8c8TqGc8~D#+pPLZxvvf2&nemM*n3k@_ zj{u}B!HxQ_3mK}ij+7_={6*(u_(;e1pklp6vPi0f2O=3L;{;4EN~^=~1znIrKP5ro zk+syQ6wghRe9r4LeNnQOV$3JHh3FK4eOYA}*1r&TKJxJ01=hmIt$=~liX|>$={oE)lz`PfWFWT1u65HUM7rI4Q9UMOhrS_X|n(5U() z8>V%P$t0`C4gOpj%NizuD*j2rMsV1(K8RP8a-r7T?J`>&`uRN3R|sLstYpSfy$&;> z6c&x6^K&XuB>ZoX2v)M+9PM`nU~jk&udq4-xy5MMJu&o>EXNNsCVX*))ivtDO_4o` zwQDC4VN13kji=3`p6MBH{l8YA$!&N}zkgmJFCN zyWWonNDw2a`%jF(5Pwp`WUw{Zbj3eh%Tqv~z&pe&Gx&(t)77%=0f(9Nz^+WV zf<>-HX(~ts*L&0erq#u0q+Qed;L~zjDE1Qa_XVim&eV6TNcIns`wlSALUDexA3}IcU4OZ|z;%eFzK|K8 zrxW;gc#~v6Rp3+nG^0MC#sLpPl%#{Bev}Cl2tuB>%%!J1Kxwzamv&M>$a?XsAVJ!+ zg+txCW#x2U{%e*8Cn?^BF)~~jx-R|SeGnz<_g0ZR(21KMcDkWrKiDdj~hnJ{!&(9wIky020eji68`M+I06m~z#Bx-CZ333|E$y)(I z;_yB{zoFi`NN%6k%O=-T1#hZ|tbO%=1G?cD>}!`l2@SNx3%&tur6-Mix0g%#j6T*& zo+Eg@Cd|q4=D)#iv99Di-fbKyaY z)6Ne_hK2d1RHk(`9|<-bIZ|VoE;&to3FgU}kBH9W@U6N}4ra^MpL{?V={^L){tZ;m zyh|PdLqbkm4yBK&Q4RH+OlW)k?1uj#)v(WT373(*_#>M)YZk&`3s?v+!;h43|A?*-X(n zH`1N0s%l^Twc4YZ$&%w!CVY#*Q_@G;7jzuI0qtOn{B>(1K0+YR-8>IX4qg9g7o({) zRUzH~^#P00qo#UsT`}ALipK=m2G0|NHM4n`4<|e(1ht^ox8~F8(U@SPlG$i|Vm_Zx=U{YPW7ZTzK~G`CPfu za~I90>2KK?s^@=q>W9Zj;L+c0-+I!u-B&jny7}7j{@IbyECa20JBf>u7i`gylvJq{gGWxhu^`DcK|yCb zo*(zfY+YLLq2&!yYPjrkowg_;UXMsD%h1St_KTUGW31nLLPCs4^6kpBX5%Or1{YSm zO*g5f;~rUKx37&?sC2g;?_yF|)i8S3o_5Xklp`3Yxi;cGPAFgVgNYGe zRK1qL;Er=(F6=NthWr`t${Xhw#-es;jMfZK?uB#@oq)+C{+jqkr-gtJ8(TvjrF$bN zY*nDccM-)lY}C4smWax+p1t?v@wC!{=Z{+@k&WCG)Vexs3y=DqzorQ>I@R!4L1a4k za-IbH_2ttcmX8CEpGoj<)Xr=Zy>h4YdyrJz(;$%AaL3bi8!1Rl=W`Ew}n3IV=FI;WEzPuoM?Tl3L!$Sr;AfBvj18K@v)MeQ> zsp^NN@*nM}3)?X~+i&z?1&pzO4eIHbAq->IRl+kb1-wQmQ@L!wjE8#pnN=C4v1>e{ zTmO9k02RZA>-e)*KCS1UU%PzUo+_78<24dZNC4fxSFCw=HcE8ls2|^;$+XoYE2{kP zlVbMDXLD!V44uy-af{Bs@7T^gK~A9W#qp?ejjb(IE7PG|jYT(aDO@?rB&`4{3hZVZ!p z-@S9lqXRcrFW!&IcI+6>S{kA-=}gtp!-Fn?Sd3p?#``VtDddF5C*7O(K^bUu&L$Y+ zM*)p;N-sy*PP(fa%6B^4kK$Bix`My4dt;6&RuE8yd~Jm7w)pyGZvs5LUsULwNNeD+ zizOqyoa4C-6mFX>gnqZjR^*MoR)Z@`^V~`w@OJSC-D1TvxdjdDoaj_9i`g^vcf`N6 zctqCAX=Xh6N%?TRl3lsAdxVGYg>!7wZ$9M!5!0H|b1~(#v?~(?c2pBH`i#lni$rW* zC0x@AzQCmniCgg$SUTF03B(G{2FfHPq+< zCDydPa$cF}1)Rqrc&0f2%eWGXYnaDtp39rBRv*5HDoPNE0tY*N`D;WtQ*=<=DXBAIK zR{QYiyN>U!P4hlKyQ{y)CP$t4GCY!hEuCpIB-iWx!(ySLExx)g18xGpQ(@1WX6%1; znTa)N8N^SRFrWFEFKWGYheY>sf-1#}h>EK_mT7Cj?9Nw13-6SF9-c^k+gW|dF_X9b zG*f+2{c6kzm9|3S-8pADkLoi@M``6sFaX92TFZG@C}^&KlGM8KAMG_L(?CiZ>iZzy z;yY+YB+T=nPUO}E7>LXviFjqRn@$GGchz1^5~286&@;CwIj-UtIrXx4TP7S4>wsP= ze#doOanFjr2oOFxV$P3EL~enr8DMnd;S z#)7#_Sb1$nl^AnqWT8gp=amq1g$B0YKkm>Rd)Lds96ohndH80gVXbbdOWk6m_0~&| zny9Uq5FPd%ivYQ^-9kPDz7s#zIbmlWem}PpeEMv3n?z~RnumFdusCOPbp3bMUtS|| zHWfLFjwhA2ld`?cLbkiTxCJzQ$J3k@J*;NGNQ|8IO3C44WP5XZ#n`-Y@=lcNQ4$j4 zPvY?=+W+W!6Z*fLv%V({)Y=B*&}btSlk8--<4~h>w;hPCq0%6dQSeQx_I~5NaWcV{ z?jPG8xk*7cRF@ki3fT-BWb@rMFAG$AUdf#>%u&4YBI0eohmvO9<7aXl-xO1Hd)U-N z1J2OPnlY61m-2sma$n3*j%_@xj^a(Dd5)@9fjKH{aig1<8Y@|GPbb-sA8@?F4&3Jv2y5sx0jrgp2 z+O^o+tpRxh@2kJyc8j6m@pq=ZT0>47$8RO4cGT8Mt{1mn7qx0{ebub@>qEkw$*rr4 zXB>wtHlIO3Gc*%?X-Rr7*^K&5*4gbg%Bh*ZLH!@j10lCgih1{CRFEL>Q$puK-EX}W zYP{Mg?Lxvr;IkoX{N9Au#yME6*2>^P!wPB zVm}*_FP&Szapj~rcBq_WNzU6%UW5~dyF=RH>S`pi**uzJ<_emvBs4&nUe>(X!}H{s z(a($5cY@X1d*nxfZYq#)U+2pDe%f+m>}=VY3qPGnzNfDxsM=aih!}}wyXcSdz3a9M z%ZQ^_O*(bw`c+-8tsgUW3I(r!f0e?suP}b8F=Sc1BKnI^vm^HX1ozMiW{cj36E5@3cv6B)qTfD+KMToLN8w+=Q0=Xan=$ZaruHxLZhO&gY?-{sa8qX+q_8tIUZnlykBrX(bYS6%z*W+bwy zRZL=4^)YGdY<@Eux`FuSt^6jG3 zW*%Spo%+7>_Cp>CKSBa*`*wV34QTDTw%)nC`QG@DGRoBmaw~}f{BU)qL-21Q;8TAq zpK^c;dUde7J4)=Au9V!X<3~9Z+!>P=f;Bx42M31_RX+fbrzwDux2`9?@EIWipaQ4d zs$xL4sAuVR?5iBW)NViob~wqvklc_fT;Tesx~D)Q2$l8BrnGH!>| zQHzH$loA%+;->wq{LYx9Hch1$zq*{;7v07GY6xTn1<1))FnN3j*S3@hd(ifR{XpfI zrGuvB<0cplq1QE;M|o0m7pVyJpU0$+NX=%)Wwu>pTL0ekex>^kg;Xus23)dLoacrU z^SI4IOl+T<&&tOx=dLxHP|?+EJJYmtQT6%PL<7Bq>OZU6wQ8GJr$7EacCu8u!}uPD z_OI4o>WOXM$@On(i+?yy|C*)j);&31uBgN;xUeX~r0orKWA)olQiR!K#h*3Wf9zoa z)UQc&55)NQx2gI#%5gUb#d{j8=NUetlGn`Z$vXKn!^_OMX6wdydw0SAvGv~ZRR8_| zc!?qv9T`c;$U2B9%1UH!$2oRc86ia0IV7^O_sEg$kiD~#WA7az$I6OhZ@=g1^S-Xn zb$x%IKkIhfZt1+9ujk`&pX=nR=o=PR2?-Q1N}r}BUv+!N46iyMF1e)>9B7se9Hb% znf1viI)y~@$1V%U1I*p$DMlDm;aBsl_z}lhH3brV7)8)O5DkDUnH!fI-*-fVpWbCj zWlG$Bi4P;WhliTX1m@k;9hyZJH;fGP83&^`nwmRWWp#R^|B z1y%^D3GmKGtb=J-XBdTT7`N z5zbne+mh+ruPstnL7M4)`_!t=5L`4DC(bbxD}4K5ukIECS?&2Erk4Xtn^#S4)bYWD zpiXG?Ci{J65}1b6s*0PuA@MxpUv$kjYig3j8>%>!vptjl8aAZfQU=%)6_yXEyrndW zr3C4xPtBwCyXCLPUikX~VU74tNl=Kp5rsZxO9q}2g$dwz`c6V?KuL%)AM&RL zhwe4;v3^{dx*zH_bO{^*p;#$seMz4jEWRV~y+L=bfrUe6wj*kMkz2QXn~PQp4}F8V z3qOht7L7iNp7zOhxthmN`kHg_`K$jK7o?O3G$_RX9UJ(j>^Pols6f5j1tS{SX|$^` z8Y8_EG}>HQhq@p;FQhe^xz*AlGHxAtr zeXj!+S%Nn?ug)+!%d?)Js|LL{fU@=I(e<)z6=k0=#|T@!I%*bW59tqf8cH^UkXzO3 z_DAmO5@)&!m--EMR;1zQx%PMNXlE~W+OMXr_Qrx{_{{Wvr|sL%6;~8 z``cMD7vq2Ph%n+eKW)K1YbgBkE;$hE6uo|NcFU2yWZXQ?@KD7IH0)Af+;@OsmunK- z`Ryp)yz9@G=-(@kt^<0O8SOpIF|{(?Sldkyo=W9W24LZxA(EmBApI~q6UQ!`gBP|7!sYV@_Jg?kHOpt=R9Xp~ zb}p~Fg>5Wy@TTBaQpH=8Ft!SjueDJ%TAdERUVcb=^`95QHI{p|?-!c12`}?9-t%Ot zB}5skX3*0JH1Yj;<@(~BXDsw&25hMcGGcxwkQzi8Rd6*FvgwK!DUix|aAalJl!TB< zuZIlJb+^}d@V94&`$I$H05Ie5rvom{%qy&ulEY22B=TW1$De&N_TCmlv{G!;%>xlG zFNYCN_Rs-T(n5Om?|mDq^IArpt=&C8>rIxnINS5W^>aIn>8ERB;v8Yqnij_EaeaC8 zgSFHTJ30U4Atlbl;-EZCqpJr(?yl%Y-q2P8cnXlHJ_>bwSOoy5XFwVY{VT-bKQLZR ziZ@9Q(Ko8MUr*nh`;R44iWU-3Se+|qx7)J8C({&t2QW_tUml$@8r!CUEJf9bl_!^O zNAuJNefn!QP6`v>MYw_}iorESp4pa=ytyV6=^Ng6`{Jv6&ZAAwao}rIq6(0e&>Kbj zDj{QvR8_h8bYHVMgztKM9crHAz{mLNyZNM7p-uT7o<>)RzOB>4$K<_*@zi51R2AiC z7ZVof&a6BIwvT6#KTeB(DIPVCcwt;JC9(ZzWd~!?w+UvS{sCg8*okTu?SO#x_S@T( z=XEk!dr^e5slv?s`J)UcWNt1&h8t4SCWpHb_!=tTao7nzDlwrB)`#{K?<+chq0Ro~ zmIDb}2wHeYLpdZ*Ao`_F)vUy344T*It$uBELyNZZ4|-O3@Z0Q_f{f@c1)nT-WJyzY z+0z%;+M%ptDv6u@vBFQ*_s=?>Vb;10e<_Vsf5ELZ6{t{;d^la$axRASh8 zY1RXQq4-cPw4*dvx5BAKAu3a(9e&sNBnAT0WxH!Uk4dJkjBdbL^h);X3k*qG=q-Cj z+LtObksfGN>6YERMD$cnN^LsCo;1)agT5x8?k3TilGK^kVOkJh{%?Cg@-Lw$w!QKR zd{XKb>2D{2DT_ts-ReI&Vk>%qc%T;~I z9KiED<2@6u+x>FEiEzXVzqGa5kfp(Fj&8j3v6$pIbKe(Y+;piMrDkOcN!Kl+wI|Fx zry3&=BZaCAZl&VBjx_KNQ0#u4n`&(}oUd8$(4{E-MXwoNu00jGj~Pgg_Trs=Y|+IZ z_71_DjI_DFC?<(rqH;j8{WMg(68orVV3XL$W37B+3w2-F(i{>E%Brf<(M`^aR zSG@6i(Yv2Cj2B3YIvAc9ZmeazH?+X40)qNOyf^V9NXlH`b0{i8Ude=k%xx}-gD+6o|%f^3U;tyOZRV709W-BZvfb!CV)mb;ZU z2^^M!9BaKRR9hug*60`)l`7Zz-TqqMdQGteTpoIJ)Ij*daU}BQ8Nzqol&vBE9M@9q z92NIm8$T7lu6^h>+55rHIA9l9a2U!`$w< zY7DKhqc@=dB$j(jTod74fL+r2+P(hrf8=+4+{93nqXgtm?s%jxTvs64)36_toYN*i zotKPm^|Tqc3f#|)wCBZfEGB$<-@08AYam?Dtp@ARDi-F@&BVuD9>(MyXL`K-K&5d;5l~3_fjC7Bl0$mof8Hw%Va?N8dXJX4BcFQ1E%5^>5 zL{~d%pV~93qA5N_Zb02tLoDGqIn1&1kX_Y}wl#)S4rx~0uwWOVrqgGZ^+RIevkO)T zZmVw@P1<(ZjeU#j4oXup@Y5@LbC;Pp0qsDY8&sx1YNDPrx zD&F8qsqCe?uc_+b zo*kgi6h?IkQYCdYqMC<|d*`_d8U{z5K7I};@5&{)74%LYseJqSv}GO8Khxem*6ICp zOqikX62NC4V)$NzC{LH+zT+6m@5^_U!s!GaBE1)5n}O?ETj;B2h;0i&2eRiQOw~r$ zi38zm?j^tG3+;CTgHn>u_GC!Y@i%yS7x+B6+MgbcAhN>s?=*;&BIIV3GE61ApLhwD zD|n|HtYtgI6gEHYXhpRv&R`v4JOroNXCD`ksBOmOysV_Id^`FkAuf2oBUx^_*32C7 zBn?_n;b5$$zVcz_`ys|HX0sBpES~;A*|hN9EA0-&Vw0$|TCDwDpL_QQileMl zLpi4OjO!aWFQh+E>t#ByLc30juGeRX(==cg193}Y8`U$jcdkeK83b{Zt|tXAvKJ#y z=Zl8~w62A9t#_(z^*hAci=m9?G7A&KPMTqE=DmYJx=KK%A1%FU8@)#U)vl~;XlFov zM?Xp2!{yNw$hc(MJ|KV!kCGbW1xzE!4n5w*;c373+-+QCR=;%~872Wjn%=&KeDO>0YxAj1xi$J^dYp-Bxg3zFLq~iVYKliV5 z?`o*}%Fh{3hZV98A(@H5i~j;k>RyfvgS*$VM_ol;IR*0f({|xH01Z7_5(MM^Yv3UE zl|03V%>ah&Dk1%Of6wb|4>=z(3tk?f3g}Bg9zH*a(o#S)mn)PK*Qzt;& zY>m~QTD6+BE6?jGfn*S}15HNiM{cxrQ*d$7l9APDTEph8LTCxlrsXrB!?@8& zkH4VgS!=eU9DJ2qKs;XurWim2jH8`E4fGX?+T%L?AJu9(RTfguY4H3qnCfl-Ep9{=KERp_0W zka4r+JZptaZma5=m}8-r33*W3b1BD99oi{Nx#{up%%ChE%}Km z$ff{)>S|cu!(qo?&S%`Z!w{{Xl51!HIVCY-lsi}%8_NilJPRWKKI*mEU9@=ZY}EqL zo&aW6lrlcVaab*T`1m25ZwCM&hAEM|+V7T2UST61&xXHj%dEW9sp4W)B;Y!8kF$$nK+VB_}J;roDpKFVl z0oj_Dzpj=jX~Y4W2FLaH@~*;83teeQ^g-9lCNhp>^`Y3x8JqV#yuJPpNZ;_BH_Q&3 zE(DL#j`H&@F@O3Sa;Z-Gu*gUv<~heYAL^=}6ubpG*5WWZg*x(dQ8U#xtoyykxI z;>CBLSyETLi)pf{{(QY9p?y!^d9s#2W0H9QBNtrD+*eHujoiG#Zop*@*=jR+66VKC z15t=)`=*={uY`*g>kpT`7s~rB6laE&rBR51+^;r{H1ne+_hUPaKy)%}HcRVXk7^lZ z=uDWMkehAxVXU+<=xs%oY{eGR>3U|KkaZGeL)8wJ=&T!#I1Bt_Qwp5~Jl1;kCgn`y zHbb!5o4F%%;Be7dUb#)aNYh$ec)T6g%DKB&W3uo`X?**{TP0`6>e82jz z>w{_~$dSapUPyTPflw^;kxqccwpYJ>sxxZw=`G@#1n;T52Z4plZl9(c{#qV#XwQ=c zzse*=KrXkladOY6pU`l_AY4D-X=wew+L(=Q9>*?m3)9 z5bMlGM=dchP|XMWMc(#{6Wxlt+{R7joE&_#z-#E-;l^d5(;PjjCi)WlsyI~+c|#wM zJFra7f8MrbY-;w-HtR5UG1Vn7u1s4jY`s=EWn4H#x!UshRjt`#ByCtyP_3Y8YrP)) zc+Qpjc&L@SfPrIo&jFs~>0F;vI1-~jDYw!wm_Ns(rN0^{gh93+q~XmWee(h^uMpb% zijQs>bq%*FD;?-Srm7|XLZI*Oq$*0Be$#cptzUf~sANo9cK2BncXQnJo7@3bJ{{=u1w&iyt9KY5NJG38$ZXp|X!fxJlp?FuN`H8VnJMZeP2T+1D zYX&40UK=%VQW06oh-xBalof$SEkAy*gU%U$$>SGQAhE3J+a)Fg*zf&EYskb>(wl-@vBPiu8 z98@1<-IUK^H=f4{bipLX8?3HzbWh;e12J7%$SoPEg(a~Y`cCb^Yo=MALiMq3X)L{W z66~YX^HSaNU6(|Ey7t>QwLbhQYTz~>vvh!)?+Z>eSVr6imV3&sT>1R-@cYy2b+YynBe+A%)3yA@~2#~&_ zvVXiN$)tV#7l=isAY|)}fi%gd+@DE&TZ#aqE#vBOo$GzDtUng?j}x|j*)@RA7g?krE4qH4&Br+=hHQ4hg$t&zq0{mx=OLYS0qqamTg7d!9r3uW64@8Rp|u^ zv3m^ezdz47p^pqic80X_QU<|fs8`M(N@;oH-G2RJNwZ81Q{hK^MEUH#rSl!m;ilz= z=@i6x!Wb9e?aL`h{xLFyou4KJF`1Uxl@`Zt*Dig+yAa584=dreLjUyS73Pn2htM<8 z;_yRV*K}v{uo9mH1o4eM1(|0`GO*$eIfC~yRaovMG>eYg2X7Ts$sY-QSjcNI z`$(jP@1l|v)#=iyZpE*rXXYtbHO$)Ra#!pJWb^hI+orr1%`&A03wmfNiJHR|G)G=T zjpIN*lhF1HgDb+cd!0dr3{O#s?{St~3!w?(dr-|Nw4*c(O>-)w0^gx`CKd~3I$LID(n56#PWa%DFE|1XNU8XpqI zCV_Cp$Iroup_h|_t(&`Z{!rJ>4Q3+tE6Q+aO=w$FlWKUebSS!4Q+`%#c9HV;U7{b> ziZoI%$T_S}nxpic()?%I8S%rU=o^F5xsGWZVLy#IcP_1S>pjZC=)!` zUS1SwFHBuG_de!s-!Kzs{Ui=fkOoe_QO3N?&7CnuT|3HxH&|M}v z_7=V|bh9*ubpL2e9r-p?7E!~MM=jA=Ie`C97r;OGiPYEY;f<^`FRqpisDABE^1K85 z3Q*4h=WiB4MJ9M^3JtrsQonx@O2bqEBnZfv{JG3!#d#SRjgVEeBdRBjUPI#xOQs@q zwwMc0>oAdJ;zTFZW|idGUqr>mESTeC4&$hl6JYF5;&3AVil&dV4xb+o6Yv=3cY_~q z-fsf!Xw-!0q`v@}F8YzPH6hgz)q1z0Cq?|LWf^EC$hQ1?^Va>BJy+DkceJZOuR;?+ zt-x2eJz(YrK%EbSm8e_(^ID5*>*J|@{>AjrN8DXtbTsy;m;n-9ay7}5H?DILa!u=A zXGhHV5jC&F!PoOK@DHF2hK~?Y^n>r!QvkPY|D0Oz2&?_6+If8~`c=B)uTPZO4=Km9vKX1Iz_(b2}zV|oJKoq4XLBvj*Jo1w)LGWX&=$e6Jn5$|3~10lsMV% zi&xGp3Nsn21hpG}3_BS$`$%z6_IYSDq_4b?uYZE+AHB%lyxlNzO^Y4Kq;Be~ugg&At zUYVE_wO12@zo3nlA>^0>`~D@-YN9D3W~Qp25n)q9y5Nl1SQAMxRu*XryKrtGaZ8-b6)So`Aoq9gbYXq%T|5IF2@N+@Bo^S(_?X&#(#X3y$ zM3P0p+vKxi?o`1wwum5E9lKx>>1F<6MV4b-)Qa{b%ae~JVt6`-B)_NQ{4c^PeyXAL z!jO(9u>qJ86h8hB$9Xl;!FULnRXya?(vz2m%-$p-6!)@nn5I|m2&;q_PO0utd&O?2 zS%3-8JlX8sj3_+h3f&cb`=wLy;H7B0nt)Jf51Iahl!@{i6s1E9WKZzD_Ebkt{IDxMYp)Rh0e6Co#V*`gzJI%46v)>d6w5Il@sBjNaVsU|gF2tUt5 zueGYY`gO!J>1 z#Xr-yLFTb+izzDuv7aCIY8S;{_}_LRoj&M!i(xduQp2yXK6_(?ycxJ#f~zF7w$<*8 z=)+16##$w42#M9N2@=X)kkvK=nv|Z6xh>Z39j&AGzRr1!6Pg;z43Rs-u-4n+?r({#DphBWaWhU54SVvVR;FO*J8ri&s?hjGv<8f7kQhz_1G@c z;d2&CjpxBGVgXTs$_=?fHN{x|0PO+rAb38L4GQtWUKQ>|o zCixj9mx|`tS7?K+O&V5fh^2$99KvL!Qj%KA$GVX7`UQuP$82R8=s%BdroLT`NJ{0gemQKEjWB0h7XmO<1p1O7djpXLT4 zBJiEE=NpLt!!AWz2??uHzX$VRFzcx8U9DTI+N5H&?c`X}`fV@C-;H%JR8gAbiK*Qt|cpFLWZQ{j4(=2E5bA|WWW6@ zjakg=%>Rhn`IqAQ9SsL1d?;{KU(9ykz0@dWpYS&TILo+Zdf4KQY%dK^Csij^!ONA#o>K!biO)wVTra_ zJvR&niD@^^s%~)nBN*s45WdA?Og7xnJ^d4NQ@@_(O+p>Fjnx&{%<_#S38Z^GxBxZ3 zECJ2f%l*bn5|k2~5n;YF-~DM-@L>D0s@fS@Uf{hGAu)>Dc+Yg@mVK?mm|2vH4~mIN zaqxmxX~oIz#LJ|Mb;tn)4nI(ESAr1%mz>>xpT9)(YeK0HH5;=Yc&hT2C-_wNCY{u{ zM~?7eUJQhO(hWa;&pQN0`vpM)D0%dwH!@c~19WMfa36!uC^K5yFcO4;@7K>5gsrvr zx3eV;g`8pZM<8=udq~Ib>7IE&HNw8=+r<~1yp_#rW(|HHQpp*$c$Dk^Gb^Io|B_CxpI&iH@&bkpG%3&<0KK zocuIaGNj0vXUx!H9vCKq!Wx2Xi)ejuhb}sGCvL_lATpfEzp?d2 z{BAYjSijYviI$glD!SLeD^~a9b?(V8YP#$~9#+30Gf;3* zARN6A*Y$K!>rQA3ToQIXa0=yFOhtCwzggI)mc>vaT|8#o`Xmcwr$#@dL#O3MZ)|$k zVfyzi*3LHMzb)i_3`F&x(_+_qz)o)4KbTjVKme|vcoC93OKd$&g$BMBc*@|G2cT#m`(Ha{ONBS~{ECb1gXNtS4W*Am75n^@)dqHIsmXM&fz}+E#!O5x1T$m;krxNE+YE?$zE;?9>W;9= zjDZLY05y}=n=@l!0T8^{;2&q?&Lyv2vDv}HVYtwXHXefElB^jc^=yp5h0aUQ@wFHL z7$3l);jabvIyoqtv2b~dUDMmjr7QA%PpkoMEVun`dsGyuuc{cnSw={^NxV`QR-i51 zz56>o>~0>!xSj1-iyA5r6<`fCZwg=(;XoJ1j5+h#ARV#@_QTzNbR@&(w zqgj!6O@FX6m`ts-1xxBOxy?WP@RQusy!lUuQm9`s1sdsV2gJkZ@eiP$myR{#tU zg>GC>wm$ij-vKX=PdP5zoARI)KcB1VC3!phHh+h9;;p0qVqgB3ChiL&W-A*&uh;co zh}>R0dW~BNs44Ea%M@5{rWdAyw5EQIJ^8D$BcP|LJ7n_A=A}8=$@>6-5^QI)#sJD) zvsfyZ{s}D8)j^_Iy#t0RaVnMS!sVH_c$VeoKtpqP98b%;RB(BY#bSjb7A}0dH-j>p zA7WCrfB}y+H}OVr@?RZ5c_ETF_jmg8Z{mO4SDF$-D6tUa_dx>o?@PsHV6Sv)nA`>L_pi|g*VqaRJ?D9rUU~!~!GFSc+E+BWZeL^Pr8aZfQLA~T zUoN$Eyy1V35l0?$8iyKk7bv|pyAzoV8|Joo6c_erIfn%gTV=hjEeh_`VQDorwLVdn zs3O(Lm)(gmE=NFPn^FM(n&!YhSpynm6kV>P!6GiXT4OWaFzglz30Oi#QXBt(dzK6@ z)P|XZR3fBAqT%ki)Z?xy8xll)YWn+T*<=?HSo?KCYz_+~(2cDp5DTX%ivK{&UEJ2* zL<8g1Lsk#3>Xf~?a%8Qm0L@^4_Ba?7Tee9jAab-+^D})XIe11QQWrlYS@#A-Q^m|^ z6xd<8EZ1XAtqt~~8b)I}$ysA{I6MH`5KL^e8yuE}TZGNbvP38Qhd_-|l2gE-U@KbiR(E+Ta z_-6&R$wWX&RxM>E1@Vbs-lwqbq5&y9d?muj%&s+95T-)O@n1sze+?~w3*vl-2*|vF zyxref#cO}LW-7R>$CImo=tc5;53fwyx^nUF;}b=sWG z=K=0ksPf1bM%&7hyr#@Z;ap5eZigPm2iDOnY(JUbfFlgeVyNnPNWbs^+o`&FvdY7s zLaU$)p2cclc8NBV|9mH)aBiW%llB3mPCx#^Aut5^_hg+w&2mnxjvqIyP0y}+zUi|+V`5Md z@T$UYl#XBR-pViS;a))jYbSe5z=LtibOXsi*XUrUuusi0UKo=b6F}=jX}_z_WNCyx z9LjDYG;6Zs-yDMs*BiK6|46L{g;=Y;0~-AbA4Dy`yl zT}+N5t*u>Cw1)5n#`q#1z@80?a-+XgPCc@UCB=i$`x=4bWr5tfZ=vj1%X=|D6BpAO z5wEsr0IeT?{rzgxUi_2|zWNOqWOk^YV_Olh#%QNk#wW}0?;cZIDRcD^z?y#byv~(u z@)75&QlmsTn@ej#YX#KLv6lilem4stfQUga#=DuLSm6zeMvyEAq*6YIRUky>@gWav zM&ShdN<-OJXY^_JBQ^uQ$EHv^HptAg< z8&1vstRNL;BWVRRp$BY8N0Sa1O0e_|Q@8Mp!E@-A7&ObVAwKLlIe6E`;Gk7(@I+~W zl4-CCqGMpI=-~X(^o~}S@sW8b^5}Bsp70GLf7*K5s-Gd_E}rEH*^az0=T0Q{$;UUU zy7}hC@rz5`o9^V|M7As zBlesQiLH4?D+%R7No|7MkI1EEcg9itqnWn*m;UDj0Tr`0CPzEuTw6Mvl{<1@0lcYM z?zTqlE9$7kK*_(dacVKTod7xTE*zp|ZmMX&?f~b?(}89kbBC-6fHPwvyqCMT+3R)c z6n;+@#A}~GbH<}FOn?jA-RhN;y#8F_S9;O`yI@H!rpMmvXd*)}e4;EmV1ZDQ)BjYO zLoJ2j_?~7Uf^E}rhdG2}CdL*tZjv3AhA@hc=LN>I{mgNcg}nnxbBXQG6zrzf41tE) z{mZ~qeg*852$tXNc{JN!?2$ZhPwLk0f-nYA!HH-)iU<6GHm^G16t}33K@w&p@i|<02nZGGOkNazyIUp1Y`79bAvClSRben-@I0eC8kR#;&V;B+)u%Cl0f8`-(>S(8e92%f*)*rQA7_e>F|J~ zTolE1x*5Uy^>~0^iTiu<%b>zz=c%^jyk3rOfgU-DVoTMdE;S_2JeOB+eV2hiDRg+j zLD;wzmel)lJ&yn1_y>}V5JaaSWmr@&Ou&><8}xYBbJgZeez#oxIrKSu_V2xa!^Rycl?$Y%Ejs4e zjy~D(1_r|zZAN>Q$Gka_(fQF~rWRN|h_u2atV1!dzT~t#pb#pR_bfm5c6Q2yL;0P+ zVnSZu>*d_+g==`@vmN7^tR%bTN`R8?c^#1zX$LL>TfGVrRww7fk@SbRa$nKbv^8|6 zU~>L4PkwlX(wt_nq00|*SsWNS#<`P0W84!Q%>-cReh>#(dtV#P5W{m_vnQ zM=`YEd_UPLN zQE#a;9W-w_T-o~Q6iv$22|Kl;db-HbABi>5y&iptc%D&!;4I7JT6LQVr|{5{$#%{$9i)ePp)xs++OcEnz-~f9B**%2H}ehv2B*!Uea~&>biAdS+!= zt>lWV=_S@?1>&x}qhV?EDDb;>F&7Ysx7B`iH&HB?k0tqK8(nL8W?Ki=N;zuo6qEmV znBwZS0$GJApoORqIL$~aX{t?n`)k~r?Gp%aVg?rooL0s;n*{S%g_AN(a;Hy{G@ZHYL%CYt136OY*yN)`Rw9-7 zxZL?s!1x}nAAzMVs+|SZ9MeQNp26oodWdWEyoY-(g--q@Cc-&r243-sMciw(a8OA> zV3B3fsaO+gTN`m2PF>F%@Mdmy8cnaTJm&-Pd$aFe_wb%wpF%Bl7Y$$3|3KmS>%t5x z(}8?Ig8`>k2v2`(OQ4EVMXK@|i0qr?ku290yX%E5EhLr-;<*MPw)#2TFlQ3bu&)1)Lj{<|m zLgNy>Mw$g_oVv-BkdrCh`e~W{-*v9tzx;5z>(h<_V*7YvW{;=grpuIF_C_xLDC4O} z1^pCa;Y%6vw1|X^l{^I+G*FA}SuN*fa#ClszOU~PW&7B%^YzKj=p|%?di!k1gK^G} zKXwggKC8qA6`DloB+8fJOjku8XScB%!Wd0NZO08?jCEChxA4=<=oS>;d4D;vJFQ#f z?-LAkzgcSNlnj;#EpcVqTh0;tpsV-V5AvxR((GVoH`>{b5ZljXdHCopb0!cYrg|zPQ#(=gUk7Se^?Bdd*3u8V0H+EqnKaI!9zc5;Nsvy@ zqlxKj(}yFKkq-jp-$+g<@_dPYb+!y|Sf*VB`{$Lz&Qc%>;`j(=3w!uD4Y}cz=T>)_ zxKaN)4x^L7tw6Ih3Fh3~U=&lsFCX$l7^fyvxA=qZCtF)}3B8hMp6Z~8lrhZ`A2Y#r zr8AGw1(f+*f~5!LxeoL2hplBUep#!WMon}tX%gKHT+*&?Z#c7gnZBuTsh}urS+#~= zumbdWRRvVWsc0 z!lVBukg`W?u}o&hXiWK)b(N^@@`b35oPxUD@%9Se$AaD6KKBCc26laNo@K2eKE?>q z7_9(vLIE4vsdynz{AM z()s~>De38Is;fi_4ZrPa*jvBR6eRC~;9j$!n%~<{Mye6L26Y7+GvgheURkFhF|$V! z6_4Gs{k0k#99`#9{3oNKg%qQ{?T-zw^BweK+vA>fPvgqE#Yq0aa%&@b3WY*_g^-T+)d4v2K1~^HTqWc z0leXGI5l{d0D9P{`K|)-QCg4_i1ikdA*=?#Y_oV7ZCK)y`z#=ixGLJ4ivq=^p_O7^ zq0&^enp`4r;RhdeskR4uC~n5B;1J7yjg9^PjFte4G2MAGZ%l6e9JlVgj4TaR%ocuo zO5H(@nD*$$QSDG32u45cj=iVX|ZWj&f z=$pUYnF#fo?#UC}X}my@K@`Ju@>}i&C@)eqX`Z#t2cP)DS>bz+0L^M7JrLR>?7?-$ ze*liDiqtMvU`fq4bih)#KW!~t8?Oh@MD^=mFHp_BoH@QSnkjEm@6mObH>d$AmvpjltS&pC=6T#4-zoaEFrh3O4&PL5$8UXqwmu=^dD!>T67m=8e&d2QvqY5b!6MKnCR#@8Lg6=y(e792cHvO^mgx( zzF5QMGypIna-ix6HKXMhsf3gkB=2Zuqgjdf!zvXd!x$WvVw+`xW-c3ca6Lf&k_3=> zROD9&kJ?jgQwZZqQ4xxNk zvU{cE?5*tnpJp#|`xDkPtDA$_Q`EbU_qAq*czWc+s)-G~7)9^jw#H8c4_LpjzL70U zPV<3;qMnh_ed$}#RF6%T@gw1^o;w01_qyKMHEhOiblyk(=|*)fH6+kHK-j$B;BVbw zV`#37CRJLqn8&|S_JrP^WWf&0K~I`;`3(ZnlmmcZ6kJ={H)YhQBXflwo~f31MBE8` zNBJ6-`z@ulW^9cdw0XF2yFUF8rn3K< zkbdgx&8=UzIm`4=CWlR?JmAY_xJ8~#3`BH($2RUkcl(z6eA=1ffYE%g@#TQx@p;ln zj{jO&)XC4E{mJmZ;~-u@|Jqr78Mwk@-#S&lh|KHk#s&gLsuyeO1>RM|rrI zX1RjKm@m-G_<|ruj+Rb%dtjKM_OjV2eP2;I(r4Q-Jw%DAjrHe`mTz$oU}Xh&8({K~#48cj-#dB$vUW z?rCg3&h?VLV0){HoPT?ZpeGrqAf&D2p5rN~*#To_mV}-Lj62ZOt`^^t=3;N))GfS( zmbhq-PX6aPt#Kro00>c}=+i&`woOlkWDjGj14&?Ez7Pr4S z4R-*M+-%f&n};oUYB}lVF~D1(1xJw#5JQDBVW4anQgK_VN1Wt#jDvXdMD6IogHrH2 z#3HNgDv01vZRJbPNr-F}!OfP$C^7)P(3&tQom7)2F*4jV^k6C!KR>h0lyd!-iy|vdL_J;bB8UW37D;`g?-hvHt!~Z*k5weQA^|~_+`dXZCdu*X0_XW z0>`tqURRR5%sOQdcOAwzJNfPRn&Gw(SF2O|+I{P)xm&bZ)a5tS%+2<|LYxui6u@IG z?y`199auOdr>50+>39qE#wh-1uUcXHEa7`V8oNMiWEvH+U5u_Oe{7Q`U3HEUAKULpFa%Ixv3SBy<_2<^Mt%HX7|={|Kt zVEn$wH4>Sc69NuKl)^v$1=Qlbo8i64nJwA$D*C}3b~T#Y<=qe3QLr}0n7eNSuo^U! zZs@qo@w$eHPDLF5yajr7vcye+j$I?V+HB85-{ip$ZW5#71k=e~$IqARWOVfOT<1^5 zLtDF6?2Z2o+Y$CSz!i=MdKb3mcAO``8>yej%={Enf@RfXn&F#Xf9-IX=$stAcabK; zI36$Zyx3b_iY!E(oud*2Kl8Vji`t0HQ#X)QVW?wgH%yv&BVAAOz}g^iSTTf$DdC5b z!2&k|>5;hS4tVQK6O8N7pqivO*i4w@lcvD%mkk)-)NvrLh(uyunMXahA@mje(U=51 zpg#?^1ZiUBQz1!zWyo!?9(?6k&0$ ztKd5xwQvl}IwX*??G?ssk^RV^8nhi~mukN-4|~V;fNbnd;pcUHEs*p?juA}5xRHR` z8zM9qaiAh~P@GA4nL-BOotNVkGar=J(XX^6mY!4V5DE#0p2E_$Uj{o3;v?0$DhP6+ zJ3KO*rwF0UdWcL2gXq9lB|g*hv7xgzg>@r^?un0AJiT^*yo^#22xF?b)%#SM2i_GV z>)D}1R1e}og%1o}?=@Tn9oAc_!R*B2DImwcLef_md@CT{A4AFhL^(xDmWW|(R00MG z0xu|ct;zM>gts5ir@sb>dzX3gXYchOsqisw zn5Kbs`~B&ny>~}!(sbnYI)=xguNOw^4cYelRpx2F5p>`$fJIuqq45|s!u$EYgtVBr%PWUbKjc<6orc+AvyNDHn674*A35y@1-Uc1 zhaQHeTl_EuqS z#O|mfv@tTL%Vqjx&yPIx_=)QYc62g&18#6z%j@}`L9@eYl)dkks>H(`rc(UjOn&eg z>+1d=>EXbp0}ot5_6O6}y>2)Nc+~do9{H1(k$L(kZf?g>Z|>pkwxGTe2R?9a@U#v47irfGQychRMo;=j4`S=tNeC5G_*4p^y>QHqt_@El9&jZS+G)wKk z>}#TP;b|3MN0e|Q&7o&+ELl3gRl~3G0C<&NZ?xp(X0@s^Q)xJ@3qunwMJ} z-zn7cP}O0KkV605PMm*@jlQ(7INc5)E=Mh_;YM!edgi4@ZkgZW@|%gEKW}Xi#A`8d zBhj4kVQ@Rg)dc?}V32kXxzTUDFY^>za;>;)%p_L-xjl@m(tUiiTKXOMCQ@uwD@zwA zjn588H|qE5_MRI}i{visPnQ}iXOwCVP9?(+z6c;hF$&Q4BVJg+RmaARrBw1a0EbqmZt z9vOkX#K@RhRYF1kPx*YkwxNdu8b4bqc_H9vqMl1Vj@nW?E~<1=`Mg%+4aD`!qzonn zAdn# zZTmGX2B7=6?ukVm zIggrA-K_35(AE%q+mu8#$;eybR4BILlXU1p!ptsrdS@05Mp_y{mXUUu6+Ry?lMjD?Z05OdIx&X+)>hSba{Qtgb<&YFu$dO~ZGqw)?@r2W}Gl0o=h<N6dH3g{|+uJaqc z0POcCqWy!M^^bI2?ciU!IYho$vBH%4jbTc99@3miR~z)2Nn-=O`FNYNv5APJeB9PE zc65WnZu%t;dHCnscqaZS&rLnKh5To#iudM2F6pubwLM~Y58B&?Guv8F-vN(_0PN4x z^4~ANzY>*ruq%5Fbvuc|RD3G)A%p28yqu`_f%%ZB){kXferel~hk-nA-YB8ccyl5- zaR_Snj)0Q+4%c6!iQA`X95=Vqe(Qk5!9I>}SConI?-xUftqcUSoOw+q&LzPhSzs@? z8vH%Bh>!q`n8_Tu2G$t7k6Q;^6dzGE_6cSy2<`qLFi?{TmCe-T^-ky0=;L@haZjSv z$VXRrn;j)7mp;9denonygkp9EEQLpb+%b}0N2Bxh_QGZ9p&E}FuX$+0arp}?aoQgs zL)n1-^~u-3HU27Q zD*H-bG~;cZTBYN*?W)Y3YsdiU3w|)XPVWN3xZ}Oe%BQCkyv;Z!6d@_A~qv7fz*Lv_q?JlZX=#*=1wDOL^r#ZFp9joCC&1>k7 zLDf^3ExrmIpQV@V%jqUT`+o13#oN!)8}i1ISr|}bvtmEjQftP&Tf?4vd+*;6b;}Jz z=wTn_?0uD2RxZ~X+AnlVbYi?L%$v}<$2QO->@X<)d#(f4pSUBRr+mX}#l&;|usvWd z;o#Vzu17*(8{@Vo$XG>|Z$c67C{dtm;C^d>Y&^=+=!em(oNHZ5Egj6ZZGZ33%M_c% zS3YE!T2GaJsqf4}ZcNrln0kW`?&Y(@bk;2?U0f^6XfI=JM^({p%M}sW@ApJA5zljv zP{v|+qJzM>&v^iYybwSp3Kb}7XvWLHGPD@-J);)G*qxt9qnW$#^=L;8#kP`=ZQh(T z)~RCG07vVXz+XHM;X{#WoQ~}gHYVL<=f9DR#)k3Zaqp8^jy`-K`ho>NScn(q{y+uC zoMx8$UDtvl_V0IjQP?^U&(`adJn)R3(myX8(1T?xrUq)Ift}bKkhHtI^m%OTyj^Yg z>1+95fuA(=7ImYF+RJ{b_hjBC+l;E`9o5&p?5LkD6W~Ita(LS|mmet7zHb{| zZUctSIb37ty+tM(jy}@6Gtec^Pb$JZrYYrpWaWOp%od6-DWeb)&py^ZN%R`hw4^Ao z1%&t}mIA`K38#%-uQ<w}L-d7VOv5TiHV!1VEYqE;22;eE1# zUMFz(BqN*Q`@nL{OVs|j=hK>*AgK}90FRP%*3&R80M^(lLdvxfWAR=>o#i?TZeK?v*OFj`v-4!dO$&?DbO_U$s5<7wu2uzx2`eR<*h>+2)d9o-tgwx5?#4YfE<4FO%M0b^-oxuPhRHP+jJU&T@M~ z0DwsE78(1d=Rr(65B)UL{wx_o2II{o87aW1#U23$W4xvI-iwTIy@h5Wf{v(h2n zco@|djiUQkm%Xct>?j+1j?49Q=#b1_(tn_)ig)mq{5AN%;}!n1RkuSj?q7*q)I{UF z325lbX3tN8+B~C}uZ{oZFXmyF5(-nB?HY%wx=%0fO;nu625|A?I(1;1=ZMvqpU-cI zpM1xpyWX$pkvdIy{KZbfh3sH29bF;%iEZP=UD^qV+2t%!;cSi&Wi^Nx9c9wRI3l{_ z^aJ(2Vu!QZ_3WSU&7Yxla?Rewtm-7P4f?u7LfhJYRp+I*>bf8usjXNpt z{>hsFxe1!UQ z8cCd~i+FZ}t3t(e!M7s3&$Nv4{U6J{jJA#wNw{`0wM- z2`j(w&!CL%?%5?*MY66m3{O&B6~23+yLiD2?R`?>-E_+@4>v`R%m#vV&h&wvxRZpa z$WB%j9~&N8$*Av;9GfmHVpXVi(q$eF)3=-&j@%tOnVGB_Svpa?(^n8^C?d)?F4E#5 zrqckk(9GLbHgJ@tof+F1QLWt4%tc-7Kd0Q?;{I=~^*Q>CHAx&fhU_gR7<~Ej7se6| zY9lwi?u<@GWPf@s()qXUJM-JJaz>SHqcVSMU=@%uc4}vmfggv$9WFkI5|WQ9%78?P z$fMGk(1>B$p++>V4zpjhDEe@LP<1KkwZGviV|3?UG^Qb1a5H2Fpj8*kGJ|qN z*#cU07eb?`CseLy6tG^4(yCR}kJEPN=J*dO`ad1X3>j)Eri2qg3` zx?8=7Hd9v>#)EfsP>3?8QB*pMs*Yf>4A zfNVl2+kIWf?6=j8No5nymIho(FWCoLbiO3!8q$@Wy@Qi6!a@vPX30GzP?VAGbo70v| z#yhf5e~2~~)enin<@g3+`pZ!+H~sQw1)`_3AM?dm?W`X~+_#$Z3WDS~EPi*Bva5O* z@cMOa5k1kU*#?{O4yWn_P{grze{NM5D|Hn`AE9eR{8C+oeQ;#dOWp*@U zeQcilX6qeet^Yt6)Sta!(SWzIV(-U^?nX#Ht#d)kMHR}hj|QJL@L5KV<(X*uXWIQp z8*CH0;pd0_SS)q;;b38w`rcXrc`Xr%Eah}Y{F#R{92Y2kFy&o0w$@xzgHHd#mU34p zPSoLYZpdAx;ectLIyAPkG0S>I_~F>`mfZ~AfijHq&2tHt_JGKdhB3DJ)BgPwa`Bh@ zgGr6 zOdky+H9u#8B{xcfBW*U<>21^>D?hFMAs@LZ+t z{0akQ*J++s?kC^b(4$MSA3((fpF6TiqkhuCmT9R^*X(b}49~+#S|}Y9T>E=jpkde- zN)7jkutiB%4{7rw{c8=T-^~sPdCD0fysg)Cx;uS)w-tv%lcv~x02D4*WMC}s- z_MZdnt8C1^>fU-^`|Uicp`bJ1+E|qjWi?osG{J)VaF7rfgp^p4(qi@p3wz%eZG!Vxswg>*#bbQPipCg`UpD(P_{|isMP? z@&+k;c^Z&G?~H?Kcv13jp6KdsY$O&t?>z3=oA{h}d8YJNSya40`9=4vU>>yj^0?J^mUK1k2cmAe z&@b7nO$idZYqRpk-Z53nDQWPct4lI-(f92gCqj9#4&B3Us=DZ``;q{_jN|uT^);uR zXURYPN3ve|+fprWXmW*n5q6sA2$$}*1d8cG$WQOH7PpC7f4;a9lQgDqmVyn2yq2HpOvO)liKD;-TuqmsRq47-34Nd)|eB-6k0 zw#jL+CC1C2?N|2y9KW;kUdU2$Xmn+fN!RBY?xeGKvJkNighz@d-rC&S!h<+WYkwww z44@Dy&iud01{xdBYy(-Jwt;h&m5anh8v8;RBDQa#(v%&QrBTi1#%Xw+4vI4^+s1&y!s~0l40or_ju7TLlL%PA zA#$G&<-=&=-q0N@*tAyAME#b=M2XFc>`$AWVFp`0DP?9DX#BUi9Z-Zd23&oYRKu^w zr5?d)3C?O)5|SH7q48iax&O-rcl!L^$a!=q|J+tv5duB)qkQN2WWLSi zzOJ#GVGJzD+Sq_2M!jI01)@J0d%h#?ey}0?m}e*zzKJo>)*FDUp@9ZEzhxKZ)Wa)b? zJ3cG~91>rmkl!*FqhI~GW=e7)4m!VD`{5sy?>@m5W3cqnI!~cN5KV|GM0=Luh`4Z< z7s8v%l6%>0XTdPWML$jH4EO=2_t}b{=qzMRyjJG9)s1oZrJDay%r&reQfhW8ubU>j zD#oyOt#rgo*{#=+tZMIIqIa#}S1D?mXNco2Dei`SwduW+-5+xk_TArYD-+SriUaZ? zZ%KCz9yZ1lrGgBn&@9!$$H5-4#N~k=nbCg9Y|KO#{o|a@clXlL;hH?F%PH0dEYuv! zSnrKbcpS5>uRj4Bebrv}NJ#zS9G-7tAFmdDQUSK=9XaKK-2NSp-L*+k=F|4I&l6~e zy3vpqD^vEZblMXR)MB~hwS<{{9Ds0_4twGxySgziE5l^pPV}}0Uz#Iz5F7#7fkET* zZBwp4xV&Z;Jtcokoa}p#ujJRQfAT){2T|-Ls8Pz|_NUj%o&ouROVlZw z+qmHGL7}4fDP5@W{!0`o%^NIb`X*ldu?5X0TMM3`b#jms&r3(4B zy2C}5GT;F{KKpv;9Z@*tedgsNK8XVa|2QBC4FQMrj4!pE`TvgkEH=}MnvwACKh(ty|IPY3N*vXvH?Xi*+}x5eJag5#TAhT!8yuwooxC~s z_M|d1DvCH+V^1lG*2`CI)-*7QA?aExv_|95s^*669p0i_EG~|=Yy&O1+a3c8EI=xl zet-7OHP3iQtUyia@|;Qafx*-PKx*ZQNvTGqEcj}4vSFLH8c}#6NeiY>{}y$Nm5T` z?6>Ge+tr=y*CuT0zO-AjuUJofw*s%L60!-PQtHt9?OHOZOIbA`Av(T>5i?aD&h9>{ z?H^%3E!b_@@I9m)k0_J;T^C5&^86HZ#8sy!$5mmInUHTTI08Ikaq8k5-~Qs2bL*BX zX0%1v_UAa?!-C$bI@~s;f6>>f=35v3%RO$cI$B( zSlig)p9?yUb7v1urk;bask^n14PblY8ctJYJR=oxFxP&K0ZGg28~!XY9x1iw(U3f? zFX`A(vfi}FqPZ`~Lkj)9>}}z%7eF)BD*)HWjb>xa6sN->ski1@pWHIh(B{s_V8tOE zR1V*na%-HMy41$5mq9NKtatpm66$h%xjf2T4SMqiYQlv%v4LsM?X?Db^yEB)+uo<3 zJD5A#H$H-7{gx}y{L`)JprJ48a*sxTKKP4c`Wz)gHrl#YPer-S3<(Ih0ezB~4g)V| z5{1OJj@4C@%q;bO5t8tJr*ImqR;H3}a|=7=Om=ZKoaKxcqJ*)&cU!VJZFBIRWY}ix zXkZ7%BtzN4&h~Gk2u@ifQ32BR5c(~|#wOX7`#acqH|sz_iGDMbhUaDc^C!r_+&@2U zesdwOEy<*o#a&K*A-fl%g;f<#B^qzmgX$(9(HS=A1zZ0tvb;OM=GAb{K6i0#+*d8g z7(j4G7UG37K+q71r+28jGA8G>f!VYlSUi^={7ypZx3$}3-%_Rez9_wr8XS%Rwk;-@{6y#KMfg@(j+hI8v^j;^WSp^0p~AW+&f$Bc~beE z<@p{5^~anr8(o4vXMSfY%;ez3bsw?a$4)XQZo%Bq+ zL+-Em-5KwTKO1Kgo_=;PtEvtU3E6n|9g;7Ai1qhq@oG9r-C@66R{EzjyGr|Pz7ph! zcFH0j^vCm&X(Jt;MmnF|GSY`S2mJ1i6Yc^MbK_I*UTw8|Ed>3eeQ%QVsp63i9n~5T zpnTgKxL&hQcN!W9F62`@ z9xb!mqbDp{?W+g7-{EWg`5)OaI5Q9b(sFK!*hTI}eYcO=9G>h+2?spC$7Oi0{5=9t zZ4POuggfs{xa*%*s&^U=p+~^rPzf^w?0dhfCjJilUdp&;XzKL`N}G)L7X9-xBIS7T zUx&Vo%)f^rJNS7R1Rv$_WXCaL9^OxSI9YBz*ddf}2jGuCRDq8Ze7b~|mzL&vUiQJT za5;tkK2h;_*8rY?kKE!$rrkJC&Pk2tR&Tfgkz+ze*MBLX&`Ff^Vnrk6$yi;(ZL1U& z6ryrI`P(Q_I-KR&;`o-tGYrgj-$F2tisNyo&F$!ry_aroUX=&?gI_`=kE{-Q@r4-t8Duc`y>0zh?eVnX0We&>thao8qnJs zwTWq1Z`HGRjreq2FCn~m?%>FIZldvUilN2wNp7fZ<(|mOVyMKHdzoo&i^&unwbYOV z=9sBXwdejuUAAq7C2YCbm2|mz9P4SMY~)>aHcNky7{Bg3{y@-qQ6>Kl2JL1vPkqvK zVg37#eaK?XLbRc03qO`0fl8eK=_;w0`by}jsF~-n(C_a8@)V4TouU%=|K5>EFU@+; zN1?w*E`0cc&6}x46_idqH!K5@29<&qt*QPhYw2RkV0Xp47e-{em$5!qw3hcVF9nJ^m&a_Zj2=i#t%K0`1jqF1# zR7jiZ!TGL;mGL_Cl9zG|JR@p>u~Bc?*ybegoF~dgH94X9ZP|E> z0V?N;Hr?SZU?zMf9_vH)h>BgZy3*{aks1ucEB zIFk^R8t*lT!nUywedCzoVH|ufw#3n@wtYd2k7Tr(FANMU?ruYqLXWHK+$mb&vBZ#*$BlLTz+`^Ah`VE<|*`^xGKp<$0&N3VhB#%N zLZc}zmPeu4AK;7!M(Zk+u0e~i{rHH(B@S;;rd2P1@FO?i{QY#i+@|jNUF^mg?dh?@ z%pBz0=kEgop$Z^!Q+moD1uf)BcIooLYL+~79mbx3`VgXq#>CsY3(X|=kJ#aV=Y&7( zo~g%80oSjml22nC=YkrE_f0)`gj?DI8c68-V#p9g`dOPblMX*>XA~APwNOP&{OC~+ zCs=={At#URDQzv$@6NdR@l=$q3cjEqh4DDee9j^|uO2vNbayeEVB6m;`>iNR^Pgda zXuWlRMw3k+5R-gaG>PP&^3^+e;}_f$J%AzO;Ob@ppX9RrMCD#5Utn&n$xYLZvcaGq zj}-q*8R3Rd2rq)goG8rwZg1zlN8pK9msdyZXYjZW$ify)hDM%HMba}T(MH40=6&9J zhBuZudBTJRMJRQ&LVfd#oKPy${7Jc5rXO57;?RO$J&Q4=9zx_6MPPTDVqkhLFZys~ zh;a4^mFA{`PdIG0Wctm8Yn_*BWjtAQV7&IV7d`!7kfL+LZQE}Mg5t}p1%beDuyPV6 zMN<_m_XeeUZUa&#q1=vLqWYrR&teSEC^aaZGq>N>GtNRP{ZPdjH%gLMW|;YHR?XNr z__m3rEAXGPCyZ%AUFOdAoUY;r99B;4hjX7hvK_lUuMu6F6L~uL`6kM|D>kCSCp%U8 zqJ}(sIQ7bDrGcDf-o3fH2MOrHuce-E##24MBH)G3afjbA)VX?bsu+d@D1@u%vQ<|TUnNfl78lUI2^rI zN9uqZJ7X$wC4Fk5opj1mAK?F@+Z>DoXG_kOhEVorE6k+Los`l$QdAam?BT@mJsV^tXnw zJ3w@WVd*b^0v;JJ%MT{e2_sD)u-#z4IaKD*@~Yj5>Nb~lQZ)Uv{P~Ft2k^M|D;8M) zxtuekVHp_ssR@@0&P3ptXY5JI*1=F;{ojLplm#hJe-EYbI_vn6H`t)g$+qw1`!=IL z5+*srJ5`-@5lbJvpso@u{uRH;{W?ymNiXX+%Em6M7G-lX&p}mW&ah;l*%wr(_{CkZ zEr)x;-z|p0TTV8kxNq(JvpGzIuF&@0=NVoPpE^Y7Bzm7# zymXknUiPSjv$j%u4#kti=QL3uC+L}0K1xyFS83FV=E9R8-ch1e4No%!Fk^Yf_lGS_ zMa%j~S*oJh3@buyU0hg7H-7ObZWVu5A78THcKc!Ul|gmlhMnlPVC@UD*T*YAV=4g= zR&k;()_(FLd+yrJH+v}|_57w;*XP7%mU`r_c%!&K^JA!GCEm8g=~o@iZ5fD?disf6 zxehEZR1C6;N zRTZBL9*>pUO$0N1ok(D>3tVhQ53@S{49iPvJQ6NTyo+O4(9kDlE_`j6SaEH8Ix!b+ zX%2HZTeeW zc7sR2ZnPLd_OSG+L7IcdbKuDm?YiUnDn#PNu9X`Q%pUG-_2!VVdkTOgcbmR19^C$` z?#7zfW1`*5J~dOidfx*V{%{k>J$QTNPqoDSf&nm~hq|Z9R`66hCW<>lWh-M5{#*I`Uto!Fid7M`lM3kvc$?c8)n*$VI_cj^=WBXzB*Hyp zpBo$0{rTJW452|q{ZBYOgFKj_LTx=h&CPhSjKN`^h7UbJNUmMi=+eL5a2Q6wsR6Y!NNziTDmiprc~b+hIKOU4^yHahRvo> zope7Ga=Rne?Kn~8=7ug#n7hG{m9IIY^tvwZ-fU6W{(KZkrgcK;_MmXyDZGN}>;vT( z&uy~!E^tXOzb4u@GibUS#6{xCpLAdz^!=&&<#nH~Xf$?PY=y+flKWS2A6!G-MO}b4uhjze9zTO+e zSg)uZra_N2(eO=_aM=xVAH-go-4lw zq?sbj>FDajH=`Dxus+j{>)`KFu-Jkjn~LyhbQpsbEphAok>ERb{v%C`4j`I&=_Nj1 zooDd5GFroR=bXdQ?CaKWOt!rW%aea`EDmzYdO42s8o^k&kH9=R{q=Uz3@3WNt4X8IL%6nzSqy#`Y2IcDAa5F5vJ8|DbRvx z?$^>?j1^jm%Q`8Mto8imYB{<3#)F1eWD0h+f3GJr{j_y(_2ZeAlp6a?{wLkr!8{zB zZ)wV#hUXmX;wEWkcwVldfBb&bOpO19n61U%Zb89GK z44eZ5QC>VeWKh&MV<`GW_R7UF2uhzhRW3A+pQP~OE7bGbq&#VKlDPL=hjD`mL|>ou z}~^_oJci;f4;t=rjINW6Frn4i`` zHdxN@*xnt0*j8S;^SB(Oh75)<3wKU{%(`_DE3>l(xP(e}Z@i#cBmgT)J|l7U*Df<@ zGdg_BV6%Lx`=?_5P9z#U98XR5)laB-tv{1}Y8(|^2#yCYKa6Dmk+jS=s+)dLi13tC ztV6ojwHt#aN{50bJ28QV>-gsHl<0F0WM2stsCYa+6yUmn9wqu>)fQs&khQJ-iYk9g zg#2n(f9rDj|A8BE8cDYpl=Vw)Fz7OxZ<>$(?G|2tf%Lw`GUW164`-k?La|@=Rfl?$ z5r$CN=0+x27Oc|TT!HG{*CRM+zv-1{*QCs$ypAg2RJCSC-Eva4RH~S5#fN1Ojs|gz z=M29ggQq`@293PRHXZpt)xDcP1WO<0J4I|X`?41@^)Pj-$KxZMxRwgg$CqCqf}3R4 zNHna~7W?-a^hf(DDU!6`cLXHd<;vmyCfx=Pn`0XLl>6So$!KoAOJwax_P(*yd{W_j zvoPH!;-OchyFYYmcO}HOxJ94f884J`Wbm@8U;dXuK*3gtzgE6@%$r|Lv3cb#?GORY zOuXG$zTjm{!zMQHD+F$z^Kf!!VXzVZ=b8G`<0Q=>IFN7Q(OFiUa1w}Xx#T4sBCD$m z;gQJeA*!RDQv9BB!9qC2t&Vpkf>L8b-VJ&={uKK2y`J?D|I8xPLS}JE00(?ce5H-1 zrZSiXZCUFc;Hhj4XMb{>2FLA>Ib`2;UG7a*LaVyqf$ZDVRKaFE^9nd83an=EPqZm`3tbhq1mwrbIK_&AU6g4;5C3j)sr{6DYQlZIv~ZlNd;7dIowvM%!KP%O)^A5!ot{ZQG7+16Hpq72|^^Vv#T z2F!jQU!V^=jiV|@dO8_#fp`HYE9awwTDCKW?|dUl_u9PdrvBSCD#G0PQV$M)oBL18 z3s;Pc2Ex5ZODFtLhEEz72?WYQGDINkyeTL3*)o$MlEYH~%K*JsdkuF35}@ztQJ)4W zfv9A%g~II3f=$^t81VwqGM&m8+|APZ545+4w%>6_@0Xwe_F~ufV1+>_a*+plev>P> zr~FxWu>p&1s~)(O4k&$ihBC^ZUP=^tjh|PDzxT*&fqT1uTzJY-_;gRld#cfPujOyyi6# zQZ5Z4;1AJY&LC;O&I6~BLMTNM1(I0LB$7%wp4SfbnBiIc9zpDg@<3Rxv6AhZcbipR z&7R-rEE$MvbhWfDDG;awB!Ygim1dNWZBF5?*9w9xyASHVwkNVCz+uN zn1>Ea2LE30p+wc_?xv>MgNP`)&mvwG8wAthK5fyc_&Jd>!d)i{BlLHgPdSo#Gaf)_ zi664zb%btZUYXJoyu@-u!P^;VK#iA6aGK7}@r3DEHt2M+Oo>NT>c1pMr#0C0-!!`K zb=&Fof2*mk9}!WZZnKfck`*nyzd$}OXFDRA~(oQVlUt6ANPCRak>%>~WiF!0I ztJ2Du>n}LpE{Fsja<=>}qMAl=dNh{l`0{Dyx0uq!=~#1we8z>gSlN|^0P**bn;~Is zp?byUYMvdhgiW7*-Q7}a5XHNG_gzo9F-(f_EqISN2vF%|JMLGreNSV^9^j0u+V-^S zPWBcjoh!c_8&KjT9_+I&mbc<(x18;K;rraH5z+s9lJwUpAZA}x85#qJ8sarnK%sbfUJ+)XV{0u}Su#Nh)_c zdMMPP{CPIU|LT867akcO+_C4llupd#QuYBv7jMnE@9QCFG$axL4$(j{5mHcR;~ezj z5y@vrda*;1yqtd$Ne0Cm*a++g!B;Lk()dqUo!%7IAtj>FQ$17q+l(N8VG;S$Y3LaNIb&9a=(uvyld?)K+4$ zuw+)```D3kxPdiC+WCewD)>pLfcM9A_f?$aJxHD6E1ctg8dR|2b!6QToV`g{&d8j> z*dod{YCSk2uegMiW*_cEF~^mgDBc@q>o6UM-6B42+< zd_RPP;|kQ+4k4YBV+4b{UJ?azjV1+ysB|HH4fws;L{i%h^-VY`yU0{<;RETXUw2o? z$2xiJl+y}|V`rGG)zQov3`d6d4WY>FFv+8KS#hMQ?;{X9Se4?Vliq0ob~-wNCa=(9 ztjJPXVK9f9O$=w}iBGtv%uWH9YL$fa0@dhRw#DA+&#w=ssiymjt$&PzY#{vTXsO)_ zfhQDuz8WiJS#o?sC3f2E_Rwkk6u$M9@&YN)Vg-r6X)Xu3qCGU;0B#odZ25ktM|=*- z%aI1Wro-&RWq{=z={X~`peQ}nvzzWfzr*~#d~K{O|NhbPX9*#>mN|eXbN2CP(ovm) zh-42O0>kS5JMj#kUdJ%ohRqj9;>%OW`$(rQ-dFhiLsB6yIv!D&{037vmZ;i0iL4^_ z>c3FJo*wjaD_nBb)<7@iAa~7c(Gu0)`Uf5>SfzlRX`l#GIgZ~nF%_W`WFp?99!-X9 z6b)mC?#QV6hR8s;?e;vJyD61W^A)lNw&p6)q|4_w}Z1VX9Ld+nk zZx|0Eqzqf=-CLND3B4pIgip^BFJV|z+sk{!GW<6@xW`6hC zGmYJ2v!xQIdanFxtvOD9K{S+deA||ShSbg!k*gh^ut+l=7+0^UKUCRg^l-MTQh?*y zY@6#vWSaA6o&gO1WjxiNMho}F8Ii;@K>sl|p6|Y(k!KP9OrBN6 zxy@P2=;$qAM@y8$QU%Raaswp)$D#x{7)UNoOPJT*eO8%0G9drwA;kDKO#uT#onogX zZH!VW=$8J&Q^Ip0rs^id42}IrP9FuhoLP?ml+IJCh)T<%M5p;Y z=?rrvk3^K*7*bCw5Vy_RjM`6gZJFXV@F6f0vf@3G$4}=WnCg#?SG^@x3OKW)EP3#{(v9>KcmX|xy0wXoU~9rXub5iw zj$Z8FQgK{lGA6E4~nX4)s2-$+*dsX*He#MSrc`~u029H5>AGjobPLiOBhZLTGf z2i(-?Uw3!f+LmKW$=F=Nt9HKM|7>(Gzd6!z{b0;NYHe{clwC^FnRSV&VCD39qbDa7 zELR|M>*P+SZ_qQlsM88MfLV&o>qZFI;ga;()}3Onzva*@iaG8r80{E({O3Ic zeLjSwcz%xUgb4bPgcSP3tX>lG&;a?)EoT?xNGjFtC@j8;BVp0Jk`;O3jzgYDmX2WJ zVh`~>L8<7ps4V-!%TTNSQ5x#nbK=)NdA2K@#PKkhtNy1Nn^xScKar^ncr)yDO+Hxv z){YLBRSju@i*^iyJ%4zbAE6aV5~CPc3=kwT(=op3mnMTFsUyc``UOEyBwg2Y4Nq~< zGrz6(_+L$AIJv@K7U(3(@Zfspqz|-C7jda~p9=bj6?eQEi8bfuKfEAC&00vaD|gGY zWA@p_LK28zSl0D--m&?uU*S6P@fEV}LAAYi)p5(>j_2!@AUey+hJ}}8?dQXb?V%b^ zw+h%x>xxD&H9`DsJ zx__AqOb*md8|tZS7tVbjJJb{HYL-^KtwvB@61*}Xohh@^5%P&p#l8K>pr1#Jt?8b_ zOA-OX=7yUIVBBkQ!+w2nu)g*j*y@&P1%btLA!BqPU7cO3ZHjW37!_FnB3l2cT@WHD zbZP9NAJTpTA>1#Jzj7PUrpI-bC=aC|E*xUbUH2?Z>U8;5`LyrV7=l9@QwNZJ%`=eq zHs1d3+Gj6jq}{TZ*O5!U9l7aNFJ-X=P^F6M;FUxi?QM`fM7RxWSWOJ*z-$rkZfKFL9;#83+!~e94D3L%pNK1&>3Tzp3~L z!B!>wg)F0U6FZ%erHNkqAM*bxKl+`0JQc4qW(zzj^a+6;S#jyl!LkT1;imS?-f$YP z;=>x$$0-Rj6>wUbfZ4C{G4u9M!#@rxFL!?#LV8QEvD4;CY$=$Y_=!ZTGw$T%if@o zELE}KQ-{rK*W5%gkE9)@digUf9o`el4<%U52=y8KuKLalBsdt2bgh&Ye=owUY>0ZY zUu15#=ue2TjJ{HVT7R636cR2iqF9R*_su7{*5Z?fyKBl40`0uQn8Yfr-q|gC&wnn< zFHIa0!4%LJ!0vTr{95@N$-}Io$l11?e#r`}^R-rtqCT=juKHoy3U$4alwLFA-x^|j z6HCZ7W@C}bHxrenxr5os(o!Q@r-LM$QBRK=g$ww-Tkp8V^0Zmb+{M4?IQmf>h`uIa zh(a0$w&^atbC;mp`O06Qx^nKsI)cpE)^0ZqwbL)`J5-t1K8#k5nr566bn1Q4znHgK zyz!s^F974?DVDErKlK>#`J^JNmgO!N&-K#+tJk&j%2n#E*Z=nDNu=gyvI*xp+64-WCODn)No?L7@VBtxH`c`zR9@TV=#)aFvy<5`Wh2z3Ox zY1J#^70&aMMV4JD!z6zON;ERd$qsB;BUo5NvaAF@b(#=tq&?}NwFVid$MMI27@h#D zlNMg=F3*7Qh(xFK%-J5Xw;vu?i=>Q>fYewcm^@Mz@t_0pg_MeU-fgVJHr@UJARr2M zND+?Q{k1U#FEfk&RK<0$IAhB=$19^gwb0htmY@l+8i ziGs~1o~%Bx2aX$OBR3`uR$6^4BYxindmva`?%ngBx|dK!JM@P7VH3D%bYlr1-#(mp zqjKdIP3!Zu(ZL?7>)EAP9?mo`Oq&~!c%89k^N-KgjoM^aP>GCi_!hbtA-N5ad0$lH z@S*QD6Q*u3}OD0C9{Ocd^rS`7-L z3LAW6=zYA?I&j?Nabnx4PILap@rW>dc=pdgY)Ub&4Y8t_=E}*TFMZp%#jF1yNf-7; zNFMxs8T%T@Psnx46jWtJXqYc{|?2yshNTMgZM2UUBND@P{xJU1%WG zDOvAguiW{DM~)wt)13_Z%b(pMob12HOE?Ybe%2fKLLy%D;02ZGuNHxz62@i^^@>9v zokG4RI$HR)5jBeQO8ISw<+_3+2`6d3D08l35zq(1>;eUkqHaP}v3k~${j7^Q8>Vd?v}_KR()PndgAGpH?zggW|+*Se28D0l%1$A&hL zQ@LEP@Z_UwG;t${9CjE7=Vn4%U19-5ugO%D12Hdnar4b|>cP|8Cf9(qs6Rf1&~*VAJlmJC8Z(2nRE(oQHxq9*0E7C@ZKa!WMc8vz-d_WbYY8M z?5|#`Adl9n+yA*OclMgaetAsiS4#JWD2Dyq>U)0g<4KRDj|f`Ehz&+&{Km5H@1bN;Z(_S_C`GhAyN)qmV@wtc`?4|Kc&MJ z1#LiG5l=N5OwUb9xx3ozcxd%h)lN8lR(QSoqeGSwylpUs^akIH!W3Vf>{p2zOm#3{ z3O??~`11WZ<_z>$@f>`5IMj(})FyXEfA5kz>Uz}Lzo#Dc>1tf zA;^PGIzgWpru2l!YtJq*;{qux$;K{~0R&oh|%7-qLc3kOh3^XyQyi+@XkCR2!6g!M%Q9Ntf026NtvCfwxu&<15 zjhGvXC+soPWT)alaD1xuI+j0ijHP_G42V$2iAuQC`Jh0(WC^f+r|K!R{)3t0reAbCAuYzf2)+LJE6 z#u#Xs0BjR;!?Q&n(bBGjx=0s{Jp*%3k{KtEzaHm;O4b#K$953bQ+Tu0fDS4&@ZN2N zF{ku7$M&aS>V6LV-LbF#+%q-?Pzb^L)#<(EEam!JC-H*mpaMI}Q&%w{iKpZzk;J)z zfepq(sL2HyNMM%NXd6wrayY{=E|{fkx52kLS_1f2Y=+uP>0RJbYOq@>CwR5AFg`E5 z+Eg{9Ron5Mm$b#xOuVIT9lTdUWt@q@7ged5gUQdEd6MY*>F#5QZc`oUQ-8uE_!6Lf znFwu$2XJ3$9>f?fLGk!^SLa7hXyNulEm(ePAyKLe*!5eNklTv*BCWTO@uc?Os?enj zz5FJ5*!Jm4=c~jOKCC|$t5@y#Ztwd01y>oqyg#Q)IJcfGU-enYr2hPJ3&1pp7 zM^a!OW)EMER@P{*#uxV{6gFM>mRgb=;L>}qSGaNHQ&hnvyB3u=+2M1;@WUeZ;Wtdd zR6Ppfw`}Fl&iQ9O!C8j9J3T$RdH04fizN2{YpDL}dyC+`veHcLs)eo#s$E5l=EQ%d z7X45P2z>_$1d2|R)<{-o#-+MH-6D$y%|05R0oq=r$VIaEoq;U6=@ese{IyhbH*$H5 z$n1DK_%FvOnOP(CmJMw%hp#u`j!UN}Vr)frN%bHNk9keAklQcc1!8Ii5>Q4%NkF#tdwBsrOB_oxKy=LEXje}G<)-}EKcRD}c-UYEfb%Bip@KO_AbjE-o8-KBq z+qVT`@KeaY&qAhUt?CuQqpmy>HA3Bx3yMOE<1Jld;lpdQ8au@ZA+f1%xp%?z?{Gm} z{wnr`HM|TgwA7;EY^F1@%`FM6RFgW{wF*rkE+r%}l;lB@ZuRy8!gJwD=K4gH!aXP> zR1<1^PM#v$5PID@Q3S&HjKNN&fMKj7g6o)Rml&=2_Cgb5`MLkc)?0u@mG=L`!vIQ% zbcoVj0t(V0f^>I-h;%6^9SV{HN_W@LNOyOMbcm#YNXO9cJ?Ogo{-57<&9w^)3(Pt9 z`Q|5hgZU250!+$L-=KeMW!TKX8t#ah-ovW0#OJYvxuS|z05jD(bbNYe;ty%A8K=C! zujAJ7ZWqN_eUCpkm(_T`X4G@pdO#)-DBm1Td#8RCjA9ISO47tn}28$tk|f|9-Z%cH{$7<>I^2z zJj(d|)eBmzQ1RQ(OiU2L?lY zQ1Yw}Is?8a7?+pM7(1Dn@U-@K&FXHBuo97P8t3IFE9`I?fZmTq(o)HBzN^734GpdC zxd2;rR&uqP>+Snj1pzkxBLEPW{55~R%F|dYIjTHW4N5J1bjWt*r^pK79ZO%{x!wiAxVf)G()?XThIwvwJmicc9I3k%h9d0dc2c9Bs-(}^F0ItY1U>JvrqEjBHc?Thl}2UzkR=`6G3i1YQhG$ zw|Wx3vB`A^&b7lp%-dh<&|LOMGx)^COcCiv@Pl8qSW1ny;>6UjMdPfAVU!s0+p#J@wPVex9SqoOl-y z4ZwKA8H?3mBg<2^s4(y7-1~Gh2+2t> zpIv^FC)|ql6Rg;d&cM-)N@zkCnP%dO{c$Q5^*Q){E#qeBG-x1l1Xdzd&*hJl%}n1V zOqtH+HaYJuro*IX85KotkkFn15t30yFrBi9np8{X2M7h4l#<7OjN=g+9n1&vR;s9U zT#ka^jI|c!VtF5R(}WRNTmT=)o_DApRZVFM6eKj!WC0<`qZ!hi111TxZ$5Vd6`6uE z)O$o#X2~w}8&DY>Z*Gs%gIq7*$>{jR|Mp>&fdR|8-b>-`3cLBn%Xq33F68ELT zN+9VHzVKFun>4FK3oXS^A!9&GOCk*j1pLS?_dUJvg50POZYq_TQ5GwWFd-vHz9 zf-oxQkokP%8@F~!NNz&awghT%6>*@i;qa}~A% z1k)Q)zkS9aAyEJ9s>AiWP4$UV^r?lX-a;>VGX|PP!9=)q};>Q;~j&NgQg?B zr<&q#hG7&Y*BsGIwol8tV z?C%zdG!mVZ(s!V&rG2o&Q=)@QhKIYaDU#B_r9DR&`z1J0E&Ko~u~2_^9kc6mScel{ zxu0FMM%q(tPatvLPm?rq?v(rmk0Qxx9A*;K7469tB^=Ts79QeZA(3#8Kz3~nQ?2+x1VSHG(*7=|7Y4_6l`{4*a;4T3hyxktS%esRyMo+8+gNeM%;Ma)}CJ9>T_ z-dd0BQDhNSCbq_89TT6jWT6zd*9J7^2Evxg>QkdrVW5Y1SaMnRX^tG~Z6pKUtxcTn zKvTyN5XaIoNWTIr72KYYCNHly!1JmG$c(c<{kE!?ZodIMc(;@y{ko@(YPB0*bHa*G zk#f1+;XCQQnrfN+YpL(jVVa_k)sPof532njL|xu_tu>IC=fxUz_ohs&DwAa(pHoSmA&5!hqaVe4h$+nb6q9kZ0f<$udi07e?8mDk(c@u>C zH-~gGk#pwuir#cs|3!MxJCcRr6K3 zPf8s6G5Wwy6uU-BU-sam!Z*Sz~anzu~+%+0*>>>v#nQMOc=`_7n(P>$`m2riUH@^co3(g!JZ zeo!$8cWG+TNhgawQY(Rm@O}42PKV(jkO=~yZS*kB0Mwv)eYal~q>F>VCO((!QSOVE zAX}ed&>LTY^@TP$B>3*IPh54+6{3|!NbnKRqj*#TxqWO6bk|n)gAX?dN&DqdiJCdq zcdME%2FQSS#pR8Y;i|+V1%Y1#NLsz`b`dOA0iY>-F>ti&ytWUP8+6~2Y89g*>z}IM zKT+MK8xULjP>-}*t&#ik6(BAWwKxLgnLfdb@6A4q7`MqnfN|Zcdr|c$9O9v+HvlsY z55=J40_o19spk=sj57$lS8Y1v`+YBD4I<>^*77VYG(V(Rr2Q?hQTL?aumY&JMt-6( z2!lf6+RXX=8%o)|-+?m1G6EjL@=kfmGDiQkL!oZfkrI&?zPA}tS*{LRWCwlNa z-f#{wSnY(N*W6V@EvSIzKxwzxIzJ$cdlsE5$~PQk#DZLimsJpxlqmb{=0o!$gJfaZ z>3>&Of48eTn$3^`9TC*%-;>mJ4Cm%dJfQN9jD{kPe^VFbbGKs2c#Le&0@c8Kk#M|S z4y1=R2}QU*V)rn8?tH+iA|-|@QrY95j?jzvaBt!p@_lAU_Qo)Ia6@)4D}@)r+bm@5 zHC+qcQFcdC7k;WFFTEQ;F-QduWD8fiw~5|`s|xvax*mc4Xv8%jW|_gO@*V?4r@ca% zT;txC_=;k-ANMeS>hD}lG+mDQkNUH6-4;;E9M-KB_dIO`H6i;!?}%*r_Ipui^LYZ7v{K!S*2!>HjT7H z_!{)iPa#C!69_DYoiBD*<}b=Ac6{mLNXy)?mcjiq3vF$aqTY6 z-czvQ7Lqyy@_#2VEwz{Dr$37`=7WL5M^VNik=GTZ%-bcfWnSqhkGFGP@A5U$^9C>- zyR@4ELgl}5ZM&U~*gT6>{KA`-UdR%5BkQj>;0k4SLXxshxqXR#Of=`ex2Lzk`Zu)w zh2@VHA=HqHE+5rh4}xrWzPDUHwaRo}JMqt8U_&RtFuvms@$ojyJwkCF`3%`mVnf3Z z7t>XLyAz1~(3`_sh)O(Tm;c5&HO>vA?0Ym%V`X{ADOey4Xs@7?9(Y|#v&;|*B~=~#3?$m^nyc|vg{*7VoXljuCv9RAfxY`ti8`MU_A2n!5|q46vZ;A6$h z{8o+t=N%to$IHh2NW?bxeZ@7TtGn16#<$`ZhQjT1Cq};}TTeuk?y=PFW}8+okD^3z zjaSosLKwiy+*fa5MveTaX<)c$0x{917g@bMqiwROGlNRw=-rkpf6cjd#*8ztInsk5`f(PqZ155yk^go(P>d! z*Op(rZq5|}=850@?Hr^+FsL;393-M-WPveE{#{TS-Awj&7ZX^|jJRhA=sJP5dGQ@T z?VlO;j~yo+cEW`tx1o-mCOpo-TO>8V!Daare|@43m62ek+@NKb*-)#3hNAYSlTwbAKt6%_lIm3z zZfekc7)1MAKO2vz$pq*+EV7v3DsjTydbe$0&u~pq?-s^>}FL+g6mQ zbv-Lj8&pdR5&5q>KVC>&^)vJQ#>15e5};i%lwzaC__q>${hh}|F?_F7bypkz@9cr^CZ3QA8c@{Rp_eG(AP ziL(m(;UBG*mF_AN?Lcz{DEG=8r~w$}+h!tXaW|nbUIKpaxm-aok=dprD;hxnLXSZu z(_v$fo6WFvS_9g<3m33^n;WzFeUV7Zp+G?DXsiUDzzfbMHBN9)gxZ@-7@;~UlIe+C zCs79Jcz@i(xmG_+p>kC3bDf1NaU&y`DH$*$umygTdyOn#d4&2o(d(hLQ12I3!|HSO z%&o_TumocdwPNmTavk_rQba5lm9O4f8lo6~cgM6T-cEN`{ClY=m^XfhXEQoS}-6#_KSt(>pSi+VSt zp|8at0rg^D>LnNpY4$=3{N3n!(1X6Y&vKq53L58 zpN?P|$~mmXI228W84~Yd4KXwt3scDvcdA+2^n7qmE7HCs?TU8L(VbLG&_hvQcdxLC zcTa*^fw8OuR!1>ShZjh4W930=AMwT!Z{mF_grb1XgK`#U{DsvYXi%;$;%H73_Osso zo%cea1DK-Cvts<1hHl)POh^(o7$uLH@F`J7N8Ovf)!wM*GOtK_gRW2b6((UO0VWO; z7OYkCsa|Rsmy|3{%y`ObjeC9Np)Rt#|?o_*Hj+x}MP@%x1kC8Zi8oDwxs4hg!< zWFfcc*%T}v)Bl#mqUv8)MBV_ zoJ-6;d*%t^xBO@MniJ^ZP;A_#R=wix(Ly@`@WVpgcMnHgt~ z>(xN$$~JlV?Sd<&QWyaBQ;`-E8jx5+lH=%Gp`)`-&n7_gE~n1iu} z4#Mq@f6Yw%0?zHWV5oB7iFJ}d84V|O*(O8oF1D=S=g6mUi-i_g_q)s$R4M$f(m3^1l`vqgt&)Jyuv9JTq z`x8_Yqz$ySXPD1XM)h9{-N22#6!CrO0n!+ z9P3Q6!NRLXWH1bgl#t10c^Hm_0W^QdSR~QV^WN8ZzT`~qpE1dB6VfFowve`NcKLiI z2KR_(BNU`2DAuMYR{A56f*b>PI@@Ek_zl+*mcuAI`NmE@COde!o83<)Q!uf)K3Jq7 zQ1@JVNH8|BWC+O@qwyD*fKHFd0^kI$ZDVO)3|*Zo&dCgxUH|7)az-365p7XIS@bj* zt5%-cN}f{mov2|=A|;8?0SufwuNG5Q@+X64(jc~UcQ^yzAwUF}70*NlcKYyDU7>n@ zzcTlm9kfRr$UfW5X?QbJ5bN1Lo>M6EF_n~nl`yOG^&#WUfNXwFOjws|@Kik-$fD0F z<9=9rku7S&D9U}$;O5lNlxd~;RaQ+T?*agZIWzWdQ7eOXrpqjV#rp!Vm9^jjV6tP- zBf8>Xa>O;ofNAcBj9SX?y9kW>W3AL_pbXRO0m@-u$m3uux?l{!!WHd>7T>C6KE^0W zxjbxm87ah?=V|aJy5c$y|F`xlaL1Swm}Qygt0F|~Fvv8%zk_x|t?}NnQ7z+VlfM<% z2z$q@?1iMYKtdVPOeva-gYnGJJ_M-m=f2-HKBtAt?3dD2ijo}#-tiAwi}Cw@$LdCa z%s%#Gst=0OPfs)k=Zdr?R_I<5FK0O5vmmA7tU@8_B3s2yxHMdQS&Gxruyw;PR$N0jD|jz{3+(lO3Yh|f&;-#2buCBU4iUE z7x3@=xQ;~RU6QpUo$%&zDo-V~QvNgB--EWlM|`v%IvYv*6+iW#s$0${3nLRr02&1O zIN~1tG%r^z>ZWJ9j=IJQQ^#{p*h|xs#dqk9cmY`?(z+e;`Cv5zVS1K#s-}`yqUf#0 zR2Y!NlMQnPSW5@Gq?v@yFqC>mDR)oZ zE?Hk)KNCQwP8SJ`7q|B9@)5oV{JGxElr3V-^v7No;VM8Hxl)6C82qghJ41Bbvq|D_=i;Vjj$S)59YBq}=mY&Iz(8(*&!01^p^9 zC1F!&E1y_N(7j1wDWX-BaQZo}0`wIk)!axEg{M(dknArwOioY|Sp@%+Zd5VsO``u1#t#nDIOGl;Ie>n7K??BN$!7M3hnL zC$d((_z(4lMh8xFl5e(XuK5=IJ0Kz`!JtuTtvporc`*=Q1Kbv>!2+fBS~{J?9q6{q z0$qoddD&!OTyR&Uaq&u;vH;*rohkD{=iNv;4(gB}kFyZ87r>b51Nfl!!q0>JhsqdD z2FS%5D|z{)nOOBOPA0bS-%oBchMM=y<=uGk9XQYr(fGdkDC*xwdej%u!1)FdA&_qf z4UtsITjS_0_(-C1QK&GqN@W>Q76+sECdSra+kDHz0W+W7!FqL3EDF{OQ*kgnYD?@v zW(*>6$VfS9Gl4%8zRmAv^l@BN@Z85mteL@L4*o!aAF$PUMLZu2wB2*Z<-vqeOKKC* z5}uZLiqdvk3X@G}C`FF@+2N^4H0`Msc<5}e9Zv8mi@3eI+1DY3u_it&qHSszf-6561zfZ{V5lnKw5~1BGLGpXq?^j#89pnmraw zE{ikhPOD;?NHdN%4t>6LdUws=nU)mwyD}Ap-Q!b+`^6EQ9auo>iCw>!nd>?h0$YS5Ywax{ZibROM zCQNE~l5!^f;65!TteN@BOv6-wkaoZl_qjU0x-=}ub3=Xv;@+P(#+?b_Vd|+i?)Wn~ zrc8l)i@5mEQ!nb?RzszXM5RP8pngr1n*~Cs$+8|sqslBRpi0;$9!B;?oO3{!Sgkx4 zU#QRRAu%@r0X9tw^#(Dx44=QlKfC#oJ!Yd~FO6@n=7%+bFVm}u2Ddi|QRtfeA(8ou z-7qi5U(*%)sbeL^ZL@dG5*P9%Dy*kQGza*B{njjSHCP=taI@ig(NFRQXhS-i=L@g3 zyODZ7UpQMmw^l_8*}&DcE|u`mdcP8@iX_Z}Yn?@PFjW&x`VWFCHw%F929qFfpY1*IEA5ZHFY2eM05S9>8Y(+;@sqNTae}E^Dqo9HK?CAM;>cdQ)YrzB{faubrzjTy>g{ z@&%e#8fIe%L(}luO+k#aEam#Ls8wty`sgE6GmA@(7rZ&=6|XC`;~x;Qtq^z0SVl0s zzoD%}Xw?;am&Vh5;1be>(HLRi(PXb*v|wO)`bjry7{S8kxBlHw{oCbG#z1FN+vG`i zCvOaqL)KuB;g6s}zyj^2Wg6T9YV zgvaQY!YuJ+@Bxyp#XdehjR8?6;b2RwicZw$*U#7so1N-q-xV)+jlX-=a<4M+`l`KKN*Qp(Z?0ZD9^omj#l+s54RmSCEdjkin5)$>`Ek7 zVK^=tQKR-PZ)hqsm)1OR3_TlNQ$`{q0$YS>6ZSeG$M2*6YS=se{-GEVgKlLT`b;t} z8oj^+l3`i-riKjZne+>Le{7^mUWb|7JtBv%A%Yxx;_OCDqW3qmVhr|lJrZi3S08yvP=p+%bC=s!YMt5;XC*P92eVd-n2_0d6BlIfeO^OSV z)%IuX#24GIzVUSmTNUyl%zcHel3pF@Sv6W|=w`7T>Po|-j-?ZIeDYPu3Tx@InIbIn zgR^X0=27IDQRGt1aKdr&%?`#JHZ4$ap2;WibBEmJxqk~BCIn*3c%?UL0YFyJ>>3U= zCd(~UdBGOG4a6oQpk;rD-0OUIr_b>6!7O4z9x#@#S_W zXlxM7kxwDjP=ZxfW={nx51)+LlTd$=Z3MY0fcs! zu+nP0AdtniN4Uv@3ZDyW=8aS(?CbZJ%KTIx<AyE3U6* z%~RRmO4Cl!(H)bft`H<9wFvXcKmC||*9mfDDWZ=nyE1)gIB}lhp!xQpjkd$lk35|-ID^JvmX2FRBrc}l$kG1!ewV$N``GkJf7Q^og!0~#Xlmxche-Hqw^86{ z*-a$(sik9eH+(acxq zbi}{E^UjshqfJxd{urJPqYP(b=P;W=b>)J>z}cPc=fAJ)Lo|4nq|lOJUCaOqjM~rtJ*e<+(u7tb`6U-S;o@ z4OS#rPa1O;S*A4hPZa1|^CTeq{%m2D((X})9)To#4#PRb(S^6#j>+P;p9U{s3CyDp znLSI&i_2YGj3cgO?4$VpIbKQ*|5jd3RL-98$Xz&KbGtBVN~5&HOva{Tnyi|3ynHD< zmY$MT_B|r>TA$T?S@1JcXb~r@vV6!zrb81A)y4ahVnD4W;pNKD)5+^-c75i-u~&hf z5X)zpZa$3jS5%B|kD$m{iG%S=pWR&i||9~gIOn6G>xdY0v3 zph#iA-;ld?kgjfE6s)4SGVjTiuI!$+Jgp!0rj!JAo>utT$08&3K&vdRF(OVr#L-5) z&^V61?o)QY^<4kh?L0GwjWBv1k(H)VHR&_avs4T2UP>rjl|J*KfM(8X{ zZq8ML6MM4^L$6vq{F{=d_d9OHDJ4ax)S#-MhqYvzN3rhhqo)*zW~#&XR@ONyD}tJK z0^zXeu^96V?#G0W0-Ek`2o+{uR?#zlMS0>rKEg1abJccs%c4SYa*VpM^+${?Tt!d2 zfO~gpe@4>}OW=w?;g^@-*lXE!X1$7Jw?rw%*X#31WcT@hy(MBP%=$3Qpu#p`P$j0*-2eN5A9A zurmq{7uDiNk^uFz@qXU>MO<{H`1$-5^up;s@R-M_BVoYrC*7k%HSlHOFPaB5V|Y34 zmE&Gh>iC{n4m`zp8v1HCa@-3nSq^`A{O5T4;Gk*%Ij&n^ijNLVKegZp!lNfxI!Tri z<{LX;$Gg!K#5)1_fFFSFa7OF9tVAt<^}xEluWLdD>4r3Wzj8_g;tK!ID_72&k5XUA zzi10pYh669=)L-hT5UVxNKVH<@-ZstnPqefOu1R6erV>YczWsv!=F0gP*%`4wh|>6 zpJ6Lk1nHg{<||<^R2+G2Tt-+0;iuSac&%1Ni(gscreIAyN`JAdsmq&1T_ba)I^+5) zOPGaom|44GaxsHwK{u;1(GC$R<^mz?sZw1}=JM>ef)U4ZlmB}YFp<`OC6FJ|a?EY@l0w(AsDC?+1@rxc>smY=Fw z{mOY=ctXJPESFAWQ&pfsLO5@vS367PitGA9tXZIplqPT29hQIIz6@&iMnZ1=B1u6; zub6!AX$T3fX4@jFQE~mAj2@O1fVeJzh!vb~0GzETgUTP-OcN9}U7l(wdQQKRJu6S3 zpH}cP`}$xz;cuBBnt``-obXiWRp`v0M?4;Vv;q>Gz$#;KAwyda?#CKB3eLtTGqWXT zSY<$v<)LnVE=c}A6$Q1~la{+oGlSn; zQR4^cw~jy|gwG4slfaqY)!9vQNciqn14vqmRL%3c!k%p}75SF7a;~qbf{_w^kPNEU z=;(&(8HJZSf!5(brHc>l+tk81Da@HhhD5DtI~`+Y8=>t4lM}^XK}V&1Gb0-D%PC`p z<*kGcInvY*$0R}|QkdxihjsAz#w;ZTDlo?OUMo9C9m-#{6Ss2w&NfK+-S|?5uFGzh zF8sn<(x5s6gLjsq_lM?39#LZ@MECVbH0x_OVEL^;a~4mt|Mc5&BQY4{5CBRsnefHB zVpScQ73e!3vqZF|%qutD%VE$D+{#CT$AwG%o49|cZzJfTd}QIAcNq5bXG`H;06Be{R7L*Tazy>EfaRQ0Hp}iEjS>VM-JzI2!Wq zo4aMAj2~ncb3}MdpQfpo=|o%O-pqnht-NWkR1vW7tWD_@ZR{P*DHoh$$MecsnUvVC9Rg|Vgt-#EdE6NNIQxW5$LnbN;p zwVtVDW7Vx>6$J3RG|ggzkN0LAQ~KSLd#mi`$u)}fwC~3xRgSK(6-*-N5Dtc>>qH3$ zu@~v}TwZ`4im93{mlkN=7rh7YR3dVTyrr9pCzBELIf^M9#;qu2N#IYY^@ezf9Me;^ zg1S+}?a3FX>u`F8J8Y?lr%!k|{48}YuRKb+@F;3i>q6#-<*|99$?w`B0-dAtYp~Is zeW2yu#IR&PFHeal8sG#x!rucmxc4hl7)1yoUo~)DS+dkhfwwcX5;yMn!Ry|lSpm}- zW1!!$0C4BiCa$xe7mgL1f_e8YVuU)^gAlR%e_lya*JnuFOl@A_|NZLlS2W7@joa&> z!Nr<~ck;9XzDXCOR=Ogdn1!M4UPbXQfQU@5mg)LJc}AtX;spru#(mLj%(gMVgwHnd zN4`^{4F97Au$wnc>jj3=DgdECtH1(jS1;o2l zVRR0vfaS50js0_8X2fAy!+@f>(QdgM&&-EXm9&1n?4qDJeUYN`b-X}9l3~$uNStCu zfT{B_cR_eor7@^Tx-g#eiDk`_x&sIEDZ{dP#Aaqi&>Oq$JtOXV(ds zdcF=kqXEF!-h&?Qv+n(wQWHtN8f<ROYBO|B|kCpEJ7hIrZq<;L3n|dT@OgfSP8Jwr|J9f|}9W@9p`+V^gT#?Q26IS1HxUQhULvwVPbk z#@F}T9*R~pqQ*rp-Nl+$`0rCKmg_H(O<4x!-5yaEoA#>nwf+hXc=8JH?ER~WtfZ3D&2~Ix6Tf&cTk`clBzpqgqp99aPA5^2q zH7_0YCB4t`0g?p^5Ei}Qpb5k~29u>(djM6Id_1n`b^3J&0PC6bfNkr73B{RKtgwgc zn9=?sOVe^T^L^sKjlClc^kn;Sq0@j#( z|4~TgVnfm5wbEdWrSYSmhlF+Se?kj7CuGrRci(f~xdNPi$uE&8D$9r{ng&?%EAGtJ zV0-S)>NQzXHUDi0hl8PcwU)+K_?ucRA2m5TDrEz~ ztn;mrb)og49lM{)lB<^s^~@TfQBbJ`fS2k^i5rhU;84n*w5wVrCJ=6IoZCwYdOSF0b#aj@$ zToQq%qXZ_Xq($ycg0TMCIwB}Viz!H)oJDqB%LNUFW0{o)I1dX5UvM)Eb+!ju${WD{ z(^ffq&r#PRB`7jaP>B9fp?#M7o`J`q`fU@j@Cq=XWUNkC)H%Tra4waw=YG*2y91c# zoiE$914Z?w5u z)yuQLlZ>uLgK!#c?$w+vb_WK>tzg$h2vGN-jLl*|lVj@@bt^FJ)|nzZ4yxvf__n>< zXVvRm|2tb==d(Io(cG=yoj3SgX&!}C%v*gPjV$SZ?=l?}E$z0zW=RQfXON-iYyx<8 zV;~)8dN(jZEd{#g$cjXu`>nHAR}=Rc6NQ`mr*4M4kRAwH)JVN^_~de5uC6RO^#gIW zykB5F*E(a@q#qhaSO*wYgyTi}cTLjIOS{N13*05d!u=*KfyvP>Nec<|VP!x8u;U`p zoDyq41pD{*09MR?^aH%XV7m+n?^fS{KzxQuBhsW3$sBy6bo2;0m5)mN>zO*Dc}g;^ z>0Dv8h3KAp>5S8ZRVkg(VGMQ&?8Q|}MWHb~{QLF=75gf7v-iUO=dE5}Xffv&8>A&W*mKL}?_LWA>QQsqNzhd_kydU70R0xFy8>mDIC1J;$lmlM>F zA0R_;OApT|uiKMfwQ8FF9PME=lno8<6MDxRu4iqHcr0KVe^p*Mv z0Jvy(A6YYEmf)zCP7_`{POWe~++b`?S1fR?uwQ6pESm;OX>m8LwR>be%vM-Yv<=Uu zC(_}T04!?^WD%5o{YXBb<0_EJiaukQ`)dBnIB4t5&UcBku}<1OMfT+c8|G9M&#t5L z$+}Y|y=lH0{V6Y3sWVRj;~$AuRnPv9GKqe;1_i;<0Rjp2h>G?p=k;^|2=ySd$1Y~6 zA!fLX7rN=S>)8ptEe^%YK)8UB!maDy`ok3usqyA6YjK5{mS>BhVlDHm=Jh4=c?yP% z&;kO>iXO?lr0Y~?q=??1z{AcVJ;LL49GUv_-={~^6pEIA;}(h1RH}be^E*NqoPds= zwf^$-r|IKy5KbvHyIohJG@o4pu@+*mlkh2{#Pp&?(FSfi)-Ef>+*aeh;JZZbfg!L@ z4=#X=N+bM}+>vdR>jtgvA_b=D;Ok09j>dP(Tu$ymvKz$JEe#vK%INUFNT`zTBRxqD z>2$I1;(tjIx?}VF1zAs5pQI^}<2ytAKMOcObV1l%bz!qdkn-9a5g5(^ z@TX6V6w}E`-G4fo;Ry7dq+D(!4aL=^#y6wOu%_nCEdV|HPs{VS_sM04f>9AB0;yQ< z#Rqh4b}}d~80!H#!-rV<7YOI~HVHNL7NUxKx&Do2h%0d`&`f%LzeNxmI9@RL1g0uT1zuuIng%DZ*1aLagZ?gj` zw)6RJR|ZLNRsR;q)QDJq>ua`raQ?H>GNRTLI@##OS2o`k%NbnuEs+crS|^g_e%s{Vf0$T6DBhasX^9 z0WriLlt|)Vi0slqPvfzqYoX0%1=c0TyNewQKougCm<=9eHVPH{^hJZ|m zh6J_(iaK@v!Q-H9p8$m=>$?)ps9%6uS5(VOn^RYorJd00KSPuddV2K*O>BvI&i>jB;N-*WFj4Jv5J z>>9e}W-oLYWZ(4rL7SFIC0`X7ErjG3a@br?kp31LzdsO)LrilPd#|TvfH3*O8PtPq z$k>878U}>>V7c34KTGJ(h7j?~n@d}w{KmT(06Ia%!;%Zk% zc2|`%@36&Hqr^2QBZA1Impe2L?P+oulJwv|&&xHN?n*n48F*Au{0?Nv(xO*Srd&}w zl3lQ>C;t7rhDnjc&rZJu`~uM0Dd+87P;J~s`z%#Y8#GpmP?fSj9MkkqcHicN?qdJB z#~eQ7^4?~Cf=x0hEPI=cL<7-KzP@hS3xDx)>^@w-K=OK|>u0jN_z4>+4H zfYC%4v`_)PQh}5S&%j}cp1s!6#+0=-!=IK&*Xg21p2Fal4HuEYFSf@M|Hpv`{i*^A zFifkPX*brebq4UAbhO|z^VwHGvxK-U!LIR+=V88~pFxAG1TGP~|I2>|`aa1r-ewBx z4_S*n$5e|^j(^Y3QrS>8)~~WqwQTraRbG#aZ&Np8tvMP+X|8wnBnTK_rVYj34l=@? zd+jI(XBbm$637@j95sYbXs7Hqu^%3EVCenR+$$-f+!owvw(q=S)*e2I(9&4|(##Id zKys@Mbi-|d49O%wU4jBp`OVApi3cF!2Vr@(VhZ_f&;+?|DAaYe9%FnPNw~V7j8S|r zXRli6k2ggBp=Bpd2s(yfg!1g{9!S%<*r2NqvQ$nQyc!)cfzTC`Hrq z9@F~K_XW}EJ%P?c2i{%(3xQb07X1b?|4`eS& z!-fRgs;=s6Y;uPzH48mLIUgncu)e|$D?)EzhRZSBS7noz)1v?NItlM= zu;1&$4;L`kbNJ-Ak#IpY;n_Icg?K=QvC!7ij%>Xo>mJX~`VG=>*ne-f{~|%`2W+`I z)JqzV8@BxbSe>Ehtsn2t#8-pph;-E#!;akqCKRopL%nyuX898b$SoWeBbkGk<@h=y z+iIs5bV-b2XcCW>_l5MZpRm@xl{%K?@0wdV->Bm?V-^YB^fVh1`uEiR_pxDwN(`+N zv1;K2ZKYARFMgw@X5IRX2$sfUFm>aFR=)*+RK`TCK6nA8uPNur*p=dyd)%3%}8Afkn3NsG<0tCA+wR_Jw}jaK6Nui}FXy@RXof+GZxa!*Kse z>7(-b;-K7sM>qz?O>{F9N7@xSF0yC3Wr|km%$yRpvV}_wBX^`_P?ZBQ-&E|%-N_CIx$V?NTh6jo&C(vEt>CN5JhE3z|1n#1~D zg!OW0$L^|>Qn21#9y7&8$lM_*dlMI9LojD{6k<9_km)F9@z|lxl=Dli#QCq)sZhkb zPj!Nx5W4CV8)M=#(t}zYXj9xmI6^#W^1NsT81TD@+N3&(+e(9XubZX~z%yGv(aE-6 zckhVeMZF9$74`=?mPo)Rb(UILovoFn@y9bwuDDE}+W93ho1e$xDL=^H1Xj2Gj=1V* zJ6<2pC}}G%$<hPjF~CVMu%R}1&zT< zEjZIamHa7}(aV6>7VlGwZLz_F*48DSf%ASMwbJdBb6UAk|={g45{OhSl+lGTH(! z-<&&BKh8Ou*hM>tXC4QAMM9W>y0Qm7q+7v8kR({mJE<4+E$Rg4;S=R5>MK!C_ZG{~9r_HG%y_=w(#z)u zR^X6ksV|6Gdl<;Qnz1vSKqsFf_IJmWPi#C8vF+8Ej3@R}9C%jF+DYqh@kt9yUON9( zl4Ebrt`6HgZMOWUB0onFLgO#O1)tzrJ;CUsh5E=}yacWoyXFTye+~X=z-ogaVg*2( zMKIChXljAD2`&L|!S=)fCV$D*ctlsni2N4Krb`5T4N`m&ZIV@S!=rnzCyk8(V@aRz zWx~vWBAzruz;d6x?Sgs&E|0a}fsXm4!YGdOX~WdKnHA&({{K;haQH}3UWw5>I6m;i zpe~TTUqky%_Z*m6Qy^BD{$Ox>)8Mik6q1OcU6AsL6S@V|`)1!`YeA#bsCu1 zy0>Wv+SnI4+qv6EX-kY-^EF0#HLdy_2!t*=u4GOVE9mY&lQ_$R$gOA#GEmGh;KR$a zI-boCEwY^R6j~D&MnpAXSecc2tiQ_X`niZ57HwN_B(pTYY;nkdp7h%E*=K{qne28> z^e-xWU#`?>bAD3e@=54ASoY2j_z$KP6(%adX32lVy!uwG$b{Ln%e(YZk^d$lbe(c=b{wVE^=qY%4jqIpa~EnaO~EoHEIHd25WX(HEQ)iS%UFtA(4M(S_ZP z?xNweX|bVsx?7UWYiST9HRgNN?DFzQhNpT*H>#TA@Gp1Tk0zg5ewvpoAoQRPWj2~_ zCI6U&ccJRuD_bT1BXBII(U;93V$XWX61Ih^WAYm}zJ9R_}m^#y>IxrIr5|CS_~ zliIoM^$e{enAji8--I@2YaY69nowXy(92^0j|l53lIcF>)Avg58w2%%1quvd&FjSWuOV?@$Nchh!rD34gm#T&E4Nq2b^)gJ#rUE;!S3BFKv|^jmI4PGT~b`O{6Rq` z(_N)rk;I)X-R5&5ztu&dSr0{?vX|NLv4rl7j<;5EHDIRnM%-(v-o+$mu!$|02Db|k!|&cd@wr^ZI| zMPW2Z{B5QduT0u%p?St6wZF($Yhy?O%rZ|v;~>sq+|f>L(IXw2f03|4(J`&7+@tv6 zR@kM|l@$&#jKqH+?3}&35JHHVo8Q{WTBl;6EnXK$wZU+rXu#-jfagSXQOD~@lqoA` zSGj6-nswIAvl~A>a@wLbepq;Jbg$t&hUO&WswCrwx!ueMfcE*%fd_lvIhrHg4T#Dl z1crcfEQup|J{EgZTq3o(vKOD1XKj+fiI-AV3-;MM0|-tqpM^d=rY^L4QlHD{MCmQX z_Bu-pr(RVj>L2|afyRsqG#nTOLPiAgSG&(m*LkfTy!&@!dDG#|XGs-kR}uueAN!L0 zVe><*(8QLGAvaUaftB$GL1(MA6RJ269;FVHk7|^Vj!%n$waZHmuL2iywT=m~sXTv1_*c1~Ay9%s zEr{&LiWcUJ(5yN2!^F$goXCF;d8t0a`vMMn>9@uzO*-~l7B9K?vwcRhpDYn)oypt> z2McmLQzO_s(*=M+-C%A=2(~UmUA|}?q(_u7HY@6O<=V-&mp3TcaBtJ28e;nScw;@w z);N4^9k>6NI+{(23RmE;`k;!X)!y*mmo-|Jwj;GQcQ4|?2CIy_199jK#C&-a z(o=fxxN$%SyZ3T9{XrIXCMa)7&9@j&vUdUbf|!)Ym=an3A}4E|kzek|-W0oFvba;v zc>lGn;n%&AKGS)aR*S`d|6AZSo?Itt8HT9Y-JhUE@*xfr9AbPL7k2vz?lMXJKs=r~ zX?Ud+1D!H>4hn=BBoxJtVqKr*w?ix#{EORU#EB+ZnP>71rgX)Mi`eQWBpeG-vz5%L zccvr`v?3oEBu<#@M5cW5WMep|MR^`q#9r4c54ARbc3Ymkn_C4gp=F^Y4ky%gOw2k} znQPVwEB@ATzxY|yI83ACOtG?UlA9)WIPt8M>pVIv(t_yZ8~8(%zeR=|sN1i5UAGij zGPD*>{~GuqYqCB9so;}hHDC60m#+_&L>J+zNDs@6mXBoYID-*OFXlqWSuaml>SOyZ z@Hrs6v025w(tr2a;wifiC@jrfv220Q$||iQg;(R%oT!mpCo$3A#{vqm&2v<&vLtRJ z{mfsq-*r8;jmwANcG8tku2ZVoWTF#^fK`izi6EX_gw)Hl|jb?LifW zO%D-geUf)ULt@$p5GTloB016rp;yt^w!46iG<6{lE>U5#uHey@*L(Y)vo1NsKuZc- zvc0DAlrAz2L7i^S(cNA%)8GJXF`}8L`@i+Bjm*tpc2<=9z}59T^QbtMI}Fe2RYKw- zEr(fz3@>e>&F!3sU@iwHjsD12?ooIgGADyN-XFx@KA2KgBg-X|d5Gna5SGKLh#L?I9+4^d~1)ACE?f)`M8Dr`${XF;I3i{5gSk%S&q#g>sY zy%BgWpS%n5+AfME2r)0$T0eZ#oh?Wo_ttKmE~ZHs6gVktO;BIp>9f5YS9i-CmQ5XL zKP$yE!Z)0CW;f3|f4(F7N6oG6X09s!R_Z)E8uL}grEh_}lqZV;6R#iQZ2uh(gf0px zrGd>+X8j+S)B`*K!YBk-iyn#r`o>bE3BnxNf^~s}F4eBSM>81)By|1Dqwsz@MCPWR zw~??JBgQ(-zn_OuTJ-Tfbv&Y)?!T;ge;q$I41r2r9CMQs%25QS2XC4Fz~{l8vjo=v z;Uk>}_spy5f511gXa>EowXEB)N}`J#7h=#occCO_?+Lcz2rSF>s_DxN8o2LNH|lr= z-sdQyavyy%_G!$2lqGh8{E}0|?|v-$j@!IEDv#Dbzk; zKa7^dPOLIB$)$)%)M%N+e@hV_R8{b-FM$nXU^-b!o=X0T4MCKs6$c0@kLwg>*{xf$tD|td?jGrf5zh5F-KhpijmEqF541AVVp~EwK^jw^9|5M~?~t%Vuv1iE4ot{h zJoqa@!NzG>J{_DD3{eG>p1zYkvX6q(*zS%0-bsI*z)4a@cdCxx^JFlNJB|d1z664} zAbPX=vQ%8rr?6+nkS-Ud>N}`pI`SG+*zOUC5200I!5wy}wCF--3AdEqrXvP4&$_IQ zr-PiZ^KEcU!kiC1Cs`DWq{Vv^$&L~@m8+Ev8O`vm5+QvxGG1~tB?b4Lh&7j!SlLYE zed$enwRX1E^yRBRxtM2Jvv(9z`3aHrycM@9iEInj=%EvsR?>TiXC>>xCI|mnVolkQ zS4CzfX10;r<12aX;`;fXJu`)T}^UI-zd^JQMJc zF6KsH?*K|1Nj1oFlS+1n;3=2#_xN31y_QDbrTur@_Mb<42G->7NLCo!E(uy3ySvVu zaeutvZJ6ipI4YztEX9FdFHZBcUP65ls&iy`@ zWLrJd@BO|aMIur9JxPHeZnpYow^eGNJe#y2`9yzBB>aVSH|V@TXW`Pt7GE`86S0)$ z_9f{o;g?K@hI%4mZ6%PzL-W(2bO!!I$sXAqBAo(9Gm06@E+NQZ2qZ%g7D@D$9TLB< z-czWhf2ds66HG;ufq%l)mb5tnBUAdxaWTo%pX4h<==AjpGz@Pbkq>4}&VNUCodrpD zRs0n`vPZ>bg}?XuO@}|9{w$jVy9$*wo+d`(d~=6Gwel=!EZM|Q_!;<|Jp&=BRzNyf z0ysBZr0jKH)YIW+h6eQL23@+Nqy-*-_~6?6InsCfTK>s7Xp6b9 zEyGPv!>aU(Q)igBrWW&ZzAxzqxwm}$`zMdXTqBD3j|HbPv5x!bfFROFj3H}HkTE<209`>GUDY07DT-MjmLAl8(4czkYoBK zus#J37%?Mx2v?v$d6jO>nrAgHbC0SAmHt`R=XsG`-2Zx7-Apz%L(uu2Q@^?JVHENTXJYEBlFkc< z7qbzXm%majwT>Bm5QLb%^u1@FeS7r=9Iq6G{fbzOSGORhZ2pIVzG5RPoYgGq)JKm@ z%njMYC4JaFF^(V&(SLnq$Oj0<4g-a$-QyS}H>2@u0B;uI3?#Zxq%Y>c3z?&Hx>|SY z0x}d@sr9bn@>iwv8d6BrA&KCN6yCSIV@R)RN`T2Lt?#`}fBmH`#G`}M<0NvL9PK`E zA6=y$WCrIyU(Y4c35;E?%f?>|(>osf*b=j-;{F72;S@0aH-tyD)3+~MJrA=#sTtW75H$ltzkp$U}*cV((vSVhUdMDgxSCTb}ktyn}T6E z(heOcICAypbqCJtc@JIWT+Z}EH`vMbAQPM(`*oGRT##+etUJ0!9@>!3`J}LejH_#1 zqhMXqq_B|x>XAkz(9dGDkLPjrkUHlDzd6r;Ur`^GfF^#3lR>k)ZSS6wUH;fR-R;nN zOd5#uFSqlp!ce5W;;I99U|pv2hSCHvVQ|_KU@w$jwgz4Ub%$=8Tms9-19ud*-P9Ib zuOE!v+X5pKT!1+a=RvpUhzEvmmi;BD*LHE?@=2gc#k-XlQSO-6j6K_6hU|U~%EHiv zHlIFwtOFhELd0k1k(}QR7sv#dF<=x^Bx48tZd z1ePmXNke7=p<36~ES36D^*0Q_C!s z-pS`F@~HxLbnq(@M!)e%s7m)KEvwu#2@_<6g`{?EmZxMx?C%F!AU9K|^!|t+>@(5@ zf9Lwdf3bnbOr`~CV1s`||ppv#AyR=r?~_L+t>3LtZzpcQ8c*4LSX zu9)fXr2CnSvf*A9n~(f-22$Cm70bmt(1kamsE|jr6rch9oFQ1 zhi1(g4q_o1CgEZb_1G(gvph$A&=M>qYZZ?qjOlKd;!_S{uEAw4Z0ReMU|1#lg+kLh zHM-Yf??gmCmACp2Y_=R$>rxHu$3^^Mji(vd;t(j-Q~s$)$N?Q|;_$Y+`uH`+FhecI z)?g!A;7HN86T<+(*8W{5K499@^U$DIB&~54n7;^6sH;d+0<@YMn4j~Yd&Tb{_vj%_ z1iV1vtO8te)-Dl%8qa0@o%AgD1f2ZAz*@kM?A|I;weN{uyKQ{`_}O6W?QNXq-Qc)+ zk>CGyDsIxFVwE4R#8x!oPsq-{{`~KUeRK_-f2`;SNyT_TQ}XxM4S(31l`cwrv8$2u zE94}^i^dgxxQ0Ua1GvL2NsWQ^=d1|;5px?ud_E$rW>}x~Zf^yk-3>G7(Zh1l!$Vql z*UlHVL{P)3K=&qX+tEh9ETULRf~aBtf%HaH5>|v(C?O=21A2v`FaF#VsLydw+SNAq zFkuk+Z-qd(d!Cvc&$BsTu})^Iswv--1J|dTkuzBN6D2`*Ea)Gtg_EYOdxTwmP2Evd zTs#7DBD1>^mhBloV&EoC29%DLsn*qL9Y5YWH{FynQY}OZ9o>^FM{GD%&=dFB8(FfJ z7zINd!oMjUF#aQf?6!7k>@E3RvY9b~Trv&zTl8wBoB$pvnOKRE7A#gg8saIWsvF-$ zeLbn>_+lzji*b}Kk-zVmI9~cRPl@z3BWtdode&MpmkVJbM7M^;L=iSgxAyVVWnp?s zv9zK0b;1&h6_fv}ru=38t2a?jOxw@74@fQt`dBE2Q6%71YPfdNSL0mm?wJ$qtG}Oz zND^61#{1X3VCTBL$FkE1BXNr73Nn0!+lwiG_O*hTsMKRKavls|L~>*W&5$QD7xw0- zMkiR^>T=#W1JXfplcT=L;ifzd_5gk@xv{B?*9#9`^~>lXi?T+rS!i3iUvS3ju2ke>S$^V~(wiZ30Ae)POS%W_hhR_lwOwEeZ2y{uC5>D2BNKf zwPT?5W&Qel|EryYsoIsqT3i%t&XZE<%(U`vL@*=K9p0^ zXhK!ltWUFU?^?*&LQM8(QZNV? zazs~vb{B&#UkR0Vz|zFF^TQo)##`*<)1!U#v#sw#1&+eCQ|kGxN)~0&jLubrce`tf zMv9&IWKDb8ho6wg>kaUT)v{|X7)yk4ktlC{MfaYr{i?NgrfUS7XdL%jJ#7j(qjgrE z?dIuAXEE2xs^VB?>zE`wRdU(Xr)<9pg>ed`u5wttk`t4m2%K3Z@F zburjomlGAR&_zCMo+|{TCAELi4g31hLrIFBlbL>BZU(GY4gq*$_B%I*#?B3-$cZc= z8xIFKR`M~LH5l7L7TM#?2dbpMZ$YUB=4`<`=Y9$e5<`E|k-vMahq{IST!nkLns@Ht zE>pI>%3|?2w#>k6{4MwSZBYa7@!y$3ze7CqW+FxaiktYhIMO{DAYN)gcjgVedU+|< zgQVh?+)t!_b+cq4^P;_aSf2@Q#Uc3;x?{+w%iM>oh#I||aIqQtNIe|&*qDRFKfk>e zeNXWqB?U-V1O$wi#~4rI{tVI>7^fssTO>C`V`%$z-Ze`gIsuE*DS9<`3Q;-Ws!-kCxQ7nyds0i}l~euNyHPkYf9$&QD&^+YIBB zCTBeKl|G8BN~3x_!lFq{X-d3dXf?~o{IPa0N%*A**%oCr{l*L2mUA)6S7&ouW(y5_ z95qG4mA#Kkysft&l42w$4-5*apGd9LXd#L^+;06Mr>7HfbP?}0yhfHBcTl(XY{c7( zTF-6TIKr$P59OczT28*r*4M&HEqIeL%TNkHxo=MV0fT{@@*eK+z%nhyU7{Kr-k@0b z{d^Q=zf}qfs4`|BIj_BGMDzy?#)vqCos&Qg6&~kWo(;#BXz14vKgOV?T)VX^LdnZz zX-N{6#2%hq{M{n_VxV2^FeX}LJ^ZAiv7PKCOt$^V;Ou$w!)4{$(#t`3ikE7>NlK~y zpALTE&nl&lw+~9fYL#CM-04^jlZ;{0%{p=%^@Q>ui2TWYpji;#6Y*%}4fMDaA{SQD zfho{v5B}@ni6oJu`v^?K}YFMhTh>N8_y zmJ0P~Vd|+5%(Vpq(X8a-J>n+}mlc!7%j|H~hW#VU&IEMPhgIQi7j>) zppNstMfwg*Jls{@kuidg>IiJXfVRzsQ!ox2N3@EYm-6$FxUkU>-lvI)lrak{`{ooJ zydTk_QkdoUMrgNoV%c}+O_+hj=iH8h6M`%Ya9b2fq9WQ9Iw}udGhSBYhE41}1Pc5a z8wg?}Sl0L6r>Mzf0-&I)vfGYzbjkdHy{#(D)`Iz9#POw8 zxaOPPG07pa3GZb8A&ZG7-KL7jxLV9G(AF`I_NC zNcolLwiVe4D`FFp`f3baBpJ}JbV`2mJqIIB^k~O1vX0uhn+AJy7mFXya5?)a8WA4W z+44x}lUv?foq=C4&OknzAT9DO}Q!O?g(=!Qr5w-3{oXu!OjKIGzx8IM<7doA~;T z^(TV>*}0yD`k#f~#0bC;QW6^ghhykDgY^Io{>~zN`1Os>?&LuHQ1E$UPSdQ)g~(QH z_g{hn^2fl;rvZBw5S|I?jm&YU(}|l8MD_*o=qmdgXsUD3#Y9nCC=tyU9;>iWgv)|` z`65dM^A9aid07~|-XO9Znx61udwmuRtq>x6M%6O`2v`K-sBMfG#qO{YLV$P=QkkB9D%njMuw6|bP-n0*r?$r@8n%f zM$by*SeRQ6f_1&g5Kww~n}NmCOQ4mqYjAF4MsCV>?je*9MpjM-D}8+=L(uZTHus)I z6y^|7Wbf9QN$~4JHYY&JEn*xl@f~eoom(8S8Z)g0_fgRF^Io`H@O?>fgOU$3O!(=v zz|e&-e3kEDLF3$)gU56`aafvtlXkIBj|sb4H4=7Izm=Ywt~S3i(D!6EqELg<*4yx1 zq;}yrT+5d7mu+Y;@67ax2mJbeLS<-=6Y0 zYdCO4NX411{(I)sXI0@V<*APvx<9cm&>j)$#Bj~5%0Bv%@l!lr@_rNm7vLAjy1Z9)zy9KI*Y6-Ms3mW^KTBeW8AV!UvYn) zCn(3uDxV3yHsl99OTFz{N-+rYI!&c?#YqjL5T_#8QxnhW&^y4`}@z6 zV9}o6@d!P8-*6e}HN~@KjpuXYqp<8O<=xUY{*RE*UJI zJAQ{>w>Q%ny1;KIV&*PadND;PVZICBb4}m&mw0uM56n5o%?eE#S&UaZ@yQM9P0%1J z?+8uDEy_V^f7$Ri+dk^v60ujnyh82hTvmTn|F0jbhyt~>`Xtr9A{>9n354@mTbH?V zfwvzYUu<7Wcv#JMTXTuS1g>-+j9rCCM3EQa?=K^HDg45^-DZWkn*UTwwKOOr@;0J_ zfJdIhl3&;6x%-%Zi>q8cq3}>HT{m}6oAkvNr{_EjM6>ZBd_4NA1w13=y3S#f5BWCFg;rrYXda7>V?d!UsuP#lY3uP?4XAiL{NF% zH%G-i*52HdiVf(w0jwDp()p>HJ&+9dVZ6Qp|xuM{Dm9`i}ZKT)#&{kyk9i0+ZU@PL~1UxKnaNz z>nF7{8xuKG9aC0hATQwk^41)>yJ7_&s~zfmC5h9_M&|f5pnhzHa27tBKlhuo*Q*mb z2zkQ2N(ySPr^ox0syD@oum9u(|IGrprnLW)A~co+8BpDID|zCp zqTJ5myxh|Kl_tzpBQECYNxdI-m7S`H+$!Y9;j-?q+m+#V{#N@yhHYMAJh_EJ55epOGX!y$OzO2T!A!e)x)=|!J|^5jB9&Q91UBGPpML~ zcpqJFryJyPIMHbNv(~6)Q+F-hWU;M=v9rWdtsPoO>Y^or2}fwT%gwWD3?K-t>wdo# z$g6zj^=YcMs1i`n?tRACNf#W(nYC-(4^pvLENwTF|4zQMJ(qtn(cb-EwU8Os{&NJ_ z`WKD=9SksrZ-8w`mEfx+2nd`>T_W*bp_Hx)jflg+Zs18K|KxOc^6&Zv6ulgQxuGNT zb(MdTZ*FX<5&BL>&IN~EY^}PCQv^lmQQuT+J_nxGXZyW}D(Mb99Ym_yhsh87XV+Pd zv=5gM6)XX~VQQ%hK;&k<@&=H@Wm2>EFcXjyWksb2RnE2_sNdCVnOu_4WfS5GAu);x z90eCs0~#&vSt4bo1=?k+Xlc2v5ThN%GW6a%T8uMQ4<~`Px$M#$i-*dAUAlRVBt%H= z#Jfj*i|{Rw@ICMG`6^OTTOb~+e{2R6jo9TfrTX%&YSNIV`8-HYrKo0!Xoe)3)v8qjs^?g?IXo^dSOqY-LH+h z-JO|7d(qE)RVNm%xC`5%ZXH00-y-XTwX^-m+}{~QzOm56TWFXVnB>p2Tq|5>ELh?+ z)T!{&`JK%=pBRhmkKPJB$5mwv8#9%29O&2{EJT+Y=ri73=21!DQ<$8U=C9)17WD`( zXY)P#Q~7l7@}V3#R^e`c7?%hCEo%eYsv?n|{3kjE5-Y@Me*Nq(rgf?o;R_6AYe&B) zpL)3reH{oAIw6q4t$#{jKwGu17!CJ>cS{HX7%V+7A82XQ(mLerhC>87h(utwK zQQyMZ{?x`}#wz(;G=<>T#OhZ-wmT!ZT%oycl-0xY@T4mHyR!u+{@-8x7$`VVvwoS| z9$m__KFj3Hqn@Z?9%s+e1j_B!z+|LBFAm!UIgc}^LVrfY$7a$;#M=L9oK(IJy56g9 zhoe3PlDvbL&#!{9i3i}dISBVbKlYvN`IgsWNsnZ-S3R^oY)0JQL@U*9yLyAigQkMI zECviakC6B_N|26sd0YXuP&1P*+@!N!DM`8c^UHH&Jkpu8+Ibus(Ii~=edJ3BTyiTt zZ&DC`Hi?rp(e(1W?f?tK_KNVp`FktfhjqNf7$ryZwDU1a)^%(yddv4A4 z@>*Yqldj)CQo1lHw%@b6uqVE}bUde#7oTE_U|P$EdnH2RO(rW55KeOfdvhJ72XXRZ z#Uluhx8gqy2NGIaGZ2F;1}7iAI8IosBE&B%R}EY=xL?}(?1(;QM8VF-YX3n}2^}A! zrz1TPmwSaHJDV!Ci>@(G@V4)b3aB7TboF9x?m>}+0E`!j7V0~CKhY+Oz0kqAFOTao^V(Y7<_G?pFNTNUp_4mUZ%^}-qahkLNW#4-E@;k6yCHpd=_mIi*x3zvTt z#mMB&7p#0iDeOxS)5ihkkh@^z9`@&#LlJ_qG)Q_;5DkJBNfAm1B9hth=GZ~>C%hlp z|3F{*I|(Et7%*w&Rox>q3fr#}>)tJIg-mG)mdO!UgE4$q#`y&o3e+)ONHY&0j~lwe$&!Y7nB zM~zmK>M%K3h>!6(`DP3gGx{Ljkv?tQ?0JKpVPzam^tT6IdJ1OhtAqVe>(J!(xx%1K zeyBU|r@+902k0LPo5|nX+=`$IV#FYxRy6FDV572hE<+)`U_>0`7`K;Mj-{)m*A5oy zD`D!|?lg(ez{I&DD!*Mp1^?ht{JgB4424jhP0&6^dwKG4K-Mvg7{V%St`>=ua*hKmQ$bEMe^% zrR}wD4Sl$cLmFHO7AP$Za8%-gG&DZ2xJQ+m4yEuhRK%QAUY}RqxXD`khqm?p$CUE# zJ@t1bE1q_X=MOpXkKTW1sW3#Vx4VsFa`rR@5M6};=|Oqqk9~!sG;%JoiGj{v7z$TO zjn9Ki!2hf{md*?cWU%x|9QmdY_s$hfhImW&yk{PY{UB}?!F8hCzD`>`Vn`1jAbt@< z@JY?_0Syt=hT2HAnZlz_hn}^OR5aT*uQctF|7cbSTc;+=il3KXne!OeF!4C4iEC_s zPS8BZ7F_?x#tdk)8wXd zuu)04tj#IwHnA^l5>Y|8{pIap@C2K@L$=N{_@%2+Y)a%%D;8DVwyXH zM4F8xDqI+G!s%LOfTDAean}c#QVaZny>Atk=4fRai1uA24*~)& zG(_XVKxf1VBkJY-H7cdH##EXV#Eh4+T>q<_iu~Is_0_Zvx#5xt8Y3RJM_K(uIR+2; z=yN#=S&y(vHb*~ydoS;y*@C=D9SCdjr1ikKKv6dz=VRVUVN>M?G>9+BPmI*B`zNw- zFTTHm*D%%A`zT%R8ABh+BPPB$7Equ(h%}nb;`rHdy`;^B8NT-66BS%*F+9@GHD*a^ zcB^`r%oAc6Bx>ZevoRAdPoGu)D~P9lfM@sd^*si2I;M?|gr%&1?u2L=sl~dJjAXJ3 z+ViV$9UP z1?2w`j*-92_QgVqBO>Ov%eV~cCu>njON|dut2!=PI&hF62}9vh-`r|48 zZJpVy)UTMcgGu%L9aVUff~_QwA1}YCx}{rYxti0^9I_Zd6>dL09IR(m<;$gviO@VM z?#Xi1u#1&Ep>xu$;XZltRA|{M%aO_F9L;TLjK~76+z_Gd-Sgq0hJ?YuK8Js8-qk11 zX4toGW5r7y!);NCRn2(a%^?T~l=6z86*A-TPgy+(2hn&&C?0&P5BIISysN1?_iBD`j_3j* ztPC&o&BZ$lFO3OT3wJfjf(yAzSfwqFlS+7q*S;M2W#ZeX%OnT)ggE@&Ge0LokI-pb zHX8vR;Jh=XCwGBZZRmS4h3xTlkOEm(*41(0K1h`FQG(5&OUSq@0J&f-USFMdJ2uRd z>Qow2>Go+DA+5`HEWa>s{T)C2rWDj zl2*34k_p5O>)EoeW~$denW<%-b6-s2pY+Nc4Nrr8Wjo)<)L_LXY-|Dpk7kXx)tG!X ze%?kII@onMNEFG6sM0(BGGZ$G)qr?f6!m<5L6j#$v*p!oy>qmJ2J_c<@BRg;$xx&9 zT~UZEuzx_XUyzA{L4%=!E5VMZ@K{d)lW}JQQ_x*qU@JQWHvT30xhaEaeSWJ6Ud8y6 z_ZX6sUyMB>9S+!hVAWGnG0jb1hQ8u|O2$wkS16I3Cc&%NFJ`Q$RkwX#kn)%{j~ngL z$&>b4rYzT;wDGjxKkWoxfxUX;5L`4p3pJXsyfE>{lOV?SB57u__x?^xz_*NCH#A9Atgj3Vq{M5o>4W`qnFR7fh8Dcg$hVl zH|A}mk>JJ&hE2>I444reqdz_(pS2)#m=1BznqPlZxA_P>#V{3*g7~eA-kD9Hz`m=Q z`#moZ)-CC4r5Rc&;TKe>cc_jtLB7fAA$V6LY73dJ=Q2&}Le<*W{3F;DL%tdxq1yfj zy_lj08jFYZlqM1lLTAp`}K@)PLT9cG%{{g#ohTWR$I$XRp9%zLO<94Fnr#p zqvYjojRKuSn)$R-;lD)#X z*JrWU_qm%lb9n*vyRr8a3BDQ=UrRlmRmhEtpmGgFVgU91I$9pCE|1yh$|ckq?d^Yf z-gYBTQAr0pQDE2T$A4B*7xBDMO~7V}iyoJhKih1yODgya8roghiWK|jNT*rFEhJZ%>GF4=_ozOnCC zf0g5eq_tr3DfT6%3ZCYtJ+c^)l6T&NXvu@-{L2$T$!Lb%ZwQE}F`At)UT40N^n7*T=TY#r=;8eEWcJXm&<%ja~ ztnXGF=@SLNgYQoW+u54fnCw<(L0vcvhIwe9!?ULHxl|H0>WI~TwSL#rtZibZ+B|Q& z=9gM%qC^r0v5pHNt|#J6NuDROnIv=1iZ!W-etj%lr{}%SuIoYN^}jy#Urc>r1_zsM zsooj1Qr2rMuYCq3Qq+i9_%2xYTw%ElGHrq6So=Oh^vrb#zn{MQIe2URv<%dm z!?Xzy#8PS969SKWS#C6n(=_Ng4B`4%F5)B3pPzqr~ze$ z8z|j2sCU4jSnzu+cCquhCTbQS6x(}Uj=nh6%vXkjecX&ul)^Q_qVqw-8#>|k?2abz z1CiwrMNbuHUxADkJx-syzhvxO#I0bfrHj+Q5&52z@s$YSPGhU-e`x90?${{vI|<(y z@N&yJ43J(o#DwIfrfF~Hp6{NIOK&OC;wY$)fm}q;&yt0EeaLf71lBEo+MgNm6F^cZ zkqANC_ysV&Q@~nvf?c3Uq4u@>Q^Ee97kJX9zy`1cjByu`X$Qh3~>EuW(d0 z-pF6bn!!*2mmh0atYun+d)u9jr}HeEUL`@b4Y!iczXxd`D9L(dsSZElzjKbd{2@db zxjs?gr~}o_VpVHe4tnTfewXav(`8p?$~U$3vc!wGY6Df88oKeEq!|6H9zh9`JZNsaG4rql{EU$0ywAlhuijl^?a`p5 z)sMcxC8PTDpDKhE^=3JYjsWl_~aU>5XL|7w-0Y3XuU;jcK2)+oY+t`?SjLZyVTVUH{OL5@dQGlapvNd?`cm4|gB7@)$H>fhpet$QG|J!rl znX29xR|xN|e*Cai6SRa$`2t_Wew^BQ60Y$fFzdRQoFVY8AN=vHD#&Ja(%tfs%5osk zPw3IQWS}N8Tw$3dqnJm3k9S~K6~m>7=DEuvh$`UZCz-n0*GM=+eKJ{pr&i@M9+Su6 zh^^YkY|3Eik>K0|V{Z?mg_5ZiwDVKEi$4ZS_5ngmOkdMllHBud35dk*^u75=pd4hm zt*P2R+_A%!r+W1=;5-9rrcAGPWvWOC|NYr=>)%EvLnb?Qe>*`_b*3~e3ONEl`)(}} zU;eAf-w?iy+ReqJ5x%UK4@PfUDj2Ra>=jMst_USQNZDINYb za)I#wx+VX9W*)nvhE*y6PVp-QZEG{I?XI*fb>9``#HEp#>nhFki)*|F_bV4R?fs{g z31pf1I;k}IO&SSteJqkr^7W|{vQ6vi>aiB<2;MCDqY#ab>GwR6CEa`{tH}_inMd-` zzC%CRYo;g}&z0G0khZvfzLON69{H=O*HES`i{+lA{3fM{(gcHZMUyjA4(fz&)^=$% zqm+CzY@`m~^2){3AmN0t2wlG4bEQMAPU}W&+RL(3kO?xPpW1c#ek%smWHXw~>mr!T zY{Mm?Tvy}wk9>0~psK>^p%0gn{SgBaC&jSVasyQT%MzV93^zQ$5m^i-GxjozJ?|MP zDQ5~ceRKDBvT42C;2QI&FP2t{_q?yjX6DUNB28raD?3n2gf;uosxoDd>C<{#tZ@{q zi`4@;VhlX;+F#OWPhbyi@hX_@H`C z4{;yy36_Xa-aNdjd$n)!p4!h<-pgz3a!DD0GT{2$MH`&|SWRF|ykP0?88eIbRJ3f2 z_iwFoK?&^+8&Ru)gnh6obOeb$z8hAK3@-fv*q`y!jvb$&!?nR~-H!9A{cf5oUdSJ} zEF2^v6UncL&jTMw(+&YQz97Gr(*|L=aCBnbp;a17(FUKkQG(NzkT zlgojzLMid2wl!rGvIKU=0Ck_Oas2v0{*lW#{E4k!WRv8_X$+bgE9yANI3{eO$#&JO zyOnwULxCR~16!=H`gacVisuVX{WZs$u;ajB$AVxyKJBzVX1co)`vq4-k?V3}CNWc( zRQ3zRyU)MM)3dm!<4#8yj3IcIX-y7FvU&=92Q)?3EaI8h291xZA{xy(meS32IyADN zh4^RcG@CTAk~F9pn)FH9!Ua+cUsGFP`|6t^5N(F;J5ttet&lHNTMNr zT3*woyg-|=P7LNVh2&>3Xr;{6_tp;k+KFJ~@2j%za$Ajou>BimO%^B;QXYWMVC)y$)fB72!nLP_47+USU^N4rYit1LnIcXD;$6 zHfp|JYuJlxsN0g7s4A|-MJw5vxvDuy2)_!Ypctj!s!M^;Jx#mO+7A1R=6&d3WLI;8 z{5zGjhF~ zsv{jfbGf*vkmbAQ3LebwH=#0E-vreJ8L~TkcbROqUV6=CaMNbnYp5`^(%X7u#5C#< znqRJ&Tv-8b3urrHIP9qayL1VljPHWN2Aik-%=~Qc`f9JrZDY7It*Fsu08a~f1`?t8 zsh#LFg7xdQy}mcC>cK(gba1fshc08M}CDh&~NcNaNdFU86~@(QEN zkneRFLr^-T40a=^*7f-?i)@EUwDlR+&-cIRgnL`c`d$0OhSC32Y@J_lFnkO~4`<`M z9fwXxudQF8|LC!nzGGs2NPX=|ZpoycG=+Y!jjf`c!dYa_OFJFSFao#Rd_4(Rap(_P zFMNhCGg|ign?@4E`X7~@z|(zTRmg4Pu%;2WA5Hcd<$kLi>(SNY>X&%J9597&k#QRF z6f}AP#W!DJJ+Lk{jl|=psteR!lQEP_=roM8R%B-`Qg-pwN{325gIW7*>Jd8b)~eAw zZb{0UoD>!p1C+MmUQ*{IvrLdfNKKsQSwZYhBKJNWN4-I7)H@U358?CgEjy(tAYCC> zdfX>sIX(_Jm$E`@YY!_@=i$Pw3ycJ#d+8A>MWq_MvkeVjf~4nOa`i--tgM#i#Y%`Y zZIoH*jhB;|up8%cTcr09I-{owqGkq6wf-EbdyP&RNE#6Es*{(g2|M16p^w4IV#k!t zwjfoSw(z#Rv?_J0tfKaJVqqB*jKcNa!oJz`7x=U5v(#pexf!)DhrEXTPC|VgS%;m~Q{<^sTjj)+qIn+guf9;WMFkSE1BzzJaLzvotYpS7sjW zsFqbs0nQ@h&8RQ;ExFGSjx9Uq^`2wj+#Js3-D;NR%82({faliA@t)@l+s zb*iJz_)QP!_2+(`b^|`0M3+3Z&t@il6mTcDZ6VqhGCLTba#E75b zloL4@^}U4aAmTa>(Gj)hl$`Gka)O^hn?L0V_DF~VO8f3sVOBjbqca~JSX}gk6UWp1 z;%V8bS>4q_<{abl>*ibsxfCB8>JZES0?@%)&@BpiA2|Ye^M@mS`|_!tI^*VTc=vTd zWW&fhmJn)80K>2U*H2YvSo>XEnS`Q=griOh299;whZLR`;%Pi1n@sYZgf{h91bt|1 zFWJfyKiEArR444U2s>483rqH5C3GWT(V}2WigZ(tdjFNe0XnA#bDq-qbJ}CNP7sZ=RNE2y4Nk=a-0tg+2ypy(JQ5g7KgB( zUk4G4qKZH15pir1?xX3ZC>;(tu@7sO8+iwoWoek|)I-Q-_$m@U5~n@BJLBC{+skP1 zSkRM>;>Gv_zcV4)gLUumg;oEO>m(i2JnT&J6~7G-Dys$FFYF!Ae|9@P)WmuHo7yT`W0wOJ;bc=L%NrQlZbeA+pjl?jB zbi)ABT}lt#A>APoLrOOc%}~$H_xFF|dR}o|zzgQ)KKq=r_S&CSgYIzF(>L4H`c(V3 z%COX7x?aV+e>-{FPC5H1+MR1UK(chHc07&p{nO+w%Ty-4{VT$_kJ2yN2-GQ3XZ%3H zlLq9UInM)-a>6;XGe{3{-GS8}Hk>1^S+nn(repNa<4Ng&4hg%nwc0iJ$-kiM@T__7 zwA-JNLfj4I4V#s?ljrh6RVj9>v*9r=A0yRSE8NBW!dk3k+A#-=1(gr!+47zp+fw*z zhkuipwJ*S04ji7wZ9vraKS)Ey8JIsFM8Wp7_(+qp0!t?HvOFtk)M!Q4)4`K^*vGPc3%|IORE!IaEqjDZ>HEd@Q|r zS>t9ZTmK|71==2R9`bPBHP%Rh0=(*9(8wRZ0PzV6_EO$CFRZRr6A-r4xCttz$GHFR z&w*d_e@@#WR)~{16GLI}I&DG{CO-VQzTcs`h~n@Q_ZV(CGC73{QV{5b&c{V2t6_2~ zrTUw|^kr`&8MGBnc{07p>FBQubaV3G483nNZ|ctTWQf`rCS@=lVe`n&GEXo!9;_m{ z-exMkn6zEDhaiYESk5r2h;Cku#ZfUe{#Hb2Z7XCQha%+5D|>=(g}b>^)VOV85F}J5 zfz@YuDjbj`@XlMxjDOGUjgirr>mWU9N|HsiZ(9+@IpQ?3g2eHAPgXRp;xGX_gpuHs4lHUanF3J zny5ELqa{xPC`x7OC9?4XSUBSv_#LlfEn@5Oi=N2KjAFG+36Tcm(NZQdC`T);Is~of zfAr(0*N7tJqD`?nTUM(%Xrlm0AjkZw)>(wvkz3Li^Cn9USR@eJcJZBP`fDY-kxa39 zr?%_Qbg*|cXBelWX{Uu0uEW@7Bqd|rsSAd?eH)Llt{}KAlMgT(1ife0v<^fo{rJJ1 z%)I3MWuI_CPAAO3HppFoRevdCg73hTT>-~TpGCy7|2wKS)7?ErGk77e#u!Q7)YCid z*YRVzc?Sn;($bYDsK^_D_r%7OU((MGH-C&zvi~Ue%u_ z-Tkf5Tt3mqM=|OsT=ue$I}g0F%dOeX1Onb*?XV}f4(a6n1woRv-7J)LyPY&mUz}%> zn~k-qA9Ipx0Liv%_Fx-O!O|81ga1J5XF&Uj4=fRK+oX5&11M6!9`5u=p%t^;Y8xhm z73aXY;a-GJmxqh-ixZjNyUacS^?n~1d*_2zJa#o1G4yO)! zq?^>rV&#tK(P&iBK{Zt>wq1AMOPw~$I>j;M$UlCL>vN~@y@gp}Hm_Gk=0WbKdJoN+74UpzEu(ibo|RAYmg;LlU13(g&H_Fg ztTY){Wn5d~Br&Ece!Z@tys@)75i`4ZDP}N7Y6m#Ity8}-gs^VAj;O>tz{PV2S1)wCs z9x!fS6S_B}v&^Qcdo-UOJ=oy8!aa{O%d?(Mw*@hh^-haW zipeJT9diTc)>_1Px{)1#lwDdk6<{TR0oqQ&KD73X&k)43J7RCM;%s67+FRif0T$LG zz9+lr$`v)e*^paf>t@!%mFpWgG>#aFcZ-2$) z=G?WfKR1~j z&LA(Sq_&$%NI@DSTHD7zUcqP4*2qB9&!JR2X?Q#cDy!a{5t@$U@9mz6xN`p?b}ThO zjzae@8Cd$WAu6%Q{|a{cL(g+?7-rE8EC?vW&GOU)!#L`a_y>~8S5MvIB_&;WqVRdT zA`lipnS)pz%+E;Fz8`zzZbCeCRt2oj8d0>}2c6svvz7lnknuCT4zrbDV}VOo2lsdm zCdt&2=m4l7b0@50j@gx)ln$~!f-^zKfHXf$1MU3Ogrd>c?%a;YXU)!$ER@*wq{!zhU2;!zx?Vi@%y%f6=8G z^oN69DoQfwH+(VM4eW_F-|ruodwJ1*aRI*R8d(1Oao&6TIcVouh+fjtO~Td-&*g{X zpx3z1po76UVmav_U%`8$Ti>ni08W1MuHBNP^afRvYA3PjdJ?8vo;7uUBN8B_tqP~6 z9k@D!R;h+a!5$@2paCnLH~zU?VqeBQT>c({x7V_DlH)ysYknjs>#7G5*O-VHK5b>G z$jdJ}!(Tfy#uY)-C|57ZWZ{eGB@qQg_DlHz*)H6aFTO6FCSWfLuXHbv8$N9^j^ntq zSsPDp68~#LQ~SwZfTJRGY(AA9^)?o}r;`pA&s>x&X*%dCla>!$80acr;)k{i+>7Z2 zV@>=n((a)C+&e?=mqowe)%@W+1A(jg=6``kM%Hyg=5`;p`x36H$h zLRIY#33SnCxfeC(3DK zWbF8SKeL*Nj6=x52ue4ei}$e zz-C*j8D|Js*UWQI6nR#M`Ut~J_(RoPKDk6f&ypN=)eS$Hn;6PJ7wzBtkO9LxCBQ21 z3SwRpI%$b~%7VyA@}!p@FextLvC66@b}$uK_>zXLd*nC|9)I%-r$4Z!&X(;iKkA%- z2=NnFDEwL@Jqfw$W^9e@c#o+WNfegk9hnBV5a}cdpI?a|eb+@&VJlpGLioCh%r{Vd z_*qQ3`g&)@9_{9-YlN&RyvFtdZ~(GY$0Q++fRI(aMXdiWg`cFe9cyu%0rlkZlA!C2 zaokm^pokTA`t9krQ#_F`u=pIQbiyL)W1iR<0*Q88 z;QorCFj*sE4r}Xlu+6lXFVZ~?0L>KJ+few;7^aESP0^*^E+~V5;D>pj%Vq>@PeY-9 z{615_!Lh<0wgc~HM^J+5Ktn+?xdZ?++xne$q}Og+{C?e@YV~z>W2nAJ!WF}uYL;EQmj616`MB3(jUfv?Y){Rn<^E+be}xlSuxv!N@0c0P z^3&(R+5{_H#XH|^!^^u|QzFMCaGDFhV1Hx2^Z$B(dp*F87qcW$xGL2F@I;|edvk6D zpR(8~{?tu+1pZMcVxlDfMijU<8Y%!w|8&E z!42mJ^8(jgX;d{{;}vw;hK}H*6^obtjO41oeFVN}UK8-u>*1Y1}_d!6kM;Bidtg1!gDe@ag2}G5JMFeSx`#c=SRl$W@LbtcO^U zR<#*)un!0-AC?#&T916kW*6hTGXNZT!6oE8^DjCt5n97nAgrV>>gMV-&*xd--tPd! zVJO}Kr9;3tyJLuYmL`H?g1vd4TLh56<9%WPzQt67^%etW76fUl$z>b1hwtayNaE)D z_$Flw0c)O&;cTjqXMnRL`0}Ph?Y){KnuJ@dpr#Jw!1EIcgur^&UfTECif2G*Kj9M@ zUtoJ1f4u1##IwZ7i1WRPLvVZBo*`A*o`tk@IaZm0Q3b=*UWumfm3=c#iys0nmZQLu z`C9SnBSK8a_cP7h3lkOyW^iWGU-{@$Vc?j+g$l!cl@!s~`uG370NaRTe0g}?mv;5g zxLFM<|3z0hg-88E9>%X^Ik7%P zY=ZGNCGF|W2{+v}5ZqWpsM^}y=C-l{I#7w9q%4!|P9lmMAg2YzzoWhCz>EyR%}alPXl;8p&pKYZ^ZW7!~x zE3Xu~?3=V;@vkMHpS0umJ$5VB3o33$KSWyjDqdPr?rEB=r!V8%KrpUwbVr;=3@w1Q zZli*+;^1;RHsZVFHLyl4k6|s-Jv$526tTXFn|J9dO_X_uKcM77K;-`mPlR_7kV@+A zcd6zPOb;si{HL|*(~eO@@z+4w(Qv8nm{C29L;PXNsrhE7ROQ(cx5O?-dsXMtro{7Z zp~-m$Q`Wh~Rv)je0@?4hUCGvr?F8vV5{$X>nT_=a_jg@T?q;f4=#MT+tf)+%vsKH9-w--Z?^>=;`M?2EGB~ls{36@X_H)i_(D~ zWdw<|T)dB8knvY0yb1{z^62Y&V$^PEZ(432&DX;m?NKDbDAd{n*}Mz8mVvFnC$g%ABd#EexbwI-aZ#^h#@jy3I*gq0YjfeH)+} znnMGnt40Cat?lsHp<5Bfmtm9|@XCo?6iO`+J>^}@SG=3`GqVL=)E=Q>D85`Mmt2DL1 zMhZnG4;M+6^(q6b!g=v;D@cSTy_a%4-e)y!)8CUF`A!`|_V88W2Zr9XjQ_;2DSF3f z`%)roFF^Wq+bg9!O^$CgNCcIQ6va1*Q~i>JZPrj=G@{X31g!yh{}4@!j81Gt3U@U;G(dA7 z&lQ@SKVBqAXE~wPyjnzx%T@PQJp^_}qlWo@m|`qZNkJkAUo?Ws9rxVdmNj0uDjdZ1 zhjiV|G^VnEGKP7zOtE|w!=lu-L~fhe{6Ili8ol>V(Vn8{?9Xcp)~TZ{V*d_ofdy0G zo$xs^*EUy^xWBS^6rkQYGy@RHsHZ@xUN(@I=`nEBP4vOo<+09FQ?Fp^LSFwT!;dH6 zuWV*MMgQjZ*s1u#-uG-P=xHAc?F2?24d{KWti9kk4Oli(I#u5hj;M)(*3EG{=I1CMF=bFt16bdl|gCi`L#{ znIZ{}SrqfT#z=510bVG z4@}kVbOCv|dXWl-h|EG-R20^X(WM^^3X7pd(`biyDo>-+uj(JM(_|s3Afpo3O52F* zNqukZib|iKT8G+&P7)ksXs=%8&h5SNeE-c*TtT&Jm8$Cw9hGL!`oAOYZp=5-&)5I7 z2FhUxx$Se2|BfmlrEIHVR=gYBxU#SfK+mduq@!N|=`yqORKAu#au{v=(>NPxZsNL3 z*Fok19NjdhreDB4AEL*t4NM-#XH<*6E@7^4*Asvi9FqZ3OT4Sar&TF$KJ$7M9Li#V ze2G;2O*tbEw8o)(mVdQ$Y5|-$Fwsc=S)Gwf*&6;omEjB8o8WKY&fE!@pY9Y@J^q*8 zR;)J0uy^MwP+An5X0hLH*^<}YDV~`PU`2(xfm5iZ=K9b(ufn&A;Ta{X`K+L%={sFf z6W_rVM60x)CXE24Nxb;+-x_vUI(+c!!yd;)HU65zm~;M~Mt4QGfV&344}z}U+&+8R z_im0W15MjaKkdI~5_CcQdQA=v{qFdfYB%kSIm&GBk){WY&v$bPJ&g$tT}~5{QjRW> zP$}v*%Ij!8#zW}HveX^hr+Fx>w?GiJ3>~GdnQJ3UxzE=akRkskyQk8j-a2akcMsrV9pqZN$%I{cT0Pp8-Yfn{p6q4t@%W?{~mh0My6SHcQjkOz<3m@>yB<@p@>=*T;-3ei(Pu}k+){8+8ykZU!c#xdLPU}b;p zC%Q|4x!e){d57xAukj;wgXGQD#%=s%n~r;e=4nNqI{$G-iQo-U!=Su6&0`?+@YeN9wOWCV|;5zFwN*U&KW)s2Em}$ zX)2ybHlOXbhw2K4W=_CSQ8#oCO3k=08X9_;!)xA+VKGzL$8^hW+rClf;{eQY!0p7^ z705+VzbBq0Y!jjKV;+KUC&mt1P9b)GBwhgbDt(y>`*Owafj|_JYe#wC3oJfXZ1(9| z0_0NW+|2#W)sWr;aC=KjwD46Gw@+tl|2;5m9~E-|F#D?8?>COj1ZrF>cWIFef&;2T z2OQISIQG2OtHaz__AfSOMD*7ZXu!Xfo7b%<5=GuJsAA11*P=5{2y(D{p`D;0m1(yF zsZa%94d*_IXQ1&Jpj#oJ@b8tUhBS*vqyw5XAI(RDs>EJmhEE$0gRUwd?Vrz>I9Jy3SdV@v51~b2DXW2Up<9DKtrcJ`bj6?bq|mdD0vsOI zL0Rq}9%PCr-dAI-)3Y!u2|K;L%~L=Z{6<&VfN}hlt<4$Y#bx|VID5qe=adkBhZ5n# zb5}M^jxB^0vqXbe@rGzeGwQ1g+~*nCBF#dtbfj3Fvf5oE_l&0=sPE6V>{rVfpdtyqc1IO{Ts zFYeSdDSy)k`D{Mwj?0;W|L4XMhH)&R3b3A|A<)$=-mPrQEYbGRF?5emRj@M@U*w>{c|NACSys4?TATTGM~FiRdaEE^j7$kQ zo$z3Gl+>M|ZI*KH8P#3b{lAxfnIVl&$EHZia}y>)c*NN(^s89NsK%JnU;pC1dMovE z@h1s=cB_EBipa%J!u__8%=&p}!5Jhj4*!f?)w2{#ZLQw!Fu&~x>`9Ky0Pz7@xtFNY zU%vA)_ENVUnp3Vzt`^{McnkCCp?NY6`r?d@R?ot{Z2&;|<#fQj!|_Td2q4x@v0N7$ z6AD&8HVb^va=)AuI!2)WE^Za8vb8^g11hbgro5zpEdqB%S=0La(iKX*IV3%B_H}G^ zM^t;db~=g>X}X0xM&GxlbnM=aF`*x6JT=SStq0k@+SM}>Z-rP3?iWGG@Vjk!iQ<*} zUE-4YhRH&;u@9+u$O~2SbT$F6Nv9SDD~4~00vPEogB)clqHng|Zds0G+?(uiKCsy@ zw>gdjdq=b7W%!Uud{m~muQvhd5Y98;V`N4N?g{!_1mW?i=N^ENTBHx&-$yRnt8*w! zlsbSA@>B}X{vGoY!6RhZ$ieZ``l_NVF9yMxjdEE1hx!0qG6K+DF*xELCB($aNZ2He zb2O8uLX4NRvuHx@aslU@5T@wm$2fVB`>VD1Lz#-FxRH9*C*{qDc7U~_ zK@3P%4Es|N4YRm+KtIyar<*2xVc)DF&Qj z?_M>xNZ7d|ysiC6=iqA6$?VV4O*hW%fck4^!xiDTQ(H|p2P*+HEY(#`VR__B>|OfO zr2@`r10rRJT5Oa195lzj{qYsWG=F{q;89G!de7`-JoNfh=q+_ej+bah{u_;@rAhp8 zT&vke`gRL8iuxZ;RBfb1wQ`uH0K zht2zNBN*_f3(@8H)&AA4#-R|VcdjzuVj-wYi0qNphQjJ*&9Q(C@3uPDk>f^t$!`L& zae|isW@V=1?=d^*+OIJ=ghRt@t#}tje{Fa1#zYEClM@dz zqeX;g#GGJCZqqiu2%5@4q)FUHdFZu!pAgYjQc)zt%A|2>R#MvItzqNl^AGMjF?c&W zk=;YtvWiCghV$!6?C!;RK?dTJ*U);*NfAy4Mo}WNp(h~Nyr9v|# zr4tJbA$V$$mdGneeVk|CqFQ%Sxi0k5?dIVum&hia|5{bSM1Mf1N@iJhI!$_+`TS-M zC{}jz)b~CEU;F>Rg=@vWyN$lRdKVbah41k{F2w_k?XTfx24}S{ejErEaVE<0hN^~` zS?<+w>RJ{#6)q(^rsoO+oJun0+BozCzterE`w-0S8NNXD zfWmi6MdR?WG^-^U=hN{%hX*l%>XJUSa)t5mYi2*^c8Y_uYWBJdH>Y|UGT&6R=_)K# z_rDKfqy&}UaqEA-s5_rFnD=|QSqmP+4`H&Ip z`2C;K(j87;&@Pw#{GO5=d{5+D7jLT{(>XXJ{Ow)i>?|Gp^4Iv@po>5-T4}cKKF0G# z`-(<$#vmEKyX1Z+LoN!R1!mjCy|;#BH%F6Q<&^c>2Z9vULA4L&Q0yJ&x}MMWjLrTu z7zU52#Gc_btRaA^h2M+$SyiYfA{t{)J2qddTRyG4&+cxitsDQN#;fmSA}cZi@E4qB zs9gAQ=_81 z8{ZUu0|Rgo`v8ut>QRjSer%zf#_tG)9L6`29L~b>RMG_jE5f^lJha>_qaPjhYFXlS zK-N1`gCfLqXp>gaxFku^RrIh~YeT{rgK~f=ztVoXD-SAD}Wr ztpoct-s58Oj{BI6f;H6O@epycGk@z0m0&Bk?mL&)Wa9y7bc0-odf|3y8gJ=Yw!1$+ z2G!=!a09^%x&>?-YKV8&5j>~k>>oA)NX)|N;T9S6ge|8a8Xi^AKjk{9C8c`$8Y28J z;af9&@G{umv&EpDN<-8ip6+sA|Dnl62JkxTDbAeDNM~xe=nC zZ+l55EeU*LSKA3y;GJS)cUeYIHD%ahr*>HubhMJh82&Gg9BvzbZ%3kvsFlf@CVySZ z!p$uGCT;qiL@hDYU~aFLIQ&WbmWhueGS~M*3vk4D=nlnCsol!>nu$%yNnIk4FDXiq zKN}I;f6NsvsVq+&Ewa%(nV$70k!~w(qSz&1HeJeD?qhRtv}DQqY{grn&bbvis&I;s z8iu=scxs?@MfCQxyraYFlNyyt0 zv~*_lZ`co&B$xu+txiD7M^@ZyLXnv?a+XR}c;vZ5lacHUkh84WycfOa_*;_${>I)h z7EU}J>DRF^n+3GIKtLX{PUOJZ3ukSGW1PLejHUko9Il0g*cE!1F_kax%;wkGVfdc6 zDI>=MOLL95TLO+_b3g)OOG&OIAS0C(S9hMqpb&TBvj*0Dk zy@;o~w|k6odZs!YhgU@-)X)SH{8MC~END>ij)PyCV!jx+0JI ztodgdv+1g^AZk?5Hwd+~_;UTG5@I=u!AjZ@w^Tv080#-%5j(ftTG$N!l3;_b#i^)W z1t|KwT)B$K_3V;If~2aUEw$VC3ts?PW5y!%{{FIk9&kq61Fd9F*Gkp>a9Qr@-$7XM z`46?z`Ydw{5foJz!w?|9ejm8rDnnrk{pNk8d^3hTirwFn_(4;-!ztoLsrb}pcALXr zOpIwc(}51-q+ALbi|5-CRSC5FRbSw_ml-vK7?_nG&VWuAsk5s6VE1KaJJ!hcgR*At zjE5VrTzjOyQ3NRDs-t2}Du~nkV^GzmVGariB*Y;wqaKu1)H_pTtWpS6U;UW=0g357KDuuv5gb7eHG*UL$3jjG|DN9f z2iEu-PU9slDsv|=k^4Vm_NnN9egV-`BWb@-o z^%S}&s`v89m88r!0hwo-MQSbmrY{ll2&|zndqQ8=2&I9P<(w+6e$A>M=BX>;9+plF zM$Ai!7um|pPHbTT=A~!ynY-I*MY`<1{bmj}e&>np+FzJpV3&y1nm}0x`Al3Rn7Vb3 z3m5Ubxy8b@VB6NSYrQn7I&h@K!E6qKegIPh+3~^KWg46g<6|sKS^pE_;8~D#bt*V9 z*5Db{lP0kT0Q#m4y^KyQTG~F~cVwAB$GQ)J7ipN%_^chs;;}yOVVmMJwsP4*AOLTk zboO?E>&Qs3?lQm0^Ki0MJ1-#7BqboUdRu~(_iv(WE1=++!tgpKV%I+YiE6##)r-^2 z6(4ndzR|s6xGI^@i?1tudAC>Qc7t0sG8pnhzIQD@CY|<-Rfr9Z6q?-~hqv~fhtFe^ z@uE6w-=3rG)>!W|b^&DXysr8ItX?&*iBINV3Ao;eXpQ!#JhpQ}7zfl9D;b2%>4fu` z&R1+kuCF|N&KI>DAvch^&gW$LidG5{p5}cwc^v#T8#?568Nxx^p#*W`XGdfem0ykh z#@lo&tIDRWE6aL()YTQZTG*`DFg5^1<&k3{(VPJ8PSp@{8cv zXaV{N=Q7`O3V9I>vuLGKQM$jE>`$xYEc|7 ziW1+Y*9nW^0W1|H#tYU=5>AP|9fV9qom5A51%V}=ONAeo%p=&KN3(!2bpF-Uui}9A zg`e+um;M2Vf}tn=z$pFaYUnmOQ(r8YfT__kQe8@sTS@d6+b?-c&f}ThlIwp3FWWc6 zJ}Dq$OT;}BlL=4Nf3&&ysT&NF9YJJ7+@vN=ynDGdqdz$#{#2~z&>BZcQ7MS<+h*2y zOpGB--x&ni@`-ipkYq&c$Mp4usaCN$b(fIi2P)TS#Hna_hR0XH>0{Za{a?y}15h+8DuglH z??1Cgi({wGq6?vpyh8D{{+M<1lIg$J!DBnWfDUiD%NuN@Y~olyhW|5K@^AaI>*OrK z!XgDj-)o{VInhtzpWkwrxJcEojD|Nb#33WxBX2_e{(!HO*(?>A)+M{CgAhP6T8#(# zVOm|#SOiDgEHDzXRl|O*=lu57K%ogK4VBRfT_<|@fug~r8pAKH;92ILh91#`0${|h zxv#CgMsX}ix0C?!{Rn78FqicR6OJ*pZa>RXO)Ij#GS~Ri zz#rj6+6^toChLxJV##UuTF}UU9e|(csIFhWpsu^W62e=Rt|7-rC;1sl)fWU2df`g{ zGpV;0Te{gjnUcJHk&tT+{p73;=glt`+mJ@(8UVFflAQSu>-22LwRzEXQK1=dCGSHg z>U9MD-yENp&D5JoPgms_I60+Z@7f=@<#zX_lo|KMP&wyuG|!5AURsgOw$~rD-F$V3 z&*>U+FeMQwoKmj)>hzG0z59|FD}c>RoBWl!7Osn-a)0Kt#L)VXmk%He z&V#cwSwyfzDn}dz=VzTV4{i3L{%OqH07AG>D48IAsm}zGi{oHhOTU81r(7pPpz>k6 zuf2wSQM=l}xoMbqG|OIE;D|8&n9d=ptuuw#C1W7RnyvHYqO{M}XvsVP)almq3Py*J zpF&$%zEgyRg1!WC)ctZdvdQfJl}Jc=N+W*`9Z@uzsoF%GYRF197qOO*nKFQL(8Ej* zE7|{UTn_v|K}D!4Ds|{Rw`09X0Dl!Smv7?F@$yLw&37@4caG?r`s*CY^<$sGw_H(e zL~B{*Erip*lBX`G-|64)Wv8EQZVUnXJAFX(cU2PVvGngQTnUH$QndTOl)KREo^m~M zS(y_&kr3#Yd=iq_7(Zd}MV0LCxlfTkT&9VM zS4HgJ7DkN|n!*PfJ=p;^K8B8Oufli{(b{}D@)}qJ!b6t&(eE24CbuPy$ ztWrEJi6%Fb)H_(sQxUPw(J^L%XX^$JdVOfrK3Q9Xt@d4b-dlcO2>C+f3`EEXXIqCe zNA=5YrLO0NKsy!M-aCJZGa~veUC>`B8Z)yO^X--uTpYY4;WBm;pLLtR3h~@X>`E&2 z@8;pkjJjsm#YP(sseOm$Ny#wDL-62Iq|j%F8Np(LYYxtq#gP{RhAu znGaR5MJ@D5omJJ*G&YnlzL&X>>5z^xsv%P&Qv1g?s#>NA%>JzBwxFTFp%l3ip?ETm zp2o|uk!+)rMUYV?SnxLZU8EPl`*t`T5vT{IlHmZF!l`=6Z$h{x&2flhW=VVJ?@U|U zsh#$<@b5{PVMT6SS3)DDWc(0WGK3z?cot_O`2H+_=b$R$o^(uOc-G+0J3k#|V^ntI zGVBxTodQz!FR_%{&wEaPy=;FQcOGar;5C`ASO)}_0{mP@pk?Y>;w!!-XHdQ;{I6g= z-XXx)8Qx(gQL-}jt{)W_L+s?DP%Hvs?t{-|e3iq)UEA9feX_sH&%8n# zd1BGpHjF=hH?q1ke~dbv94c=_kiQiQS+1oYcYn=>?GGMKUX@I%&KO}tVY$y+$1!(n zTQ~d)6zZS&C_++#pBEOkC{W2!gopvFsQi7UQePH1-J<6hk>>wu|D;}Vub8;l2O!_X~Ge6 zQe33M@y4NEewXxnnF>ZzkP1fumUmU%8N=RKnyQkj6}Yuq*sQr69qy4aXlzL*kiv1c z=N)YoDvwJUTVyZ1nfKB;{mEO=Q?RkKJ6Z$g<~^1*NN?#0kHEn5?NuqpsJ4CaPHs6# zT(Fsc!W@zLgKDFIDa$!d3JQoj?0f&{G23kpt(dhIFltyA)<(Aq&5 zc}4sV6GW4>MYXC1Ls`QSnOwnD&)9c<;UD)3s#h>$mIye1b+vniwrK^zh=BJOz}JVO4c7E=(dET?71@OY4|U9wFZ!w;;HraCa?4PeQRu zS|ON;kpg4g^?HwM`BsjZ>Z`1Xi|Xui6UDx3NAI~1&7f*q-s;%@+eA4igOpF9X*VD`r1zlKj7Kg}4zD8N+>J?A$ z-7HNwLVdv`k^_9bUawnTu5#5Ojc`s432aD47ca7k{&wh0e7W>_wL6Tc?ri@xL|6C2 zB$nx`B3SY&xPfkb6>r=P^wOSrIsS9}w`bc(DgUbY|K?fQaQtthn#rk!&?YY0h;ZIY zc0>?T_vuvG#Q!0FD%YO=3cg|3BkGa@8IjUEaZz0dci+|w=}G&MXJyNK=CqfcZ#Z_k zV4mjLhiIH6IwZFBzk`;5MkDcQSU?|b60(sk~Q|ffZxyMn8co_c$ zh;jy^htDuMGW*f$v>dxQ^7u`V3N;zZew|WpUFx8H~67P5AIoAPZ+Xe z$8%l96`zSS?8Oe}uK&o*e~su-WIOI_e%IqWjJdHDN$8$H|BT9lPr?HugSP@YFqO&q z3Tz|pAep)QtWfN`x!pKi187)k)13{drQ7IWg+5VsFD#4k8PpU@J0{ zNt^jGmL;CzbL32BintO2l&H$IUr@77fob*8gxl%*HYAJ&2*xZ(1*B%WDW|wXc5{%1vdpJ^$$tCE~8AJfCs`XHWh8S8eK^f-Suvc z*M8dJrpLLw{)Vdj!`sPY_^6RcOCmgcgiDbG4&zFZ=$3LRR6%%!$A`Ow-`Mss?o0aZ z_Q&3i3;2v~A%^QVi*w%e`6W7(ndW}SDd;Z#a6$Vahi3S0FadX;sh!fKP?0ubvE-;0-s1xcIAunojp^CLfu;E$kv;Thylc3` zC=HJWZs4$EgZAYmwbP+2Y%QCAR1nowcNH`cI)7qi`96%WQ&pkUe<+^(CHebJDRy)# zWHJ>L$MqXQNaZ@U#%Ank$Xb3NrRBg*AbQ3Lk~)nmHR76-iHXnU_isURR-lXzs_h4a zxJ=4=+Ze8bgX!7k!r5VMi3BY6{VJOnO?F@gc`p^kpR3d~ADT?#75Ftna2tQuDb%H_ zb0&~y#8n8z5}GLgUi#h80^T$;jBGwzBdS@U0j5SGP&$D{gu`IRq;xGY;y3}D& z6P0tpeqAskweJqQQ;Pd;K@cN0v>+oIUTbd+cv+*%S})*=+&@4~tHTCILJGMWv_D2| zv0fW|s&2XEdQZ9PAI0sn@ zQNd~pB6JxtOl|Tv((e;NHQ(hibrS6J9x*QQnq)`NZ6<`>V+$GO>GN4&#@ZD4aAjpx zgYT5z*9`qPRh|8pCi{k(Kw}l{6+_b#h4h5n3&HEKdrx$_-PNHOk`5&&^<3-&`txyP z8T!GuF2N9ncSH#8cBo%>iA)+TWGDPXFVO*gfy)Y7&JcP58q+S9+*FP(`(0#@@g^zW zCREi+yjO|R!rUOj(*m-NrI}G;rXNUHRjfWtwfCtzCbN`@-?cZ|&U?J5HY7{f`pf(1 z%GiT`e9&&!bj!=#?boNYbgI86g}HU&ApJ>f{b_`oE_Q}!uLSXLtj<#qK%_`5prA{T z{T5pT0u#8Y-cyJQtWko7*y;`Q)kLqWZ@U=&C2d;yEaKZRV{Ge*U$BOeMAW`AT)_FC zFyKEO@Z{maU9IG%Z`z{OU>7J65!BhbwB}x`4djF2AsV+hT#_#7(dN;3FC@4mXj?zL zZsf{B3YBCu&<(JyCPjBYmx77gDkBGOyR8z<7b$(rhyacFX1~jp&@oDjs%9h@yx3NE zjcPw{6>+j)Hf)t-q`$RZ!k!^ATUo|mO=AG*nl|zC-J7Y%Dr-JDwcSrxEvl4i!jO0G zGloxOfv#+<$>~KW^{vQB`Td{Dkn7N!wC5CL>}=@>*e4P5IxqaPHrsvtmbGc?km{pxmG$_MC)+tkm+nxR2L%&4~JWoyHV-d-)-OsEf9Mqb=g}HB`3^r;?K^^P$7iyXpwc`fmWt>Wo1*Nb83=!TJSjQd@9n?ONE$NE8{^~pBk)!wdWQ{|K_6pIZ z-0uEqmBeG3t`6M`%69+pVI*b=wtRaMXMR1IgFo68f;)zKWuS@r!|6MjIZw0EBMb84 zu|?iX)CdGpMp*z<`?^KY{NH|MrprLI_NzP7vX4U2JaUq%*3xZ0BeVMqh2Hxjw7e!J z1cV#$EhpS$7NfW1#?JZ}Z)r+^p)@TbI(%DTyn7g0Wx9?|cxS^6AhI0lSqS14;HwY6 zJQF(dn?`He0p`O%PTg&aXk(l^>PU$)`;Y|1ALhH>e9dE=S7dq7X_jI{e(=h8}DOp|rAb273)0#m2hsnsAhjjlx9yLdd;~VHh?-X2n@xTLZ5QFvPH;%c>E0Ry2 zm6gMBbLAolTEgT0FRtD?EXuBH{})g~5Trr6QF25&Mny_WB&CKBkrIcJ7)q252^pkD zLO>b`K^lgkl+FQ(p_`!x2KZgv_w#(;_xJwA0UU5}UF@~@T5GTKb3QNDQwWvnxG2bA zd^-Ll`jM0y7be8!UgW#y3e>Qn5Fbqqd0>DwHv@XnuCWK0x@{7$j0B3Db*?rRGxopv z#D+;yx=@g|U*eqd7}nmJ z7CW~9)?4)(wpXV8+m}@{{>^VPR!ADTY!G`;JqKM`9iigksx>y3WzEZSNu!N7LdEVc z+ge@2L5CK$DjUt#WgZqEb$>s*7s%}4x6;0}wI$6_t-Eqq@y0^ltY6gCY{!tjVsS8e ztg>oZ;+3U7=eOyqmC~sR%UR2IVi*7Dk|8@sbQaFjqFj~F{GJVxP`Y{Hn^>W_6+$Pl z^OvXZsBh`7(!*`3`}n>I5Swvksksd$=8wm%!FJYB!5Z>Ryd<1M;IDGS>@LT#wy)2t zt+Wq|!KO8ynPJqOH8NP6O|NP|zR52}-Fe6IscEgd8Jia0t0z!TPpfwfYFFpdbrp)0 zpDoFYhzCX2g&7`#1)l+f2v@yaLx2XW^noXu_K`)bCR+cz()7nC)<@m(!#pJ60qT9> zJsP$nqWQO#dY%?@MUZr=>`BH}vT|((bM)+MaGWl}3R(A6`>6hsX}pq5f8D8*9=KMv zD0&^a_uZD{{-CjcwEpAU3fyZvBvco8A_6R34E*o!u-LYnmdpK#C)ItU2SYN}IZhf! zMF=lSkHy=njzAHus@AQd8}1t~s#3#KZ#rZ_9q8ip$#Gy6QNl-NF5UX{`gf{wz@WP6 z%EwB|XKzuC3b=|Rq7?uvLdi>y)9F0gvQY7dBtN11#gb;h) z>&9#rQ%>SM@WOW}uL39g;pagXWi!f=-C4|W+>`oPkdz*;|gz3foc9vF)6~ReXZ!fu5q%Sc!*afy zl~#@Hr~P(b%i$m?9PTc?t)a%j9ivEHp%Pqw;Kjc1!%Vp7o}UTcW+>!n$3Enz67K{@ zLK&UCVDJzDEZi=nrC{V@K$&V17<_&~uJZ^%3O7(cL8`vIn^_tzU@J;LN1sK@%QkF! zN)~!Rf~zP2MZwdE34OCzDs0YnnwuMeYso{S=vm*%>P)zR;{I=mCBo0J(IFbm z^^jmd#yEO=j=8>xXZO?Pa6-YU)0eb@&(Ldc8^pB@ER1`vc3|K-``SvDM+l;!`cb{y zd^o*cCI`E>1EU@a+9f|G!^F$-nYABjo%I{ z_Cj@G>BmM-B-Mn$8G~TBIN`5x$F5tu%=YOOUCN1B%3)~0e>`o|>kx5?@AR&8%|t~r zC%WD%7Q%8}a5a7!Wce@5DA?%zLjZ*olac4)u`%|?qqw~7W(hy;sl2zyn%#;u4A8eb z!!5it0=#YBy$5{K+Y`%@9}I~01MkaJgz)KG=7CDN>%2vuP7-)quSY{1V(k;WHfaXa?3%e2vCADGkZ8l zrQx^Xt$Y(D*l+z9=}ew4;;`}>93X{F2XpEbJpl@@XJ)Y2GTZVVTe|(8fCCP?U8rL! z>6P^XV78O1RR_Ui7;Db9>+L=1)kh+6rw3~}rE(c8dkv*J?=!loUlWiDM$qVECf+kx zrKbI_u>N=6i-Q1@{uM@H2n>Ys8ih@;Vy#5w)`^b^Y9N`QC^45+1{V7*UbaRBP{T*L7Be#lO#l)H;W##oqfu2_j0m(@CFerbOdwQN_DWs%f^Za3dv zf7)lOLKw?h0mD$-pEKsBh*jQv$fbW*RG<&yvC5L2)op_gNoBM`h>arT9t0d2Jb)BBEC}Cy9q)CHME{a46y`2UD9S5ZK6rjuw0I&{w z01>Xc7n%-y5$sb0oZ#oK3njAIiv;QSs?(RAow2-jB_qI%iS6H}00I1`3h; zRAC8z7*Nt@kDLpS zV)nH!2=Hg~ixJ7EQ3osO0?ukwFB2(@lUNzz9vORAy^78$QJ1POYbB4_GY*zl%WYTy~)2_-TOz?~Q{EF*TnQp^hex3A!FNupqX|7GOMM7^JXVRs&KH0jG zoiBymx9yCr24^%hZ~gQOtUvlSbuZy=9)738AU`1&Fyfo~=jN2*XqDMnc(Cf&{u}hE zFLkxtp1t_pj-U*WQ#-FQ+_AyiT^kG$mBK*4*@oiQZb*&J>5R8u9+1UZ*X#}pJh!yT z{}f#OmNgK=e1uFaGwAwWGL*Fe81X?O`)?9mzQtR6jmcgX-i}{8oSMB;Z~n-&;V_HM z136mrb{v4;>`7c3C@Fpe0>-Dsq$+$qt8HNDrQd*1r^t#d%-3k5WaA(m7ZQbZBs%ZLr zVzB3dGNrxx{SJ7|sXeV)(2gR~z2byVSX0ww*}#pg;_X$B?^(n{eB&aspQhGxXD>1n zVLg0E7|FDeWtHXCxY?`dhnsZBK2pkunKo)#lxmV;4=JzjEtz+9clUmab(X!m+nptf zF=PQOtC)(gtNd!DvaD_ZD`#kcxkXOXqyjJE!4-T-A=u6Gl`eM^SP%BzBYGlUe?B=*}ALaAU~8E6cX zuH}AU!M-)xqq5^$M|LY5bmqQf!w#vvC->}odhPjRt>28jgRE7py|vtyC$AZ)@TpO5 z)LA?ql%k8s1WsP^$bgJIPi?rfNZ!Z26!Zvy@LN@# z4GBUD@!$XU3i7bXKZ#Zeb2N$Q@9$*@2H;&4_|WaIJMQn&r|Da$0cSo(vuXL`oy z@^2jXCiiBjdBQvQi>{An)DotX0v+l)ceE#axj6ex$X zdeC@=Uy4P+yZo>3M#*=}{LEBdh|5^B*%)H=@Ub_Op@&5hFrey~i0%I(FD4}RJiLs$ zG~#xM^zyGu`VIF{AFuVxDTP({4=y!JI$s16%J~%}Dp@(gExWi?qpMpj)u9fNJX&VG zSZ8wyj3J^tA@i5$9t?Bro~ZgTLKhi*7vPV2DrRXOb}X- zXbrXH;_DRvg*7?)a&1d+f-_@4UMx*YUCvRo58!FWK|edcO?cBr(Mcog6S)!u_u0E| zyJ^_B_$Cu>(kRMW2z~u`O$~Aov?udDCg1?-m_V8ih_omBjIO-di)}l12a0Z%{8}%h zR|K5?iVKX3kQ%sA?|?pr%FbF%|Djl@WX5N?CGoh|W`t9=BFH zjHgacQ!Pf%Z;*MCm76B(K^WRYRn&h-Eka902BhjMSP|y#T)B9!RSni=;NG!!R~e)2 zbT^Op#$9Vo_hHBf<)&MWdoBqj?*25fN`idC!Sufjc${Cp3CDem-_-IO(p^MG=$exB zv9{IlPkk)$ju$@PJIbQke`K2D*l^bvSBaK1Sl)eZK|N6V9Bo%9y5LRS!Us1`sGwwJ zA}o3QNni38kO5YTp8R^ZlR3I!I#3A;d|_un?cOx@;aY_8A&&AVzw@-wzdi2%1nCvX z)h1FLk;1VR`Yk10<|6cmjTaryx0JkWB!f*Ho$iz0`cL*5ylOPde0%bAGVWC8b%bSZ zx!6OV9mUy(E_U#(8bu1EIT3B8lkY^o*i1^)EsqlY)lynl6vK8kSPo#R?9Vq!>q}i^ ztkQZIp|o(`n@ct1npEMBr`lp!;4UZ>EIrwSOs45*6TF42d;{m5^6jnO>e0MJ+%oY~ zm7|0a@lz=rd7+f&`_$mNE4P<}ch*@D{obd=qrGpMmW~$rL^9+&e*n`7tz+wfU#lxQ zrPY4^HPZR<-2AW8PF2~6(?-b_;15{^Tz4d?k<@e4UY5EkCT+twdEKogd^M$^A5P&e z=vHLO(ZzB7xhNbBjkq?^3&(iZzSyAoC{fRS2Co(ZS?s< zU9$99dl}p}el7XFx}Z5C;|C#nU{p*Y=H12!L>fKe30_m4%n2C5o$NDS1f6I?Qb+j~ zdPMoIV;VA(@rn+~WF}b2uN;psQeyv3Gz(MDq1f~3)vhIkd>V@cLvJqLrjvbihc92H zFM*Z)u+u)BA>DWxPOIIVu9;4y-gnCmD?uC<)|A4e|A$>xY8B#^Dl}8fEjtlG+4Vk- zXeC18z9YAFl8jDpGe3>gglNOwk$987xeNHw1kGEy+I}GvqT~U`_7N@R`8beA}w+FJ6RT@oIz+cbKtt3DE zPsG0VCoKj>Pb>yxP@aUr$Rfa5YXTtCmGNTNe>R_&Y*h{GWO^G49Hjpv z#{i~|>S*F2bNy#6)n?Q%Lg3`H zU`Ipsq-gqF(H5Fk%>Cv#H=9kyKsEZ1dEciFdu6OvT2Waao}up`l#Sb8nPPi49q+kS z-cI>q^1qWb|2IF*;rg1w&CeMd#w2FrwpXEcFr3F!F2hi0iq~T`tKw9+c6X3|T*=Ws z!!x;%enmMTz}#qOs4gS}dzMiN)X~E(W4{v+Y?v_A>_zGz!|>vYul>|1vXlrak4UNZ<51HhNUnJK<1|Ni8 z1_GE|Gl0kbd7SQCJ;?p*#&MpgNwwaM&FJw$S&?zw&aZo0A%#FS|aiWV}Sq`zcLKyW`j zFjZ=JqFUh)DJ|4tU%fNHw{VgDC-p8R=9YrOO6|@V7(gY3hP|nR+jPIFTS|6jmG`so zI4T`msub#tFr7CA6vyannZKunBI$};iTa0kK6{jumFllc4mP~~+a8sfUNQXr1@|Z2 zwspWhHfJ-mqc={y={d|)Q|F#TGUI!pzb|ih?Wm3?B+{kX5i$fmSg^7RtTao+ZP#)G z|0O`1zZ#m!f={I}d=PiCB(q3sI7drGS{ z&srhQoAKkO&}S(`*|6meVu^))#3VRFZ!n|#4rc2 z)SUnd+MVKNKz;XrmgERyFuk!kXi722?~~USdQ|W|QpihS0#L65C9Q3&EUM|S#C7ti zP>I2U%#O#uypY!l*OOq6vm+B6_aYvrvRU#CzCK2H|K!GY*mvj0(3Vj_TQG9_`G6Ws@xhbWxPQ<={|`+5Pq6?Dk*JW~PRxOe z3+aT_=4YR9*tmy}2Q#T@QDGG&sX?aXZA9fX+K7wzNL3mgt-Tmr$ui1Gb>jHAbXwyy zN4Gw2lPdr~s?GpRXO|T=sYn&OC7>$#b0w8wxzD?LhqgwLzmRTttg-VKb7wvxAT>F{ z@I7I_glnet_tLAK06Lhlzs8r6idT5o@V(!=pm7u{f?m`^~e|E~hWE`9+ zOR&?cb1v|qA|w1(GN?xEkUbR44fEU^ZUC<%;w;8(%i@Aw9G=5$>2R=>uU*JBDQ8~ia2#R~ZlKI$_HXpx& zX(yppUyyGCZU>)Ladl)oE$$W4Gv4j$k?$w`!+1ZzxYd=lo?+ z<&wcMsXZiJ{dhTYm|+0Xl$Y+kUW{);ULKgD3F{Pr-~|LkoSIFpmyKKTEA3*(({l^8 z419&82{P0-EMxStZ^Cv3p-&eU)faZ}t*JEdb!rq(GLvWmNo)p#Y$mh&cTV;L%)_`; zz;NI`$%3GA#c=F?)P9cAT5BDtz+~prdjg2h>CGx05!ex?zKKqzg9HhI#&XSbD2vQ< zc)QUp9<&{aJ69QhB<`a!HMoYf)A0nk@KWBZhaJ_+kG=Pu^2-Fu_fd45#@=fl2Tk6| z5#};z`WM~(D1?OV2Xb2(gE(A60}`02HMAvpyNgOY5m5o8H=2#kdd~y!gqyyjb?$d=TmYoX$xeD*f3LMEFZCzUx_O34yD|_1+8a!U*Wx?OqX#;Z)F9cge?;`#05kfW+C7 z>Den*&&_BJ@9ebQwpY6}C?8zgy_$$03ArBZp*bO%O4>XlGN@1 zRIQ-v2ypR;_WY*e3k(dt_z^GmcOig~T&MPRy8q<>#J<}B+6-rjeW;u%O)aeY$yiH2 zU#A*~qH$p2MSLM(AJ8o(#xP6pPf98+^N7C$y7JIO`2V!*{&yG$qm(&ghpibfZ!t3L ze^@(jQS&^ZptA4KjnG|@u~c^%^g(vZt5myT|G|ER9Q}~W_*ot!wffmp%m2L2qintx_`aW5i<=Om1b?*&+bx(Xt;bEeZL)yER*{D$ru8Yt-h2wX*6y$^)bFZJ zti!~!t<$(ppSo_IhtHE}l)T4Kj}+c0(^Tw57&FCf0{&a3ww;5LK3^>!seL8Bu5Jzp zZ|~61r@0U&Us4y%-#iip)@6Y-p)q$O z0(NR0D$l-yHElX5VYIjG!ox1v*1E3VC?DGkZZ-bp&!wxaTq`7h9iO&Ex;(d`vPTUI zsAc|0=otV2VU^3t!l)J@61f^8zkPh0Sd9R?vTThStgC;9Qs3?LgTBJg6VF!x*g+IS zr)2zz-Oq!#PW+Xv(M)M}&*>py{HJTyH*GnYDBcsqgT59NP$2dQ97D*sCn%oU%FhQ? z-oj9X?^#b}d*pD^sON-#6^!9jr^&rR9UT-zCP6^R#r2dE*LRs-zy0^8+qR!rckk|u z+1YXZ@yK`6YNs7T0kNNbC7x8Ci^Kwp?M1_$PrxV=D%A?NTn?9C4m6#yN|I6yR(op9 z2qd+3(|23*)1?J91v}VWuhbmVoE`>&f4LFA$Jqfsk!TT#Y4fC3kAGXHINoreRb?mL_|ljQ9-W+gq=-n++#v|9Ot zO9Ou1jp-rmz4y01o5l~~f9D+%uc@`=hj${LJj%FkNNRhVVlBcrQi$=Io-W%07t^{9 zFJHxN|J}H|ee1;+jq*^_O)hNQde@7E0-a7)O~r~QnIAq9QEboC2GijkERg~(Nj&*I zFB-bQ(C@JUbq+HhHev;0xPpz}nV`5-@$Y$vm*P9WR(Sro+o$9IVf1JMJr+D2}M}SKU~9fmPEDP{8#Zv}tEqZm|nOneGJd|n{%cf;6i7LI55k};I!is|LukJn>6b`dw@aXgfPOl zXJy#4`NSv{I&Qlb2KEu&+Fus?2DxxFtvt8g@$>f`&m~G*ft!1|_=;`~VCn{ev4Ees zKtm7ISUXlUMDCDy(CEqzpE;iyPCXY)hoYF*Rs#bdmD@d#e)YJ%U!~}GhMVJPEWK@- zOMbxNdyl^*EQZkYx#_C@OWaRPx5ZD%DO^^{yT%QRKl_@Zb-aql=1_gDv!*KDqLkqn zDaSwEL{xOrB`sBPoUb23nrvltRznt#x+W*7u%=5?su5On<6S z+*lJw1lSQzz)C?odb~NA1B8K1P&3bcrMFoIkjb$Q5}e#HNr1>RS$j}{ii!o8f!)-6 zjuu$Eu~K-hXEuWbd9;+m#73yLkM1I+-}AND$Pt1m-0ZuNB1d{)w`>b&7vok{#eg20 zO!P?@gRMg_Z>9_;%;ZCfufa~dvAAitCp>U71TJU!8SpH`FVz`|jHW~~zUc~4VXYhK zh}PXT6~DKWk|Q2X{qdd8rW{FrU4-Wdt$g}$ANlY^QtN)U_WqwXLxzgNVd~jM^(N$R zAEl;p*3XPe-Eh#FF4(NuE-3B(4s1m}k^QbNH{yKLe(DYCfWPkZVQ%RR(kA+U!roVW z7vKd+#Q@H>_=Fg-B5%wxaQ`s`2tw1mfX4O{`R0JvgNA1?4N|cG>4D)7(Km4@dtl3; z@FKpQ%=z+B(0je*k6Z=u!OELFtrRZd2IX=MPeca8dggM(U;`{0M+ zUjz|BAwvMQD-sCn%=qbE#VFv*sheBKz}7SdzYDZL!|<1LcgngL{m zDU)-`Xcw4po4Moumo5L>ZgH^C*~rNl6LuoQN_*U5v8+#;HNJlN<{+~wcsZw3mu>uQ zYYHx1kMX1934`mNHk)0{pE{ROiE^NeoDZm`lrn(Ab8SddOdEgl%(bL$vKY?Bn?9cR z%hF5Y_ti%DfA#`m0;0`)JJ&556g&SBdxxU*=6F_Sqs*qH4cK7%l1JFjqs4(E=|}{m zUbpi7b^FAW#AZj~w0Gd-M9dJVGyT(r*X)+dAqktuGr7~%95YW9inGl|r&Rp#`kLO) zOKkEodlqn!&tnmQTjD#ak$;uf@4xN7)(g3CTOVQvnTDHTr5Z#{$~kYb$vr5+!RGbP zs#@rLO2L~6#D%M;i-}9LML6o_PWE4l8QALnJo$^g*OljXLE|Njr;p)3~&<)^@G#BrHGQcDBy!&T(O&C@g)Fb)kzVPD{aB zUL*kYCvbb`kMB*=(U}vJbRNLet{#PWOqQ(i2sD&VDN8 zIeo7IYey4sLCq@c-3g6VbWU`nG&K9YLd;V-{^A$V%^8;HS4bQ)+t2hkW;4!DY<}xm z>9T?~%PkqVkNqjSf%fq&Y}Mz?9^C&* zLL|8ExX2gehG@S%vQWTU+&o!%(@=ZE!sqTgR6Dq$D3E;XImi94e^~T#(Z<`ZjfZ0% zfVw{KMdd3=3%8IJ;1bp)v;KcA-e5bsAjr7ihS{uXz>GKGOq+g^-YS-ittz7OUsYba zMVa!~ImEa&e_Rf6d?|5|vn#@zEU+2lX-9}3Wy`4S~OBVJ_#bo{CH&LYsG6F48 zM}jy|DDpx2uAR`Or)bH|hQu20Eb&ooST zH`9N_;d{lD+%je2(%%ny%O;z=Vyxpn(KAQ($XIodD2XpUVn+Oh7RsXF_*L~wxpskt zy|z7tLiCvDdvN(dNa8dT3kQf1gGu*24W3tF4os5FKgo%8j|H*MZ_$Vnsl!zy+f;+UC8xLy%PC{N>z(Gz#zd`fa-<=_V0X%SoAqmgMB88=5t{QF? zzs8pUG4;V6r?3BSPyC;kNO7Nd!Z@c<5`CQOFCBmL=vDsfvCMyKa1K`Q9P?| zapHJx>)O9N-|;8j3IaYv45yO2YECuaTON3&Y4;{W7f{~-bAk;KI;ALJExv;)=UE4y zPYoO5JDLi3DE)mNck$H&M5G}-H9W$j^PDBOV}spbyUsVuY4~6qe8FkqhCQay^~R9k zYGwn@Jf`cZijL9;U*H()oE!bQ3U7LKafKzlk5YvAzx}ni_qcqtoa($cZ5ZWrSuH1I zlU_Sp>ClPo(EjmeGn^+@-9B)E+-{~`SB7`%qHs|AAdVkmiK+nHY`$Gt=mMdR4~L32 zoyNIlOw$#XRQAcO=_5`JD*O4sjX9}N*{?9I3(sNe2DEfH)f!yj9!FmgT27^PuW&gZ z@hQ=p+2qYrX!5Qmnc92a=vSM4?~@)^Hy)!UVrKoH%gwQ^@FU;;NYoX@V0@i6Z5P5W zeWOsuqBD*P)=n3JVeqbe`Gq?#!D%in;+$Lz z??(V1Qh4^g?K7+mB;B%*h$xvv#w1PLN|Z>zhS-Rd==_w-MCb#xyyR78TTDoNaGf!< z^t{c0d}5n{GBQo21bHv%1?RpEH0M-K*mLfM)@z0Ng5e3%=<2tv2n`#y`PV$s9b(Nm+Js#-4x5gh)PBYIiIHHJCfxEGIk>4}xA~Pm?*|Cx&%$sW z%Gz0Y1tDj{{R-6qfAY}NWGTcXN~!J{SzsMCkJuj}+wdG9(>LkRtuG}<88u?Q`omgj z{l%ArSZy(tqKww7M3h9GvTC?u)52q!z`2%ZSJu%BB%!>Qwmyek0Q~a!oDCck++;|2^V? zeUHhRqMT99&*|3`50ahZiOA*}HGApaIe4z1cVlgMLI4>Ty_*WMRh8Q=kk0#SdE83V zyzJhr4!3~${@U37;*bxIQ|B3bk)L`CCiwxf5sjA50NUqkAsl!sUnmr+f1_DNi?p^6 z-_yO8(HW3McXtpQ`aMuX8b90_In+r5?gK_-jtwt<9E1P*80+mPqXJ6XAnhn%t_ux( zXBZe5$}i;0eARfFcBCZ`a8isRo=!(K<^3H>-Pn#7J>Mo@b6UakX9QwJ{xs2fzaQ%s zK%@ihdqiQt{OX9|@76C6IVHBs)=Tx{>9g0V`@gdf+Q`L{xC_Jruu*FOZb$5+wo7az z?Y1uLVpt##o3ry~;q2sHGvE0sBV(@F8$WTEIy z5P<<{_4iAu8b!AD3qO7=gD~h$qD4IE_dNl*dfq}DSn0}(Ztc4)d1S4na9yn5D&t%u=5znH z=s|K^(Ji7V29bM#k|$=fr$%O4S1#{zE9vgfr(r|HtIJQ~B9BxP->ADK{^v*we$;oZ zO~|q)lX-6ZZqP$zA#NG!j>p%*l) zk^7zsHl*_cbB8R;hTM!48W_pOOi2 zF>x20!}yhuv;hyKD&~~kb0=CNXTuF=Gi`Eo)5-BsX9`pq!z8TO1Jo|!&iHL}!M+J; zw<4HLeXoCS5OU4`{L3oP(sUv2Ii?1Hhz^-z+K&)Zcpn!B6N$0^ozrvLTLDyqCB;YE zxAu}q(}34Q6isk+x;qq5xP3G+z|X3-<&>RyE{|RyQbAWLHRT=n4Dq6A7r8FK19^tL#CK-_dTo~s~YPo}fzeB+!47w`!GKUNr48b-QEfC2! zZ%#XO`_u(sb50H@i@<<7U_vmCbcT<^Z0EN-J$pxpT-k#QQKJR}gy@p;?-wYuraapzF) zu}mF3<38DJ!_BEzX@7?#!Dptvr`>+TrabS&V31+-PeUIOH?~Zr!?pfA{P&p$``5(+ zKGflO#Vp)+pD2h;a%Y@=+sW01E!TMfs9@1T5C(-Hr~>Z&!=mO426i zV<7{JuoZ}b6!$(z?ui(*mE0)Gl7Z8d6hqbQ+3d`%aPgubXd&gS~F$#z0Q_h+&CEF$z_0j7tRruypB2M7c5{L1|CJO> zj0)Ahx1qrV3mKIByihnZT7Qa^gP%ncxU_6V<#Q+i)+bA4f6W1Lrwg`m%f(gGJ{_v+ zIb2*~x29wC+1qO^`vIE5UaHd!}^eMyZ%~-sjYYZqHQr8ji_F;Ne zu)NK5w#SOTa;i_rH_P;heDjv1s3>0HNLjrwJppjKM8IF#Yz&i?xAj54```&iBG&BC zbkAZu|70gZANtW1#TAB?ta#14#8k%CL8i*uTCx|hwYOk93mSkQqihTSyLy*QfOTB; z!ZP)JKvFo9%YUEy-xXl>f_d)jF48r>5l{X16`!@y`vrv3x$sB@`aSKS%}zsEa&B4JLa4=JG-Wr5?npVC zjNU>ZKhRHrSimU(0j<<)g==QHI{>$JEiTacTwL$Q)3w90K!3Co{BW3Ez6{~T+^m8_ z7OtAf#4dEk=>miLFHT#1X4Nb0!I@su0qq{UW|nYE*?`{(erc<>ZACG6ERj{iH7YPJ zFRO3V^4>=ct%mWJ#7<@U59rIvAx;33(gBmEMmO=8R0te@*~N1Bv8)vC{KfK@KB(;e zse`Kp_3&=|RBlEnqiyNC$9j;HWU~B?tcn$<^FKEskrCMY@{U7kL%S-{`9-B);u%Ct z(~r{2-^nc^bc8cos_jCZ(-Wj+_wRW#Sw`0bMg-Ni9aJEwzr!^5#bF)dyN|ILjNawj zk_(dkIAxwiSO(u)pr1KH-Z{~MJ6OvnU(1h`{_?i<-xcUaoV;Z}>L#Wn{ZBtRc*({J z6csCHJAWH|K;BLmbc4`E_<9vnI60ZMtRokJB{{2zDYuT@Zk2@6Sa%uf1}!!^r>ROM zCzJ8SElXbk6xG4nB#tf+2u+dq?s!zc3nE6h4jP-_=hgTrkrC^%Ljrp}GJA~B3f7fa zg>@|$N~(in!{!BWFS{SZ)B{pQtx<}Vwghc`=!ZhJPFoUfFr=M(WDFFlEa|Jh1A6-g zS*AOC+A*c^MowlCX|4D>a4D85$iSFE4@*@WKSzb%bjziSebNM`Q> zL}Jg&AW5aruQ#$HKS+Cm212i?@^d}2;2<-)-O?rP{Xw*ir!b4MLWgesnLs?1u2pv2 z2Njy0MGWHyke;)0boz=+hiLq5TWeOsQpdt~>%6bCwZO51JHVYCTYWBALA=bTG$qNs zGE48aE;PL>em9z&ByxU!L+t5P+$_j{Wq(Jg&c+f+osuqqBVn{1Jdri-p;Evtm|f88 z2fU>CV>XkFSb)?ao$hr)bwp{*p_?N4JrO7KRA5&3DDwybH+tQ89<+;4Sg^ONR$%_A z&0{Lv`Y;-*dof^C|5zm(;VhsuydQ+ev+A+v7ul8G(=Fuw5m?$t+cw zdu;P|N{htOCkC0+Pb^c-Vzjj|8B!L;>-p{-cbi&<;=F7+q-}5JU(9u|Y|Y8q6qfbj z-~9LVG9&iA@ECM?C!P9F>+vB(VxU7z>opchat;H1C3r#iI=;>3YyQU|1~yZx*YjL~ zI6ZRVb$Z>{LGJ62Xb#(fZ2lt$;jW(ARvNI(sT1hwMTI4FA?zbZltgTF$qibUgDgvi zis;@yM5wmx{=&RNoiw~Yi@Rf1Ax%Jsz=BXSYp1LWQ`1s2ZP`dWi7hRa+LD>i_nFWA z=#SyrV`O5b;S$TbU*vE*9})v&k!2||{r=AdnioYZWrpG|GK7`x-GPzBPEhnjrM39i z5oGI|gj8F$<$YW71{81q+=BppSJ%73b%5XoEyybU#@zR8YSwb*gs{GtADSYC6fWYp z+N~1^{J(r#D2(o~1@4l7(<_L1H;)|hPX+HxjG?08438gBZA)Z|dm=c(FYTRtGdV;jjdzufBvM1i66lxJrx#k5on{fd)-K?UbQ7xDmI=G)q4Zdc2i9uHBJsnClgV&tD^N68c9_f zC41Ku#SfQ>#kAyWdH;E_tt-P}tUyGQf4r31h$Ej-IhmF)t9?8FoG+Ur_qns?=%m%Z z4-TX;{~YU|K)1ohdw+|RTwjSJfYmVoVGchF>3ZbnBiQM0nm)dDZXHL3y5K$h_M4q2 z+H1XW&!DO^^!a!{k z3OgA!q1I&s4fc^|PKb3@^c671dSOZ=Y&yEqQdV;&_GKPO|MF^Cf!$*ik9<*gGmgVD z60)DqR2XhLj#yR&dc3rj6`c*O=zq`d77F1TL=09UH(rGP697hch)hpBO(_IF)$R`3 zsJYJwG2Ppg5y*@MK99+f*RslgiH`+#OfMY=`LF0*W zk4JBctuISt=e7d9%s$P^?hgVMSebY1he}Rk#b_Tu@#puHVC>>-2jKCqN zg~E6D5*Iu~vp(%AC`Mfj%8;nRqWj}(lYYXGN=e_1FC3K{B;IXLKd?nn7=G@wXYl$Q z&}vIV0jjadreI`RP+Lf^eQi(Cp~SA8Onf8#=2ha{B<=5^qZ4`H!8mN7DbaSYug$rdnr~! z_E;c2PAHk+T!tDEk$glFDW_5c><=}PSl8O0suY^;h|-?{F%tiYgMW4drUr`L#&-=I z1&6pfJLH$N;72VFr)QCwtXmt|O??GrQxJw@LY3$v)upraY}YNS?9yo#gUx2b{tF(o z_!h{P)rQwdJd%Ur1Qy4zg4VrZ{6MMVwwMwq7NhES!lYbr zk7T~cvLwI?1QXl=HqFwe6J&TCBf}Z;{zc1eA%300%5{qhXkJ~e=xLU=f2$zFItUf+ zPOQ5pgqT}3?sJbP+QNiYtRl{q%w>m~>zT3xQ9C6-JK%xVr2u7Kqjk?;wSS|1RS{VJ zhnYe=f3B(-A)peFvJPtWn*C)ufuuUkP*uH#B>c#D0?R50sH*TL=p0#6suH|KT-sc%r)MK`hQ48+~=Ya^Tf~#zMf)zG) zF=Xk{(+?Q;cuEV5w=o$3gU!o#Pm^}KXqzMvdw0zhFeIX_<02n;q(ouuZTBCi+AZa# z>^T*X3E#wMT-Y{I=TF&tjQtg`?3qs z5<__0KKvfp6XPqDwvRgpXly==BNK30{ZVE0sDid6#G$fVLm*>>VBS%p{8ZxWhWBCI z2=%|GV2hPaP=65GS!+g5Tp`8#Z7HxL7Ml{^Myfwk;Z-msE27m@vA*el`=7rVui(e3+g{?&66NBMatDAdI^!fbNES-@ZO?+p`15 zV-uk)RFKC&<%8?w5G&SsrlRPLW+Jf_?&Fy?)Pzp0eYDqPRf@R9x};-1sN2Skf7ceA z#TK~t9falZ=^&YqImSdzfI6=0m_O6y=##w}0zT#9rOiBgjR}`o?VLtod5r8(T#T@u zGBidH7u2x_D?*RSgPtgviinC(5kV!^krA};PQ%I_C(HfRszKFJBw#o?_{WWQK>N`# z^#sMDpUvOfOL>o4uWclOY*7baWZa&8TIT&{odEB{!z!ph85>{)@ju!UPcH(#Y{JGg z5Ww2emdSs_;E9gPY1G)9ZLJcBR(y2-=uX9onqO>He)xX)@CE0ogV^UIwhH503Ka#n z()+59!@VbHKZ;pa*+SzIZ1OCHicIwMku4#X!pC$!tF8EFdN6sI@V~vb~j(xfAS+m2`d7l ze%7l-74UrDDTH59w~+XW1XUGu=U9Qheza(^iuB*T!4POV_cm87iiCkPsdYe?yr(wE zdUJw=V$6a1i5YvM&9@Ty;m1PDpBCr^WU=dS26=@mB8PRnWvpOJDwIj?u!Aq1c@0>i z=7X4Na2XVC_Q3eU%BKUy zlROQ=75ABr$1h()-0<4@iz1E37vZ(<8L2;N-p;d;ldI#{M zba+&wS6A1U(_BTR;~SPlTs44xo+)9zc1xrSS&-20hhVFRA8Z$l-aO z`@XO1IUq_Bc+i+BfTdqx7?db_C#mZwetzUbRX8&L{ zZ=9IhCPgY(TUhSI(BSePoy-y=l-vzeYU<{b_&{tba`^i+FIbl82gc`Cs~d#ot#h6_zD8C-+T&5w?fa?g+E!{ zVU)7cG_)X4Wm<)?<(;YEj5>xZ*UxrvAnTSe?J3%nY-4B@7abF zuk6N1?0%<5-2U%7u!@ERDl3fIseFQW)`6oV4Qo&bqFU(kD+m&kBNcPUZ~4a@c@C?* zjSM)JoGMTuHmS_id4_2@t_$JUwZg}Z8qAhL;)sfCJ#&n=CT?#uffN@->u`*g``+Lk zM&u9gmPIp36s1<}4I1}C;Lsj5*yQ+pd)dU7`=9+hCJ8L{E5cPb&#$rqNP1Z`0w7c6 z`x-)|ol$2FqSP4PXjx`bQ8Gzp)rYDMp>Ii{kq&eDu41UVDLRv$6=FR>3r?Ib*DRtF1tL&@Rzr*%>#=$b{7lLSiU1W3mcj1rzJ^eKZ@P3C21w24YiN# zj;^zu_(U0V58Ah`m;@*}J&UozH7P_Zj96huRXjA<-C?*mE$;X`%JV`d(=|#n-PKXj zJzw_C-#)X#gPyP=WCB15>Wwey|~*lfH^1^7&l-K8hvrcVgKh=klg8#T$|3h z72jyGspT7hz5HTtS>;UO6d0cX0g|O>ic4v>cMVOvWWXzIqHA;$~BzEKV zVgTV*hdXgW@%lobyr3mpCdzZ%vmjf3j%cV|3%Mj_@gw4lkl<6)=$WmZzf2H*v^S#y zvBC$Ry$O<8jB;Gzme6VYv{dG`TQQOJP9}Q^>i4pE+D&whVdEwydiOBy0K(-%K6cYCo)D$oK%R>zin zY>Oo~P61?H&Xvg@zCFnp8y_DRVKH>M3HZiX?o0JjQ9UMuTRr6&uJZgVc?vcDta?)M z%`_*_)DFb;9oRh7R1OI*54KUKD!)rRQ>H|WlqxUTS|uE8g}jXudXaiSws;)1QY=-M zpcByQkTjCuf52XQ27H3f?x@Pid}|>$(7J5nnM&ezrw9LaOum@hblq_5fp50${+{5r z^=DG}P$90ElcO;P3V&6703G3Gu%bf%!>W9LFVhbjS5JJx8|K4>Cb2ifBcPeHx7li6 zA`aq9T~UZAd6iis?%LO-TXkTugAQ_~7(NUSIqWTmrg^PZTsU_2A~!(Kt^4+;YzR*y zI-qb1Zj#tU(C}ZKs&vKqlwTD=%BG{RU;kowLa?woQl7yX|geiG0 z0;+O*%y7sypDSO5@;RnQVtyGU@_3zdvm+!*4n3t-cb85C`LMF`6)sg`=z-THT1@n5 z8s5Rci@HkOryjvKw2Byy2P?ClP^1nsu!`b0Da`jBf+cWkVYj~DwFw#gs6-ShQ7>qa znf7ET^V>!PFz?%K8sO=Ft^D}wA!zY35lhD1wK%I2Wxy7we^R(O^VO4qv_#zg4Di94 z-|tTkz83?KNJC|LtHmKh__PC7N5z)M>NP&K{P-J*qQ#%v!ebLMY>uy`1%*4#V|jFD z9Dy{e=Bi{!FNqZ8^*J5EM2bAd2$5OI{rqfDongvn8gR@5X)@lcF>bUx6y>d#Y<$D$ z2(ylj?28B}@@~T>@)CSIk$u05`AoUm0I;rKrS~?9hN8E<7rI&9`lbL> zA}u`&5w@h?Oq=10luyPA$03`vFsyZk3Be@Rnd=W=?q)%eOYizYVloKWRT4C)@pf+vl&$+iZji zwH3*R7;c46t`Fymr28Gq5sO3<9f;KS=mmJOY6X-|4HukhjP;y?E)x&2R%_MOvD9dnaU&eJ7U6ctY^CcDAK?kB^fZ~5-X#R@=S%yHiJ3J(j#>0fLe&O}%IR za#0^q-r(RE#!f0m*8YxlrOvwV*u4tPW4lDj0I!W6=|p;>lWeh??1ZtRNZx(-T0!oy z&nsOBS+NrN$?C}7;%yFNL3o@D#%KsDb=_u%PvNk8mPS;;SqiKY&ldPaJUgP2!MIdx znZin;*xStQsuf2vqPshDLhNdXb4S07wt+HMgBw$A{)0YTG(mGe3%L*t>6X~}?!5d~ ze*sXhW!SNpP!Yp@U)ame8l+2?_6&YBcd6CH)YpCJqQ1>h#D0S^?@ zvd(lO8O1K=`NBDzWCA9`bYBLA*FKN;F1H;(eg;m$&@NTRS`8b@Fu^Pl^BCv65=}K` z66w7d%xjCOFl{pK=pcUiPPxF_6oo*VRU(3{Z16(K@p4(x58X2r9Eu5S$FM-#rod7( z<|hf48GmsHxrZASj6a!Oga&!V5d_reS+?sPM)JG}->e|#%H?VJ!NKG2gOd_(aV7@d zfozlp9r3OUmQnf0smSDEMN3=I-kb%m(2_-<5roP=FIm2X0Yj(0U^HpQS!^w}ohEr> z-CrM_X?rB|aNJ~Ttv~bWM8gwf7I*POWnwt>!*9Hzn_a7xX~A(NTgziy$AZMI)e|bq zgNA?MX{u*7JV6qAV<$7~+)vh2c9*pW+v%wsf?po2I3$=5_P7H2kP$>D{K5#$m0$u*cRa+NXC~3{f1sFm)+h=C^|J98>!-!8QwMF z-o}jumEKI?ty0SVQQgf%?pJ{`YVLdux_phsJ+z0{%2@ySwi$k|Tu#fvx;{0bwNF#3 zKLIw&0p|n{B(8N>1BjMwzP=LON4=x|ga&4sSf=I!`pddjuzMABl%T^(=Nm{G17>XJ z5-4&!(fLyUQMuFUpE7~fh)M%Ho_n_$@IMO~r+h^v^(Te8Pi98t(2=KFC-lWH=&@A0 zS_rA7MZtpn(1SV$_YQfS`04vuDoRsjHeCe6bRBJ8&KmZ!f^CoDTK&B-7zDw*Hg9Gv zHTU@rqW_0|WW_JXB^9@+2R#u%!ZePTluVFQ zA+46I6r=+p)DlPQ*d=sF31)GREXm~x(B5*svEqxPRhRX~gG^Q*PshuV*o#+Dku}tU zfH>eWO0HHiePk6Hin2Pkdyg2T+A}3P5otN~UgG%pcQ2w{%j&7y-V;QJ-xYc1TQxUW zUs{vh>Bq%C#0bae(GK(6d#ileddy&;QT1|;nV=12Al5wg;pMhjfs#`H4{f>?e0q$M zngt|s^i$eDMkydOO_^gk&tXht!KnjB+>k zY$wc=M#|&A$l)BdF?Mzwo^;GP2__Mem*;M`inTsud%q@yXCTJ5PRHUtPUbYYRs82v z;3qy&DId<`Fh%=xiQH5)L^d=S=(VdtxF1u^_QOn-Rj#TD!+MLSSN8gCk>Wckr|x~Vg(6NR{IaJ z-QG;*Q`vC8*rPD<6C@^?Xo|MlcUdM@5s1=+$ATN0_!<)t+1n)75BRjYyQ1&7U&^=G z4s}PmS1HU|_;m-3A=vxsMsiFmSRl&oMnAf49J^;NaLiflih{f=QMK)#qrsCx-m@{9 z3YUuc%wRk|M<(V$Mh3s5>fW3meq;*7C-P>}Ta$4vjAw=QInFJt=##tI{i ziQ6VcvK_2A9HlzV=q*BlM#6-cWF75S*bBbvR^t!WAMHQMso72bJxTc4 zcJsT;jGvBt5HZo!Vvh7&tx$Z!r2$W*FMLTMGP9U@w^K>))7(2=P)uwPyYlNrag2^! zf#XL$=2%5k-E>XEDfB%zG zo)=w1R5hC~FCg~Ia>D8rhf{2C5ufISf^*C*u7`dhP1MV}r`gGD!{2Tg_?U=0+LpFs z?=db>f^#+8p=d~M-u9BOLu$hd#ko>-{#&ilL`t=^3=(?o6cA3^$r%3?&nek6Z5Bw2 z%1*+CFSAsa9q(U(L@%4gM98&&zfx zs_i$a!*B5w{q8X>L?TZd0szWpZZ(^AHFbb*9T>3-KQjPb-}oKy6(qBf_|;No1M>Bk z(DpPpW9{^9##3i2=HpCB_}83Mt=~HN%A1Q9ixtrtDr?C}m|QMI;I_zU(DlW)tX_{T zA|Bj4Dbx2>ykFhog5o}WP0?j|ij5CtAO}LyCy!*v0kXsP$0*s0T5MrvNm8!9?JL1R zI#5iMykMuWEZiop@x(T#>{#P>tcER61$Q=!ND9}~{k9Foc{ z|E^pW;|!4H7`t#jAaTUzYFNu zJuVti!@Ws3F}MMrN61{3oSzpG4K|MUJdj!Gn1wRu*7~a`@BtW-*za z7b_iN&T`z6YL5i`d*82ysAXx#<1d~KxvvviB!n|_=E-<6t*eTUpiF$C>+#9Hl zc}~9gV4$?9ancy)cxzh>Ky$m!KvUp4UHt4*u%4XqU%RTo8}M1 zH5KU_$FLzaq&7(!x3X%YjoNMKm9M1}RqvL`2#`pQ%|XF2He+D382@&i$N79Pb~f8v zCRz6cF_HOJoCoKi0bsDWJPNS9c_tkz=lH%dRVzz!Zl|b!?XA*!6+e+r{Y`H0M$j<0 zR!9)*b#{7K4A|%@KLXe``oXns?NZ#3V=gQ~cI0ch)#;k_lTVzZ1E02p-rDwTy~_6W zHA%l+eAoB8sev|_)7!`kn!;v2kFdQO9eFe-vn1d-T53-SNcSN4q^v;D2MoW`hH!^x zP9E_p9{op?(9SeD%k<)=qY2dkVvG>{+K*ROL;2{VPKgcolj!)xDT#@mn}pt&w8$tH z04=41u*;rRR~CQs+<>)$CLN~ZIv1Fb+Lf% zE6ikjTXFEC!@0HqZGQ0jKVAUgc1u~se*tF|H^^{${mbToT3Pb$dBG<(#`-UMWQhBF zRd<7kE^@FPmreDIIt~MLtYH`peJ9(TKr_a~Nc`mpfhQ(pJG?hi*&?p=RFA$Z#FXl{ zj%~G}?XY>Hir7p*gvy-!UD4M)MAV4Z4%^?$_w}5rAKp*`#Sqz^a6s#gw%qcXw;|bC zdbQn|(R?#eg1K6Mm>=CP-0NJ>CpUCCU}%}!QAJHhNvBk zMFJz!W*OrrSUbc#RxS8tBg=72)=L%85l6~WnJEXuK(pPsrF_t z9kUhOQ(tVO`&G79zg%`jw3L5r$Ho(D&71d*gHK~QzO?>>35xO&SD+gJkkaW&P27CL zdFl4#0vW%?D)I`6(Cp((-3Iz~{-VSh%~RSBCDcmG(XO;=rCev1UX&~gNKt{~vkjz9fTFRj{-e5ucXWWDl@!S?DZbhP;vFdUr?=Wyxuv~88_tyCP}Rq=KW zeiQT*-z_&Jx_KMU-{N6pnpB<{++An@NqSZNO_L*5$6N6RXvn*T*5{{;W;2G7%5VSi z>^5b}y6)tQCCkYI`By5_j-yGi;ITLTXfL;I%`nv+DqBsfFe8PQj)bJDMDT{15zbSyOaDjGIoE9mG@|@yOfPgS0x($67K%2NUCYR6u1? zgzcdJ)V;{+KHH%sl?-<)sK&(azs^^caP1#vQwP2{#a3Pr;ucqa^7+~l$1w#Y&Q0BF zQIxtJDj3`s^&ERu(`+mh&1;${7;pG?ZQ1-O=&vp^5ixd`colEyy{yCi@}j7IQ|s*P z3aRxnqFwBXbAr>w@w zwMQK-wrrBGTPSReBSKqGKECs}TXOg--LlTTRnwez9S zxsmTkMti7SgVdsjE{&{;-=j>~3&NO0z4j%l^W$}iaXn=kk6=Pv%=3<3)1*y9ZVH|z)a z31Myt^B~;O{MS|JGtr%yvfmIwfQyXMspCe=U7|0)*o&oY!>^>bG7tOJi0XdPu&mYN z^ENcv8zA;6OyCOCNZ&P*x^{&fNW>IpLx{hom_K_X3mEC|ZMX*CRtEOW>NIy&7T^_{ zO4wUdzx!B_@Hx48F6pK`rsPw0$!I2&8M1c=Wjrb9QKm4;af6(5F0&RZLIoP9;aIeQ z7ZTROX<+v)PuOcKgQ&J2#H4EQK0I63M%8B8&mLPT@Bvi24JuTyis950a_7xohdh9G z)bO%?q4BJ`G;`nx0HU$BJew>T_~rP~U)5p}VytS603FoA*IoP?LPN4qTjwGRi5_bh zUkxs%lXsIO<&?wdRI`{gqbHlZm!IY>MHkxhUGY@@3FW_UEy%nXZow>r*l^c4DBl`fcrH z63NwmPi5@l?#^WiS`hZE{oT|KGAp4`aX^x-Ob$fH^UbAALw``RHV~-<(7r7 zrpj5Yn(Mb)RVtFX`evxLc-~W+ZjN(t@INIZz~y^lJzCF_s?IBWr{oo-HNOK=sjCk-)xa2wHK%9+9y8(B>Knbp*`9GM?sWhlz<{gcQ%=>s&pw5zSEfwv8yxGPZ184FV-SC3XxU|BV)tn|^4q;DRhaC1m1Ax)%{ z=uULg!aBGht@+okeQ083e2+2K*s1hHqJ++r=5*X?O}TE#Z=(v`1z`8N%D}ymnl8rB zMQ~hSF+&dSzBrC3Atq<7;xvnczaWq1Xu%J~wGb|BK#^Vo6$c<4c=%!=$Bu7FkHima zI`(Szowrw9_O6}P?dHWdqokH^*0@vCR?I=%FS8`{=@R@qM_DYrW>U>q{{~+psLbH? zdr2T`QnPy<7w=_+D&T=N8XVdXb$->>er#rsTB+8ZuZ{2QUQPD*JIf7{HlzTk^lCvl`Bk0>1ZKowOH;~BBAx$kbdGc zr9|_~)*ugCTa8e*T7uSU#TkC9ZvH?*>?jOEa>seFsdR91(bVFNtkNuTAveVf%xLO_ znQgqRlHh_)>q3}A`Oxf{d2x0Mmw#^Z%pqVmRX6H<;2N+q2FxqXq<#JQ+1csXUd%N} zQ6jsGkqG>i>W!h6s<&88Bx~&mlCB&tBmtY(aTy5JFl#F6jcIBkq<8gsj&m#7idNG> zZY+_n^D6-YnqN#gYj6v1DmN8C!)eNS#^pCFNxM~w4j3dBR0qJ1zk|`fef!M!DuU<2 zrNQ?5w|HtVccutEGYs?+dP|LAp~C8kH^iv1Q@EAnTf~;fw$lIP@wn3)LDStWE&gUM?iq;Fw<`bVLXya(>yYhWr&si4cC+h1*CDiX52{G-0e(=5V?gsdWWl-IMh zGcoK%Yul}Npwoi0TXR(ZfF0+N0>(D$2|yC_y4yHSd8?P}GVT~u0iebQCE;Rewvq%P z;(@4PHxXH<0H84v71}^u+wQjLwrTjU@31Drs}M)q&|-N)<^+hS_PxoNsV1_vNvR|# zjD`46rMD=&9EfR{XD&S(i&CU`PTR|4@)gHHw;@E3e*vdN1V^ZHRO8Xr8DAb9_&WEzeulfT$|ing?{+cQ}Uc zic`=!|IHt%$~k)>2pys!O#hK*`z=SjXN+U}u9~VkXC6lu-(w2Jqw7&_TI4XQ}^u+)vWxAixla_k&Je|CzFUQOe!(4l-=nrZm8O>@fVK1%nTjSxPEVOtCz#Vq*K{Hh&D$xy0H{C?|H&idejK z`%fA3cW36G8}pco$1Tyf)ZhZ2uei(VW+~LLAZ#|uGsY*coeYauqx=2kdLm`5HdR-m zr?P?$FE9FfPFy|*qbstI#(TzhFUT9>s`8st-!PR#d3R}0HwT9jOiwez?Yai1JXhRM zA1nh@%0&+dQ$3&*Y8AJSMfajkq8;&&VTjKUcQ8Y7r`V*WVRSvn3|7Lxy-)|}#xJKV z4P||Yr!!(1mSw2;5~{N&et(=Sx60^lIa0Q>!lK;! zT%YMHF^PQCm1vP}ndjURT2_O1sdz>vBIL`Q9_9%T-cGxjptY%BdDhmth{`i&$Q!EAcg?wU?hp8}ql%arC40GQ-t>@Lto8bm{4?9g7OOR(SxX*Si&CVNrA!3;BDZiA^GG@TY&)(XA72V`VwlWih$2y)Wz%hha_<&D0?oc? zMpwd}L~E4T$}+k8u1GQTD2eaSX|vDMgCS=7>tY|`3ReoudIb_RkA%6iZjo7hkxPeP zhOh^s_?EAR<9#i&jDsBK1uCZDfF=O&5K$$&uAe@Z!laVzy3R>VY#D-HvXVVsqoDjz zG2T7<%kc5*f32y3_IHvYHGP;eJDZ_Eva7@=w`VEbW&gdo&e85^HtSpUvZg)U8?=|- z>Od~03B8VmF`*>si9hF%+{_$(fqi^!Ex(;Cr&B8@+KE5?Z8j*{6ZR-87YyTw?~qEU zj4kr#o~|$PQ}i6^Uadr%?HmKaNx5ZNc}wU0lR+mKwc<*y*J841$}-K_M)hi}W3UeS zR(+r!*Ca1Q39Eq-euT2ToVp}-R|Q7^j7GNLg{vaWMS875S;2e9l!+rRpVWJp0eq(n z@7?8L$VcT5k}Xc7ZwKW^^L z<7vuy--5Zn$Jg?vsv~ea4e1wH{_Mrmm!Y`E5N?b8 zz@;Gh80v5Tfomm!tYzQsg5l*brWj){WQiv1dft|SFN5a&OYm6k&#Mm64`}PP@1cC- z#PVB&>2KUnSVluDYlJ|+!r$tAm4K4{iLR92rrkwDo^^()^JtzV%FWF;#4-i@830Yw zA5iiXf4!6SXjc-~M&V!42oPU3P9VLDdzpg`8pNA@bEBmr%#bQB!GRKqQr}FK)vJxA zSz|=3MZ`RfOCBcj#YE$p6JP}(mO4X0D3#*j$oYRRsmi7bX9_ms#wKsI?FYRxpGn_u zg|w-AKuwAyJiRyno!k5OUr>8y`9BNluM;ZQmF-K?t3{?n3K(~UOtb;NZyqRJL&`8D zoHo*PI-*)b#29j-*?+pbaE&SyFv>`DAnUkdr*Simp{LsaW7|Brl7jn!u|4YO1W}oC zcDh*TIF$6hKNb*2RKqWh>C62bBl-QVUwpMvFegj)yT|&IpAhlJ5@F`)CmB!E_s><< z=^mfajuS0D#h+Z8Z)P3Ng?~qxe)_#X?FGQY<0zGV9G~M{g^KhrT14T*ucP;ga-@EC za7ldmx!z&$K>y>ns)39k*J($u_t|iR+Z%Ot=w`OnCiZ7~TS}XOtG#^YnDYIJ8KKj4 z1ygA(V-R7t`+9s{MI%df`uYcIbWApX0KL{7_ zi~yuXff$j$9pOM*ws4UdNElM1Uj#XC-PN~GGSeiie9HBfDE*MHp^PgQMM&|K#Hev5 zM-C>;=FiDh=V$q2Lsg?q**r&7dOWF98rl=fH8jU46Cmi~=jgml+LMsfVGAJ=t~39u zslehm)X9ni;a-hMc$E1gSAW}yNPIr5o5(UNrz3ot-|pyLe}}!4yd}asrp8~%%I7IBWHIb|JxA(w%C8eqSM4rJISW#>Y2*9~I5%%RIR%jRENX>? zmrDvtiOX>c&IRk@&*q*7eOsvmMvn?)zwpMQS&9K?p!eUFBa%Rg6H^WuJMi@p^CGLW zg*<>UUKnvcdF7>e^LD4#sDy+r%?}^Qdzwh@N4|Ibe?~kS^@g6wDer6VqpE@L69Zf< z-Bs~8m?Cf!}X^s zrmFA19tTg}(B5|Q3?4bGg&yBpb;Hd_U(#of1`&pAx zTS`9;OV^@3vVc%^njKnIRkd@*5NNq_NTKlOY5$Y?0r~bT*0h^;cfVSG>!R^@yxKx& z86Qn!R{m6gyU|@Y@jOzskoKNVNw`~d)253}s4(bLvF2_pyX|q&9!!c11XIzt$x5Id z)=pE;i6(}|I;&A-!Nbt}?bqZqZtvN1Sh6m&+TIn7xB%W)RcuW8hlU6dkAcO%E6xev>>7!i8eBrK471=P(>h#G{*K zsFmp6cXzYcEhQfrXKrg$_!&5{$Yml%Ir`J;7;e?J(=87KWYHn2*tWgrmrofJdsYz$QbMI4lr~~s+4wwu&MBBE_D4?8mDHzp}nC4J@Ynn9&x6d!I_87+_F>Te_vu^xT zX(*_o*0I2F_uK_75F3zM$fD(P{{WaYXZux)orzfb_lSz6`Q?OXIua09U0ZEoz&DhyW)oU%L#*zV!FW}vKepSB~a}=Q5ed+3G zBDnzx@X0I#G{Ade_mf9&;+QGKNRtze9x0s8Klq6-IcPTX6CzEck*_Z!?-zn_Wg zdJ_O8YP$!QUYu49XY47JvtS;w_$#BdAZwVB^;{Z|_S<=ks7`TDCy&2?KnQ^LV~frH zZFdPA!~flw|9TNR9!^B@khU5SmV9~fMJrDbC7DGPMtAH+ zlgbvla9?HT6|J`SUOrM5wh*23P1A3z|4>^n(=9CbX;1x&1Z|*`?}1-gJp=p`f?BRa z0ROol#nZjfj48u>Eg4On0FKjddz(pZUtOR?9TTRlJIQKr?+5I7 z`JXn9PY^v~vQojKyq4^sqw0KC-!eY|o;1Z9y6yJMN)N(zok`Bdq{ySoCD)`s(p8(y za)t-=?G+Aqr(~RzCsb7`DtWT&Pnogw$I@9fuT}9$S_I>Q_8QT(pZfn5rT$OIS{Vq5 zkjtVorc`5#kesF4toL`^JyIIZeE{Wq3&m0P<`fCF|J+d6MrR} zWjVGb(kc0M+lgnkRer8_pl~p9Z^Zjr0%?`5XSL#AokZeHqO3Yd&&rRqM3?DvyQ`QC#hj%jF{8nnzRgd{LD%=2RWj(Le)p>aA9DeaUV;Dv$ontv zGZvm01fG5RdL{LAnZC0en6y=(1LjWs@Z(I`aad}-elWkGE$|b?YT(S-)2k{A9eWAI zTz3IaZsVqBn@g+7`7i3Qc}KIyS%HE8(dBR|v(9>|1j#FmNe$6)_lU!Y2qX<>tbMwT z;kIf@**PkzuA=Fy>ndL|I~u)C42eTkK7RzUE=d~3l9E|vT-myN*dKaMlKAiG|F0J- zNFtPQw1H%dUY=W&KS~@Zb^8oIZR1`>lXC1g;Iw>Hzr$o!f`Q=9JyOA;>f=G0U6eL4sgJ@&_$ zp7E0thgqve_pU?#mcO;Dl zXYP;hZ5JIEJd(327Z!3-6uvybV5`)cD>7)nH)?|Rk-NoGWx4=W7N!(sT|)SY)<#CJ z#spL8n7hj7^;TjK2TH+%f@%tjhOzHro7dt9xFe_(&FpNB!sqHwQ>o4&jh)@ho-MX> z>;%T*?E9M31Iqieeul_N;moh5lfob=3`{)Q?nWyD=@xVp$aL_cyhyrjT@e^B%`g-E zmoVX9>+_$J@d7X zbF6v%h(shGGL+QKc%2I^;=R|UIG00kA1})6y|!erUxk7+8O+8wZ=mf(?iit#;bm>z zg7$jLS@PrcqDq1Mt!1CI7HEF0Lid7KO7%xy3qw~P5X@d5rlcC-{w`;nSXu8w<#Nk$ z$r}HYY5bqN6xee?#l&smf&G4#dH^7Pvc7jSx0n%&;wBM}hSGv`gd$kU8IbR4=*Y8M z8%sz-80X0XK#ot4z_iQR>8Vw$xqS9HvWHtLot7m@fi~iGI#5Xj5reG@}?c>Q;4h>x9AXeZDG9<$c6$iNQ8>FrM~$rY?~ zKX|_vnPZ@p(ZG9MwCC0f7#W{)I@!L{f}@q?LJJc*gsYlBV^U;6bbdDP zDbHTBK^pH>!*?teQyZi|)a)UOGD{ItrO;woo0EGl|F_VqX_BN3p7QdJ`Rtw++A+|M zsrPxxB79Bj&Pq7X>Sxb#zKgFFu{5Y_Mt5A)Z%=T3P%{eTcN0FA&fY9IIEh{%Z_YM* z{c$~Ws>6CGfv0#VuLg|u*-5xZo>Zc3=TW3@tCfS|%^w*35KW;ZZ@It}czHMxRM%ne zDo-kb%*$!IiQSx6p$4ctv)gl@ujF*nCCSGU(4Im1<8z<-l|3SYEtTE-(s29} z@DrtA_q%~3_O(k2Jx4$OK9p6v+jAMLv6`TNV1`C>Kx@=tmfVVdq)v>(!#JHEJfdwBn^F7*EiN)=V-S_{xMi8Hg&HAP)y zKNTaD_9bFA`tf^{S?K4=0;LRGZ>eC+5sLIzG(!!37yH*-dcyxJhBcW<1Xe7A5Y!K? zWT2e8#d1KaV*3T$dF!MbD}zL0V%*o^my7c~tX(WK50j-pwFo}+qh%`$Uw5ix1yT?j z1zCz=@g4Gnd!DF$VbN~}tkBimq7r-uOA_QstB)$WD5}#MmR3d?ihmxxf^f3std}Hu zvJn1bpyc1tezll!g7Qa5*l)mUd=(hNahKyi?*f5n{j=`L$P&@ zWC-qZz;*|Df4+)Ds`UBJ4_$n0J*wvNP6zvNTkY;Hvk5HuNYfAZ<@JQH3w<_}v3>K#Gy?XoIIB^iaQ|H#aW)MIf#N9ZSS&EgHT&yRl zn&ak}gbq;rK7qAX2`>fpzzm`hHwoTPuSLi1LUKw;msvgaGkqBS&`Mj?qIC&2+zHc_8j{x@?8HSW7`NCu3Syx-lPPP!Y|`HHOVO=xl- z;AHT1d^}kzf$LH0beQT?*fp02doIQ%kGv-cmY3X$2sS4E+qIgG<%a0N%N5kX2ERsm zG1CUEpHu&#yc^f^leD#=wywL$*y>kDx8T(|8Y=JERS|X3aGemo-!D74=K*7gyu72y zNy-pb^NXC4cAL8oLFIJQGs%3m_blya4h~LSZ0N{??AEGxec|{X<4ispaStI4T?yvh zdr_ta3)IDKdnd}e9ucnc=%}3n1Essx1ya&Ms4&w{+Y1_Ru;XL~WYs8s13H;SP~@Tz zce-sBSWh%*G!K^DOPu{0f-14F5QMr0+)$SGN!@Fo3CiITwWML16g^WxPZA$mWOJp< z@G0Z=`pfVaZ*45y?q{j`FzqBV}82^P<{De4w{UJOKu0v!~0ZR)J0ryqNha%so4tw&c& zH6t%P!59h_I({6mc;=De&QrE(Fs6R{#mlIhc}2V?2b-uZ-Vjl(+}96G%{YMtz)^Y4m*xOuB$bGwC@m%<5Ieq|XM7gfCR zb$gXtzj$bVBH3*7!!hAB2Y1A3eO~$UcdqfbI>m7u8p(t|c%in7zh@;6_Cn*#8koys zw=D-oQMRqW1Sd-mBS|eE{VCzoLIqfBE;v)%wE#r_?bf!c2!miZE|*(d@WJ-4Qnaqj zxGnTocl)}3KjZdU>vLmA$F;FiyLW^ECHbWv4*aMz3)Tg7GfpP%W%|$BV*+|XWAbBsQ)@%gt&-lp zivn>w>d~advAP>qtrQlQv)sxAug8=~L+XT0j+nCZyL0uF+#=9~PVL-+=k6A%0jmYr z>lV#quupxl&}EL@v=~x#e$9&f*fej>dpVxvQ`r3H$enVjWcJftcpi1RJPbBG-v`f@KcYsSJKp`Za$fm_&n~xnc=Q| zQiu&$^&Uu*YSDO3BQlR0PVkYq*hPB8cduXLVc^vskCO{~mlML*^9&eMMl#NR4yOkM zPB(*^)<=YkU{&4CE?b-a2Tg-#uKVdXi2q=B;jcnv41`_oJ=rYy^ZE*z*#&E*CkH^N za2mt^iSG{fA#HTfB0XV|(~{Na&)|RLhyPut*4BF~BZI>O@StwoyMaG%nwgN%rh9qU zXFsoVy!K4kV6+Ylz52;=aX(#cPx=!!B2P$_j+uxuP(IZyGCK-ciQna_#igIA-3?$; zk)`Caxw7a$Coz*cGf6+9QPI~GRpfQ{W6kAPClF_bJli_gaK6{9{VQ0xzZXR3fU>rv%Xxf55@Fn+0cY+6W8rw@f-WE4>reYXBAV+&!w60rX8uHVKJ&J!MnMF&b5zAxH zC(iIAV-6IR%JI`jN2gFJYWH^bG)2DZ0EAgD!VET~kd4E}gf6FFOFh!Gz$IjMiok^M zlBOuK7pV;b;tdR@by6UO?Yt8?-y2m=Pey32nSi$TMp^fW1ar`(|DT85$0q-~rt2%9 z1LzNwCkbaI*=_kNKrjtmI9!v;b{I4MVKFze@Wwv7aiPJv;(M9;rN~~PiIF}(b$Eot zLH`ONxZHE=?#UI^sr%iNl!g$&YJA4m(iXXWw|9r@zbtb+y;KP@ytOzv@y^r_7Tt#1 z@02F&;bZ7Kn@2K+48PCES)Z6&js7Sv`>>w>i7N}{geqOncJ_9(8X!Lg0|3MJCa>Gy z**uj8*$GL-oggClKp=^`AKOME<+-E>{llm!vEw1A(INplfe<$ei2)AoDM&l!ssU>B|p{Kky03WKAq@-*paV7x;e zJSLyTF2}Un%zEUEfkh#=I(@PrHD<^JtZDSbZOG0I^qqKL$5jJqz8Qd1d;2F^jH`Wn zOxOiJlB$GG^{DDEeWnCofLP@jm>q43(`>T_GV-0S!DtP51(0T0!RXxQ`!>|lCZHF- zFIA4d_$iqAjwZcx=lXj@Ee8JHGL}bkF|3Dj|=I4 zK?*Ztss)G!_Rz81>AF%C8><;GKzH_1;?Pd|yqP80&zZkcD0~Q>2t( zJg~*KnNQhs&ENb!xz4oyG(GSE<@{{)T%On(*lca_3|@{7xm4$KaW$Ta-lqOgP;Hkb zw1`65l+{+uKa>i?iV}Yi#>7x|PeS*CdTV?KiC$)s>I9V-I;mC zcWmnwa7xqf;a>^qi?_(}a?UOWCdG|*?9QXp7>A0TkQ zxhH;cwO%czrq5e&tojmg6@UMzR?=+;Ox6|GZ)|Q-J)KPUrzfJL4ejHbRW{u20bl>` zmqgObZ#D0}%4-&V74y$uk1_B(tT^NTem2O+M@LIU*UK9c9igpz=}B|1fUch1K3&KCj%8lw9O`x~!{MDv*N+U#oFC@I!`os3JxppSDt&Qb5iKCflmWES|Wp*hfx zQeVX`WVDTA7gv)!Ma6!^2gTCAX^>r#j7I!kFU3Ys%q=rhXGM(>F$zF zNhwK@5=A(4cY`1u(%qecgm92f5fI^kbc28(Eg;?ft>eAd>-)aH|2M`Sj6E3Vz}{=k zHRm&*S-l^toE&%vj~X#sXI#jhpHr#^xHc%ykgroIzfHS8$^Ah?r(T$=ZZpyFdikqp zrPH!!K4!$esVPZm)L=I22Jv&_?HNlpTiY}9*_3+E?Th`BuPF?+EAkh@nZ42y$kMqZ zs1=B0U7^=kIt4MBOL$;Seq9zM{?J-U{_r07hyJ+2?|<*qs2M3z$yp?O-fxBzi%_cB zpa%Jm&w$TMK5iv`hGH&C@b}ilDzrr;gxsT=`)+S;XPE_GR|!$X4tf+7>`piCr-W3< ze2ha2!fTANG`6p)@J6W@-+1F4Ou3X=4_7xo4XJ8j7stZ;G*ozGZL2v$u2kI7+zf%i z?3OBhY_2WeVB2Oux&y7oK3UL}icA&Hiiayz0nD+b5ZmNK!{b9INvl&6vO&jI+erJ%|2tS%(A;>n(^V%v_Q@P+%gOb8S?oNU3kI zOsweo;XxIT5^72DCpT_91Wl@Bec#&aY%cYlVUFY&yh$5!pUm}r*LzGcj-s~9n!Ye1 zngmYkEgrM!2lfbnP(GHwWf9x|8D2N8p(W_Woj9KT z=VF%Obx+sB%z;e61#t^fdjI!VNNI8%%J`h<_%LypUL$b6o{|6wQyf@rZoI<#Nx)*DP zL2R2&T?Aj(qckE|W@>5w=4*^M%#OKL)8o^7B_s!Y#^3K{rOk4H7JMf+qJp!m;e&Aq~T98poDH+7wj z*#GAl&UHi5+_aglR~H#HvVP+Cf=y`rU(KL_JVJRnGrr8H3G`>6ACW$@m-A~G)W280 zTY>uHyY?IQerRD;xTs1l zWba^SEUEPJRji*$MS#Vi>awx)Pvhhaf9d~b>0KQU>4OLIQ) zB<>RwYCqUB%@=+6NEeTnkN?%+7i6iZ{+wbJovsFbMvIccv}=lJ9gqYluwS*!Zp)** zq(ZT8bGwIbbGpx1NYnIG>!tB)l5o1fH@4EeK@elpGF`mMAu}!Nm5bbF zqD^XC9vT&oK1TdJTRhe@i4fO@0irEcMulQ2Q9KOO+O;mT^_tx;`_<7tHadeEljn8r zPYEBXCzCvX|0!G8A&+ywvNywiX4Pu)-Db{R$nYIocgF-#&uihMs?=%G3RlvlK|x|F zQ?BUQ9k0RA9r1auP3@6azt&3;$)=b;ftQZ%P5wVe6=7%A^!HNybrXam;gvL3UD&M= zRB0Xc4A+29k|0KB8hZqVTKAuYc75#k7{UWq{8)~fHL%);s-|^Kf|u~uHZVm?2&da< z8K~-%mM|dE4(Xw!=5`^oD)rCgiJw`>Sq>{D=ibGM3`?y#uMlRu{K67ZaR=ufY`8-$ zbB)y=iJbH2-fK!cqO{^}x1mCkl{7je>K3kngS4X;k2R+aejaA15GQYk?5A=o%|FB= z;FdMM!`inoku#n%s$Ac6-6%@|d##!7PJG`wLpag+{VY1NvU;!wV|!Jj``!!h&7-3- z&goiD>&dI(?X;ZIqqW{Rv*t+us+vJa6~yt|K`z$HrPrwKrp_;=7luG4)m)ITdusjU zw>;HwB0-60+uI%SXl*|_TkD_bR}=gte}BDxCV+uK)mEkPymc?+#;0Q3myMo>5(bv; zVOjP4$5KBfMS|xyk6?2KEX&Y;dK38frr=ArUAyiJ)T-o&-5yt^fo~lbQfP@Hig6wd z9*!nSG>RT}8MWFcL0SWv`H@Vzw_v?`MzUAF3d!|J3s2m)9WJCoUqi7}d~0mF_4w;c z_W_(SNZR?_Myq!Acx8Tne+t7^z8p#EOOl5nIQnLh-_EN=9;PF7pI94zDKTETYVPh8 zuIJW%NKS%o#C*Qggqq)-A9Xl;`O#1OVBeejp7nf8Mz;>Xx;|{RKdy2}+;)WX$X#~C zxT0!8Z1SBsE$3#A6YRD5G)!K6tblSd4bReZdwrfZY?#padwQ#VB2A2w=2E`>FLPYO zfG87huQ0u#@VDkGNkGVBMJ+Emn6n)FbG6D;{3IlswKbMYC(OUiQqs4v_z~aZi0G(O zee1`|=T33EwNt_r+c4 zzK;delPfRweXu6#D3ojx>@n1__M*5AuQl7LSbK-3^KHb2NmdMGXMDu?^)?3w%5-}& zDndm**sXk75}qJtCCmSIZ{Ey%Ykhvyo2#fsHJEeSRPWFh<{Zc0V~|+g@+H3|Yci$S z+~gpDU=RnWj*&xK_nI$!Wc3tfnCH2eJ+X4W>uY1@xk?{G)}8pp{<0rss3tP;lQ)RA z-b7&{t-7&oc*8+4q8u8{sUnUN{8JA?;(Lh10^96-(vqn%`aUW_ z7&esx%KXmjHZ#{U9PQN1?4&RJ8GLp z*rTaWI|~CqLO6yg&P1wP_xt*zxJm*3J9vED`YpuOVeK|hbw*L&4#D;Pv!0RUFyHS| zjfJW0@xqDKo)UUuI72C|VPijfw-HyW``*up4zMg85OX@4ii~}-p*%7lZB5vTa#X!n znXqT5%wKotC0KN?=cyRG&w=fXC|6}lSjmX37R7muP(q!DtEhMLX63@dPEjV&=CU=f zuG|{@b%v&qo6Ys+s#*76SmSA2{CO(>yswouINeQNtonm}2F8?M5SsD!-9V&3O9r}ti#Mcs;KZxqS<4`18k+V16pS=ZBMHR2_6uDPI2OjJg`Shnfqy0x&$zOor_hwm$ zA{iGcM&uDj#0C;GcNf|{QJO4WaQr0*Ia$%K3&(S?VVwK{y3w?+_{^G z8G{DLl$aNz*nBv5#CAm7HeHiqTYkc=yZ%C+q6ebJ1agJ)p(I;Iks)3Kw%bP z(E0Jw*+YxI{Ox-D;r;#o4ch%Sy>B2jqyx=~P*1SFj*S^n9kIsU&9<6fjxa{M39;GF z?u9hKob9MJYC6WFOZQ9_0<#LhtiYKESRd$Ut_Rkn6cwOqCj|_~xpwbhs?39Tovpsi ze}A@XfWf}a}~ek zy?mA4(!noR1AKT0d@qvs_GB6lQ=PQS#H$^jQMN>)(%`MlVGD5+AzLLHD~`!PS9=q> zT>7>NET{8rt#d>o_gi$l?Yz(Y6(&>a-5vS`PsJ=VTqBEWAEm+i#IaN_bb4_)-z5vFZAku#+a)hA_2iefLq}nX_d@^vuH^rrTK!NiW&Mt> zlYcT3bx+*qlG=QwdL_C`?RihvtzE@)x8z#W>JIIY7^+)bluA7VCN24k*R?OnuTM-A zR`VT~M@k7OSI*CR3N)Llljl-KqNLe@vaMl6loUKT*#ZT3HvM*Eb^rOV+w757b84mV zHyo^t(!3n7iwsa+V4r{=Y^`GEO>Uj&F4$GnhmVM*y*mUL<7u)8#Z zd!A+=A0!oE*pAlMW^pKx@N#x=e~Jgrq0+ALV$K69{p@SzyLc@&EiI7+wwal9CcLuz zcl8HiK)WRl70+3^zWGw%^z02`<*~1tT=5Qg!T44`i=0+@N@NR=;Z1VC7l3^^FC>@b zg!ML$nyqvT6wYcy?oUvyCDuAe(3?dzZ12dZ>nFzj;u?fd!P*L~m)ifKYkw=!Z^*o@RB$<)e315Z{!hL2_x)35WJrBQ=y*g~gPGuVp0<;; zs-`48UR`Qia3h4$vUEv?qlL|st90qMP^gz;>~npq^H<0`dY495ikxv7`&R-lCSa22 zN=db@X?V0%Fj%AksuWY8sz7P;k^Rbsf_cgb`G`&##i}2Ta$YN&D#2u#h~oE*Gwu8*pbyC6PaGxhrMUUlQ~3`jn}h~ zyInNl)8wi@GJsttLB$~EqLy7A-FeG1hwSppr2QW+2-oBr$n368VSi zn3gRVOoD>5YGHOT&6(GFv|dgxx$O&VqjyBP$xA(BLcgh*%DhRp)OgErK%@$0W-2r- zD8FW%V4b=nPL+QikJ+g^)?Z>w%l9FfvK4jTYhH6OjL7h7CRB+mfx??AIi(z`ThH%) ztk<|TYcOm^@5&_!hb@U22)?5V3({q?2TyPQ?{^2U53bmlHjFb4Ch-ge&Z=gb?tf`S z_&NW-t_L}7pv_al6K(qnvv@}T-P@{Y!~UR{{OuQB{})(89IvMT+c*z3;oK~zHxu|imQT_kJQ0}W#sADMvM z0Jr{J&uYfKv)Q{1)6Pb=6ZQO856@aC=5G$4jXsB=%GQWpJhNsXl8Ypcdz%Eei~dfy zv8N@OhvW5szh?~FEG`d{z`4u~X>pk}z{3#ZB2g`VG+SGaD*{{L*wU&iN& zLR$wHilL%cg36eYDBV-$F~zCH+X9+LN<#q%unyiRRKH6|Su@eW>E2mYO$=Aa-s%2? zuh<(b@6^+f<6;HP>0X{5=d8x*%&Hzu%9TpVp3znkTi#=~Y9Yrcp$zITlP5=2f{voP z+1xz7eFs%SWE$s1P|oN2rPq{rE8*6O^!tBy~qDrKa|0r`C}&&o+7@-)#nPj?Yex(c zv=h15?yk(pL8ZR&5v9)2t+9@UrRtTE4Zadr;#AQLe9q`lMapjuyD|OdmFk)Nw7bXo zEtY8-k@3;hg;TN_TV%xk8L_73lGP%e_+8e|aZdABF@IJ9%p*0 z;yLHqx?Ol86m_oktuFVDmWLHoxebeLIn8eg$ye8NJ1O$6P_FW3;9U))%tJ z)$XyYu@LozEw9=;EKe;j?=K{!WTTe@JQu-M^%x{!P(U00 zH$*Us9E#L;^EkEaeucft{|e?K16bXUw}cm`oBB7Try4>N3%!TLXtN>2z>l+HicF(t62IL3kg?NU=8( zwEiIJx={`)S;I_=8z*a!!0?`1wHK~>qk-x*Lp;c+g(;^I>ghU{IZSi=hS{d(ntv6p zRGh}%8RFDWF2u;A<$wM4A+r8XZTZ7MyX_@~lY!B8y7zumy|^xCJ3)L)FZ8ds6fJ-(2?`?koewyyMBTroQ?}kK?w?3s{htLHDU%IUZF(@A#S` zk)~?4D^mu;vtAZ^cg?{N-Cp78!jFf1*|)ceB%vpp9LgT%THj9NO2<)WC!npIdvCYN zmQ3xk$!7Wo*W+-sZTf?nY#Sz?)V#_VvzB(fq8^Cs2{eQtfw44V@BA@ z9{oBDc_$!&F43zH9ip4diemHzs$tn#v+{oZl%Y6{+u>2x~6J^JC2{w z>-mXaH~j|Yzi%w$kf+GQ@nS(K5O<~{jKnaQ#H_gcE2zAjT*!JghdS?dpp@#qnkKb2 zko7s7_5D;agr#)gnF2e(YTEr*^}J+iXTIbY2RS$7*LyL(YV@}-pMW!KitKUtBkn;g zp)oyCBKTj$9!SSuVJgUoxMU?D8 zpE7;ZQ1^^Pe?x4_ed7rgD{9yw+d7|QdLmUbIwgpGVS<`;=^42w`&7TQa=z15g(=NG zQZCC$ui>m}6?Lf!mSIocN2LmBxh^Z4AP(uD7cV-L6KfZWAUAUtxnJO+e6Uw>w((Cu zVJoQ(l=2%9)b6(I#%Sk=c^f@oySgRc1=&uo0mA#FrirfX{w7T+PxkmSbG zkE`oec-tS!Wuh9)cJB?kY#_tUMZQ>l!%j7OquEF+(U3{484@|3zG)IO|IUY3IS~g+ zgJ0|AZZsEJdlFfia9bk|t|(W<45Pwwp3s2IeNW}b83ZgXuScB+G3%|^g{K8cKb)+4 z%4*`R696O|YO#)Z&??d58P8Gb{I_1Wut&z`t6Y6KB7a{7N1dVn2BhI*f$^oNpngPh zwIUjnAXR{g-zz%SC0Nlh10n}~6^A3EqC{?8O64DqwK`EMQqgk?k|9Q6;Uv$oY_b5j zP`izIq2E1$c{ZQIc|BQW{el|@^VuKUlgo(u-7j{q#7p)?Y|eC3=QQeA`^A@DL)3CD z6xH$n+DWjhXpCSsusFRbc#N%ZT!Sbw0S+55czywhg$nEko^(Ug_4%eVGDKR zb7_^G0Rz$zt3W+J$d{K+uz~!-5=X&qYhATU@lwk3Z2Az1cpi(hedKs<)|zmSHlWL{xQh@9!S#R?Rwrv}slk~*`z>|rhn1T{AI z!@Tq#Q{;JXftG{|Kx)G`E!NkSvqglmUeq0Y3b-5cVW;I`A7~Y|u-8Rp|6oz$>z#Me zVD(F0`(5qWZw6ybkGPhxFy=k_n`Hcj8prVcU)HEgL)K3a`1lrF`AJYKroUoNzWm=^ zvkGZTo1(#C$;pJ0L{U;Y>2sPAwz=34DAGEJivHa z$(^~6OKM7iFjR7uPHcU{$T03#;8)LrC!mg}o@3B(mRUbP2){&Lq#5@28{JIp!<~8Y zcLjVM$U(S&f_z&=YW9{qPiNQ!OiJ}tnY2N$x|K%%pWE}xi|@qg-nY_s3>fp1&P7FP zj^rnM=n7@%^=ZDjz6iJ>p~Icfef5n%4>X*VI1B;x{B)Et;@77pP^EEqQ2zRvP}>Hl zm$UV9U;ZPN?i?M^YabY7dK`ASyaA-zdbvB8zUN(OAsbLAx?RaQfU)#&w_9z ziUSb}nfDSfsop5RF-6Bg8|K&|m}WFeD-3*6BcCRDZhj3m1Am0^3hzzHkdW&8SfOFN zH&>&2Z?1?&3W=T&@TgDQlw^u}1|8ZvbDAiJot8#FdLF!67R#SuB{%uEjOBjB+uf7{ zqn>1BzMuZA9*pm1!$5g62F^J7Zs6ABATSTvTuYHh*NGV=Qp=?QSi(#L_WrjCQ9$ro zgGZxP{Rq4r{n?7*=|#IBT$4Lt75!+jZ(1*`n04w6dK7Sn;9D@u1Rl16C~5(}yl~2Yp!m;0{Tntc zqVB4^Kk@2PAWlP3WA%@wc*rP41qz(wIbOXcS~0UZ76dhUa2&wSdN8GyW%sQqD1xJj z2|S>XA(&#WJiR;T$=d139gy+B@)-}z>S#7YQ34gG7{p8}xDNEJ<$B927OVt+?T+8$ zLgqJ$+`2Wv#RYt@X;RzvQms@%j~q01^T8)H2lgBnlGAT*B=g+)%rn*1f0K&-m&ibQ zChA;*&JOV}h44VxpQrK(k&eS7h}F9Sr~Q%pO5##y zM(u~of<_&oKW3R(ksQnVv=4scCUC>V>3`VrKsF#i^}A6BFcYspjR}x zZlxb;ZwJoz@d5yrhHYKzoW0e0L&Am@3YQ>{f=nL7r6RK=}xA#K%SCZ z#2=9U*L&mQcH`>fOlvYCjVrdHS1>APLU5jc>MgB5n$(}qF)NICXN(C)Gszf$D|!W_ zY_YVwvvJ{O2cNvxw*_#>6q+GMu%Wt26CBxToX{fm(iaV&h1N31Qqw$*Dwk59FHE~dO&`mLZgjS*(=g%9q0ZP&Ls9N3S#z_(?wG=deVPz6*Znn>zH&1!{Uuw*OkSn2x3)g3^tUA8^DajR%F@MeQif$g~n#CDd0cv zn%+NnuZEaRYw-2kh$`|E6@wo`Zw$+7C!Ua+(cRAO{Kd}y<2r$7>BQ^apRmqIiLigc zCzI=i>}A5xnymI7#~X}D6w)oPpYsq})4bm~R_W?#<=A?)H`}k_Oh{taWIyIVGWp_FEUNC%nH$f5sAS#wd zEz2J|1w7JIStPLUI4w3K zltC(mLpL7S&K%!scb-7tOWc=d1$yeK2ha z;t`2ulNi?Ne)>`uphmMtOoQqyta}}_4|Q5p>C{23u-bcXrTpiE(hfzVnq1|Vc57^%`WJtC zcLB}<&qoroR7!&n8s&PoF?%Q1fw%C;1v&(bn8lZ;%s3T3XDcb+1!7-rP4kiD!wqPe z6tzNNr>v?wfi=aP)r&u&C_k!6SotZG9natDzTa-A4JdR>W}B&$3c-?}|LA$5JQBs= zmBo;vQ&pLV`MyBLJc+hVve<+lWanvBu|$VXV|UV2kb;(;Jge~XRcO;j$rpFwN-#(& z7&ruCS>N6dc2?=mUsoyWWf=Cp*3J6YSWEQdJTA}YeiT-2waqUeM1xw9F8%gDMS&V= zTsqhH=33};(kaW`5M`D~gx6W{w(OCld@Rji=FS3`&J#ZO6bC!gZ7~dPB<#41c0SgB z>f|v!4b@ohr#K4^B!X0f(l4#5+$E6Vn&1SiiFrJSxe=&=68fgmQ<$9jXk8lFDpo=( zuyYNedHTPz?a~=z=N`0s1a@dBIX@4pWAJ+#P+MJ;<0Je&tI~(x=kl~-w;er%E?STl zT0J0~c5C0-La#efvjLMwyUFMH>SSKPIuRsge_`IaZ7D=FeG0U13BzlV!S5vInK&Gc z%kg;@F6$!&#U4Rwi{^-O8f9PQXqK326d9`7UShG_|IY;hp#_t&e43Vc=b_^h5yknf zvFla6I+CS0Ul5T)!}jl@^kma<>?BzDYMx`u z#Uiaq<$b97W2W97&H|9>?2Ns89?(27kP3=P-M;(ZZJ)L-r+}aHw*V_&a8Qeo!tJ#j zeKFhT(Ot0NlxvE}torvq($JT-3?3lC$r z)qU>|R{fpR_ZN37F^c??wctg$lBqbJ}BCO_27#3KpAioDusB@?2@*}N+$^6UmYIvTtd*=x1QWs@*j-{-6XOc^8KJCW=<_;i;l5bn0T zzo9)gE^``H62R0$k=IV@sTy)G$^2K{{=&f*3~e`kW?FK*xLXHIi0UNUAKaJ4!O4Bi zADFywr&!?+d+MJ~530!I?wXxLZRuY^>5)QB;ht{;N&Y*>6}-DiPM=Z8H0@fij=2C{ z=puE5Nb5Fyi(G-2!2ln4$jI#x%+2pdhFn zI76@JPZwCL3>QPFd@Nq(2Xe;b_CPC9AU0l&$J5rIh1N1hb9X@74gO4rc2vU{(c_9j zt(PXwkt8A!wV;$4j!YTGL1rL@@T=|2%!>}7swo9%93 z!4+)15PhlcpwMFM1FXn&El%2!DKI7sw^L)iE(K6z4oBk}DT>WYw;@3T45Ew!ufRCs zT`-rpOCiVS+*0uLdq8z~kc!wDy-n@7I_hM8jF|9G&>b<&^uW}kOF`2VyJ2{nr4))w zA?~Q3VLkgw+JhX2Jq-fomPN_t@==C+GZoWTr9t`8pjUT!i*L3*XbkMJULf|caD4|f zU&P{~FCjnqAY`2Z4XZj?Z`yosJ;XmqLHndteUy-m3`3gaKg*AtfyHVzxPG_i428_|(I}sfs)r(=?~Q-|6p+6* z{?hy54d69NJ`Taz!Ww8^jGo_%K1TFSV*d;Q{tXkBcCgp^wK)HOeKV878naA1;X~Z~*~{3Uf24=;xnVra-fP;ki|-(E#izT6kVk z{hN6XH$?SHYm2`A4vhyhsl|5(DtPIhsX5<!&`jg3W-2g_P`^@sxnU_9UDWfK&CRvC zk4HiIE8Oo6Bkznjc&;MKrvpS=J`GOjU~e(oCi3kOOzytSFh%ie{&=>Uq3YKe9P^A! zwf1x$7JP@Fs{*Hkf1u(KjIBSwiyyy2=u>lBe27J)YwA6m%D!trBnLi2skoHACqml_ zEWxlRo1W|5(>}AH_qrconB3OTN?8c7kNImRWFZtE-7r;JG|4QOJSTNuX(qXu;NgNh%F9gh-_~GUxr)=c@SyVw7tx2$8CmBvLg!l6s%(@?W z5wN2N$V!%;mwR0aY#oMYktmX8H8Flcu8s{RS5rsHQ1@Jc|3k$|BjN$pxf%4~{L4`#SJJEpmG@BB)S)4EAGYeg@J+B9n!T=C zZnsCBcp>oBB!LT%^9?y+eB;1!a(AH;4-~tpH)CTt5?oO`EuDRG!@@Dxs++OfYpw+a z1V>8$!j<2+W!ldV8Dfhd^zHVXTbM_uQ6LFTBeyL+AMJmpt^pJP${GR2R>Si5qSVc} zcs=LIZ|Z78od&#sTXV*DF-a3`A}iAL!Hyzwws^T8;AC{`Tf7OnT%o@LN}a(H*G8u; z^eLasRD}bGNQ8iNn&oDjE;24MhNhS6xS;z~bc4_xRor915Z>*u0HlzUi2Vkcp4(YF z9V?~ILZ=x9?`n44jD4#pFyR#P{7XWc(CYYpY-zGzF^lpwyljRfbVVJ}}EY`OCZecyM_RSnc`l$64pCypT+hq5y$rCg;_!q!>QC^CA10bH1>sbAsA zJfY>?o5T4bx-91w&i2Z;LVH5l2Azj1jk_QOcU8>JCXQjz6l5}%>d{N{*T?l-YnjgH z(7aKftR$4@{oJ&c?t28*AE74;Ltl{nEY)Alxtl9foF5FP9kl|kCK%BDGnNB!E~?qA zzVpm&ACkVknimxwm2`NFEU>yvXX0+ICs?yTP>vmrqdE|Ir0e_9d@$RU`Ctk+ISrHL zIr5XoeuGa_$xBkOOVa<&jsKg~WVufbptU_UEt?36d-)ZvVbWf3vGpGo(!YeRE#eWz zPQ=`W$+(jAEjs~ZY{>;?qL`*RYz$qug`ho#xq*i}71D?LyK!hvWInm;Tgm5`<)?Jx*xUUwo`Z>WD z4~#ES%BQ$KuabVSb1vGeL5K()^$Gap#zc#q|7g2I)S+n;x~JzP0I+X`2m1&ST9IfHK{%;erL{D z^ji71&i;Kf&5kI&Yr96M-7=R}^E>T9?U z=$x7?LhnQ4{U~_&%n-n|Xq;?=lOOpp!Jy?OFLOHt^uNif7+2Ts6xqqN1PD>3=L~tX z0v6>(`(y#3M9s{oVzsVIbOvDe1gqJ3xY~bDazD)ll~&Ik?jWP#8aPrNd22??d zWa4cUtNW~V3wD@09ZQPl`PW9!P~EtLZ^Qe1y@-21$-MQ`9WuyxpK&D{FWK`#l*CvB z-&ZzNRC3In`e zP=|b~T7sc+wk?{rl5lp7l1UM(XfesU9}wHP2o-Mn9P|sQIxP3T9reM(nB4slA!opg zEMsFOZ*L=ivLjH+b)V|0XSzL~Q{U1!0h8?26T@}KO7}PVtv+GqdYeliIEQwZ(#^nW z2PXjMD4Yr6_Tt!#(w_ZZWfrOmdyEp_jBR~Z*6+CapRbOIGDVu6Nz5maABo1`Hx&@7 zfntxnm}(KWAa*=uyccm)D}^OQ4+Xs{lm=e&33A3uMg}BMP8bgeZ$f_%hsiqJ>1djO zd*?HX zp$^@2T#Vat?`Nhhb^{N$Ne(-H4b zB^dbFhi&g}tea~7Fb7Abm(02qHnx_)*pnncTW)CIdgWU6-g44C`EoF2#aRTrv@`5y zxn>h2SsheJS+D}4Q#O(qXY$7iepOj34=Pp3;6L?qL@rgtZD-8qo`r$QMa_$f%%PYHTEbbeU~2h+?=x6cUYB^!b$wA{hv;jW- z3_?!Z)moXyn{HyXDSVN11ufneU+qI zL@@yxF&-QxI`Zhx>h2d-m{!tl}-d{{{) zZ#Hl^7==zUNa5*R^z*pD9h(BtR)r6VCy$Y#s}pZ3Yl^C zqOQ&{bO|McZ7FM=ce?Liqj&evWWkkylH zbNC})ia;ym&K0Sauv4=A-Ru-S4@>+eUBHkzUSX^~4ef={!5v#N3Vmme_U`Hy5SuLXyE0Cr+O0UB&Mogq|KpXI*dgc%5Wu#p70g9G~9 zNqG^3yi|}q=f1@2YadysjeXj~KUwU+Q(3O@|K@ejFEsFOM7gp#mRUEgDqD2$M=Xuh zNT<uRZ5;z@P?;U#+Xie}v(7R@p$6 zbn*P3fg_eQHz+srbDfOKKw|gx#IvJ!u7BDitr&!y`BU;DYN4{O2GJTeMD?I=rXn%& zfiDaNWIp5glQOh1D)T$Ql{d{tYMuP&c0{AJ8v!RjCOH_dX6Qi{(bpf94j^@9%W^us z^cQlMNWzg248jiWN1x+mrIZHr=yxszBa3JFN=?lxqw(E?od6DCRjME~nTRa~)-6w2ce+-LYJ}({; zUKax21C~yIgbJDif(i{^11*d|E}S#IiEYiWY*^RCTL)BJAYz8ftkP5GApGvMKV<+C z+sp%jl!<*=n_F<4!1$C!LTOqL(2n3SaM9CKqE4wEjXb&xis3^Ua~S6kY>t zQbzUrGr+A0RZIq>JVmJ*Sf;Y-^2{E{4}01eE>t?)h^ zmJ?d>qUYg6kv{AmBvGz|kTK+u=+}&rcWHE1^^v?)md;H=-F+xK4E(sNbfUjtIQZ$W z&e-SE z_WH1OvH*z-Ps3FV6ce1F3IC{1mfuX|r0LDsX7pKuW_5J!({~JRiR>vkOa0cJTBiAlr0)|RJ95d=d7Nu^_#zEY1&(Zz_w zi;TX1(oA)&xv$*76!P5)7^apdW~7GSK&XtjoS>Jdn%gv7f))4stGGPFT3Q3XKgu%7 zq}%fJ_LQf{a&}B;pJD*&A^gq69hWL;i0x1rJXscG$2NOpM70sBzM-HAecq`+nLUBM zW%3c_9V6wce2k}dx9pNty7qJ9m6cc$PBcBhH2mwvvxK*SY=YRxkjHOrTm=EO&<=tI zq&GebTZu}P((RJ?%l*w6g`yeTfbsKJx_=hgt!`$yNBFuF1Wl(o*2wSPqs$DgY!1;x z`*=8_AY>|}3ApA2kq(_82)BzxO22>^01@R3Qbo#H?STKv$*f-R$9!PpR)I;Zv+RtQ z$9Uk-0T@@0Qxrpay8g$PAX7V>GQo>LETdwf%RcwJFxjK`Al}8)v%eA+?Ap&ac?wNs zd^RyGN0qg_6g1`8r^LYokJIBL@>eeGDGpbJw--J6ADm%@3@3t4Z%B6z^?;bGSUol!%;C~(a!mRdK@NF8IkUiJe9f-yrX5; zxDsz2$!Du~+M$)CU;ZSe_P}W4=__!Y>KPYt0Kp*Wvv0(eXRuHlcEpNjw6{&%B_~ZL zg)R4>5XAHug_EjW!-YmTZ9U@f$>#t&+rXb<_L?iAmM2r*ola6+-Gr4^sI01oMSd9M zVTs76@Nfena5eM>d!f=jhEEpR)(mQV?nqnQdmp(*SR|&oHOIc?4C!i@*B}_uL#s<8 zeXtr7AT~cz6C{Wcq8&61^yEcm9#r1rzM2Rxmzj=F>nD(DOGuYisUCq~`-LGTapFfh~ zA%77u84<)`1Mq1&28*`^$>esKdKDabn=`Y4{ruFSBW9G4WQYetLTAGkEjaI12VNOO zkz)E0{ijtzjLON=y(Ov1_qt1GV8SQK*IBR$dzngPHt`D)dso1r`@$FmY4g4S2#S=T zx-l>93rH{vhQ|-kHba%u8KjM!zdX82fiJ+)2XJB9mV<8yCK;jVG%g8-vQ3QvUC39E z_h%>%FS8D(Z-X)A{qV3iPqy25WgVwmlKw}i>+b+G9x|j{@}-nCC2!_EkG1&Z9I>in zrVLw2aFGiEmp>h+@<=O_e+xxwo_XMA|li+TeTRNiA za(1vo<-Q6+F{1^iK9P(7Ek`kv0kX_4SNdLplV|}CIlZP{3O4AMHscZla@7v_qRh~? zY@hS>HX@i#^#mwmbmfuObB?N&0@ST|Z~Z#v@CZ!^dHho7my>|gA0V+F1wOb)LV_Q_ z$h#dH3hcFNQw9N;@pi8>aBd4NMg#39BR?sMo^#*Bu3d^RxyCXkT9^P0)c6?E40 z=vNfq25QRH27T!JwSzUaH$A`{mTge~g1Tcaw>M~~1BB_p$u z5{fU?Wk0@odv<2fuZlx3y&ql7`L9pg4Z_;(m-7kyVnyqs4(Z_o849^lStse1{bg_W z|I#1hDhNB;a{r&en3Myp{_KVPPpx1GQJj@$K^fGaIJ&G%R&2ac?AJ z5EW(nEAdn4-m~n-IyMvC)6_g*e4n2|XcXp|2wKraAr|N6o}#t|#WXyKhol#^$I%Ru zFqR+mKD(=0-fX~JBmqMuBf*wkaID3bMdod0sMcHoE-EHK|DhSgO(0Je>OKu|NUZ=A z>a=C#T#qa%JbkSf)r7sxdhg?k&iL#97EXMH?;t@14Qy%DVp0t;i+%=*90$X14d^$wqiIsw;4A&V(2qlxn7uhuNd9`i!Z{$#5HPj2T7VAVp)*nB-peI%%unCf5_MiQKTwMiFmfO09UqGa!ky5%5 zq!f@4Dd~`I=~ARy5JjYDtN(n&(l)mrhK6>`KGkY9IxBAEW-gs)w zr`VT14O<-RLAz&3VdSb#BYtsVBL9lrS;l`cUd(N{0s>okpvxyNh=Q*aO^X@&# zGWRII+08RN4e;nKyfVv)Qf6pw9;Uzh@f9L>fG(V};EN{Yu&nFx$t{OMK`4Uj)!aj} z2t0IJ2A6Pr5%^UdnP)mdAkAWms!>O&e%#hJi15%f|0jM0{;_}l%g01PCg9Ciz^P?3 zUQ9djT#4yB-_`?qT#63^As7OH@{YAvPrDL?1cb;w=4VW!&5e-1ePp6E$}_`r*roKf za!Fh+>#azp7%PbE{37Yf1UsJm3jJA$sCRUs=jJSMjMH!?Yns!*0J2*Sq-!TbNZz%3 z4n|aRFR^D32?*AEVliAfqw5maX6=;1Wta5LC(%Oq>@p<19o18P?N1WyAO7v^WU_R} zUVXtYe}bLAQ-dg};x>K(?eF&j^4pMWyv~bV0d+n?iV4$|E^wjj!XpR} zdBrc-k8@t?Y`&|eR4%j$Mau|96yLudU)2>q{GMC@{Dq9qkh0ql%S-8zc7qB^bwgxZ z%Vjmq$o?K`H4Zq5SCQa_{~;f5W1wH(|NQQQc#HPiECFmH8)g)M$Sw$Ddz6ivrWAQe zLR|K#wO?lowS6xAsBabaV;HE5ey!DvVEoAfct^#|kObm1*$O;VogFdprw6CUPk)70 zFrL^t8kh!Rwm)Ufn3(!ht#>H?*${j@RAeSs*dBTLR+JtC_=GiX3Y+dYTktxJqE#J6 z_FSogUSy@`P`*M7y)$0h{7nD0LfZJD_Op%I8En_8?v;tj2aFkSHZH@gI%tK_5l%_M zNR&*MN6bLd1ULl#@+_nfVtLhMTydPLo}-8+>e8xuFMhb*y~V1O6!pSh>^lr`f=LXx zhrcrd=n-=r*7M9F>m8~B?6_bwYD!>$)l)h~6D^*xZ8bRDVN6bcwC-RmOTweeTJ+5hRs>#!;ue zmND>bG?q#1z~54 zm>N(6GyQgv{C6HiaE4)QP<#>^NcpeHDF5p<{P!FD3>*~iP#DUMdb6&Q~| zf>GfR^q#mFamZLwtJaHLl<@Z*-UR~`iHQ4pQksr=`|O(miPJ`^mujO)$9~mVuQHM2 z(8JOy(FTET1CwlbPBC=$x&%dj7e)gV@+5kGdHy6uW#2>!(A&WSjIEH4&(||m`zF>E zs-=Wl{#SgNRf4^>&u>qwOr$@(N{|n*cAy>I9PNeO)iKT;;Fqa_KP{Wn@L9+slML#? zO@u_^h3aa7Cvke@0^kj6%nw9O_*)MAFDeCjpR%TJLx{SgoQj_YDVHQN%g5QspomRt z7!3c}-w3~aC^JEVV4Y0~)7dz2v{asJUr&Z>!&KJ7c$z)?y?lhft7ntzng$kT?&0w( zomPFL=q>WUqpUQCY_MwCK0m+VZ^t3)jW9;#^{6kIb3OGER6t2fS`ta3-3gxa2faa^ zCNgeL1)zoP7(bFpbnuVIF2bKH!u?NSN6LHPg~#y&nKQq93*R$kgbc$=-$-wL`;_Dj zTvixoIsrp)Eb{FWnkIiQ^_^=CY@11jp8RPZ=%}zn=u|_4B?ebH3;*VT{UO3{)1Z%c z{kk)doNcW2aiicXgwKssjVvNZB})pM#&@gD45XvqpB+({_!1#gVELdFZoH%G*DnUd zrjHR7VEl%EwQOgjjSbHp?AroKtAI~_qEkg=WbrowTAm;`3Q_e!KF(? zllp;B#@!Y_?b}^21t|IuEKXf&Vq)w9LK zn@StcZlQ{)nmau|kMSf>HMgbcj}f)rB$X;wD6clP1a8pYSr<7%7la&iFTmZ~NiZRi zMpqCvfImi1uTBFu9Ev-oey;Zx72GfA2}|UH6yn6W_*<+Av=i_`$;^~K+7A!oEge2v zen0A=B9dFg@g1Sk&LL6etZNVoA$$8Z8L(Ac%#5h#Do$6v*S z67bvOuH=I8BCt7}$?((*x4s$TwOUee%Z`sh%^ZWUgD}~yw0^lKc@il~07+FkLE3J_j-$f2tg>=Oe)K^MPkCVk49o;w*26)#_fNqW=+~NHIby z5X#JC6pHcV1opu*oiH#85=Nhk)_fkFoTrjbr6RUF+Ip-q4PZSLf=mp8@@^-0ktSg# zPdSzR8w|fr>lK~L4xg7Y7&@q~dzSNLya;Rh2_^&kfO{g97l`IA15bgJS&U`*SSa5WVl)#E*NWC~?l61lzc40>ugCnLDK4J=O6Te-yaA8E`N+d@^)S0~J9CE>a zFzd!V?B2KDvg(m|Lv_tIU4=t+>oViiq~}6J&ILO#;NKD_Dke8{YFoUXegP`;wQsK$ zh9|p)rKm8>pe-uJtdMZcOkLX|jifl`A|l3J=!3uq1b#|zh+hCbLTvRk%tK6{VlYtj z7`E8mmFr-AeHJ#Z_kruY`Y7PZsZLeCqW%C9n}NWX;Mw(U`5B)TM$`G4@WUSy=Fjv< zx+`F1GK<&JzSZxS&HU3J&LZCRC(=3%pF=s@`}hZF2u&P8VJm!b42yu~LWcKhfvnh2 z+rp*OUvi!>rNqvI@X`VO`4|E+3_om#!BGP2q)@l_A-a=lelv-oCgF6(1_LQ`s-`gL zD`P!KU~C8>!`Tkr@zjO2+uuw@YWFW8NAhHaw! z^R(*CCu1(0M0D8!S5Jh^hm4`Es)?gQthN|rqzkJf0Cjn)@aU>z_~ZVjy(vy3f$M+1o|}xB@L9w;by>5??TOZYv|6hsOAz zOSfdELgeoz_s>td0M7Un{KM9(Y*lKEq$~7Vz%|8;D}^WHb|@|RIgge@Bhpj{;G)#A zmtOJ$I#&)?7PIKxYueTRSRF~n){C&%Y4{u3!HM_D#d^0>L>Tybw(sl$RR2!Ku4opD z-mAIS(9Pc!LlqU`f|BMg9O?x~Lq?jxXHd`J!?nb*=^YC&@RLLWf6dk-42R|oh{>W{ zXrB{E@no~q$IFv88Yehk+K2P2u;S-m~PMh+ltsvD8~1Y`{J3X|^j7N`gd zvNLJ8-(HaZ%#ha^|A{zuWo)I}d-ytbz)c_&vbqO^c`$I~R$ASuonoI?B$1=`SA)owI=;qTC_a09JV zdVN3csiYw6swr4S6+)Q-$8VMx&+AT=3btsC!MA2}E|5!o4AwB06ltf0-VM-zK=u!E z5joe-!JFr0z7L1Zo;l>h;Z6<5wnTYBoHBArS2t=0_E=+Zr&Qxf$!I-2TQtYO*PRD(UY(`oMaBD@tjZ~I)qc>b-_`4N zAo#*KGIUK^CcfjEpyMzwdrVd=@7oL|Wp*(QBfAu_A|NI-I{h?Ih9D4=xwH_c{30*I z14{nh4^x?`{B&}+F!f(;Lsr4j*01i9=$(L;znXXQ=K3x)Z6mk zy)AH0F+j}EH}|pb*MZ2X!w>%8L;oG^3g2TP42t*n9n24V5ygRBW=zLyBsLpLTeN=i zD7zB>P-qEBc=YawavlwVB=8c&RtD1{T%uA!y5uHsvmipT+L1KIhncae;=(W)pEV1@ z7m_di?xK8~xoW4q7fL0m%w_jT;Qt4=F7y4{wdd#wZ=FE5x1wbsaGvgg- zHF$A~i@Au}J@?*+Ty!72ady(y*`m2`n!5Pu2{S+oY!c`v+GYohdM7^5iK@fV9{B7Z z1$(9rC`o^1ieb*ANHIsm3A)bR%;sd`hlq#ZeL-m@Z1;gz%sVi|RezJh0CTdv%E0-r zE%46(TFs5_Q}@Pa`uWW#wT>&Rpc2no#V!~3g1I9Yz?*AxA+a8Hm4>KBm+OFtXKN0_ z{c-F+@8D<7)5LoGY78?1afQ8SUP=mfwJK^a`HGm2%v%MLq53~ysbnsRJZ_moBbY$W z(G~@^?hLYz@#;rZ*uX5dlo)&Z1aFMZBvf8aKwFD6(27VG#;iB+R8p?dz1kk~?|618 zb8;L>3V19ot*#HMj)g>-VvzThIb-OCsPsj`+XanRDbf8d3?A)-1yCZyRS9RX-M00C z5ff!%d(+9a?@H&3D(bW!!~uyY&wEPLxQNHqFWe!nd0cx1J0qm828!#7YyGH(BBrq~ zLl57cbfE~`4JHLj>^m>`NKDkDg&=-ZTAzHQn`%YR)Vw%;$)oz2|GVB;#D^EULL&-3 zDVMvfAVFwhc8Lt>Ji6d=IZs95{$_F+MX?^m5!{wm>Gj>$fJu;{(TlMsm6G~N)mxnq zv=Ojy6V5B6xN=_RV84P>rcH{{hO)$x8hc#(6h<62&C3*O3-waLnZNv z5epQObaTnN_V8Q76W|_sUeF4S&sC=iGW)D}0@(W)R9ixw>~>XDg_Z444qM*jCK^-8 zMW8vtNM2k?Y4ZvSMg)yS z;gI@8--l$wRbLJSK!KIE5r&9cDbfjSI2ApxBasuDB-2Y@0nTE8i}D!@!RJTs7OROS zL)dQX?ZIHWVgV@PLiawPMBAZyA$c;SFQtSCNQ6=IdI=R5KGxWOC{}b#sDr4VtAr+c zE1*%rg~_G}dMn)3_<2aK6hRo;3yVrqG$UgXIBODZjmLW^Z8l1ol0jPVgVIEt)-wt| zApYDD1heWgt5@!;pLzrQv`(4F7Y=v6#_pv0XAr$iruSEz26yWjsW)7Qz#OK=m;6ad zu^~r)t8;Wbz}lQ$0o@f-NaP51qxW{xIM(M9Hee!B;|hFA!n*9c*XNwA<}h6dtLPvf zs!?~4%CPrB;lI)-M;Xw;s*CuObD09NtU$+>>riPoE#pU~KJ$zKG9-}153Gpq zKR)y0XFC20X9Uojf!9yxfnbEbxg!y}zdBJM);&9`Ba#lDYyLw>MxFkX018tgzy4IO z&)Z&!hR(OZtRd{HO{Wnz2}370CG2zwFK+#6zIF<4m>N3-#(MtCG7UfaMc=4O9FvYX z(CwuLevf>t79xMsf%N8%GfXR3+2;|xlv(rs@?lB*(b=hKKxECU^NTeQ2@o+Y*P6He zQVD#Aw??E5rI->ddF$cJja96AFJu5|V|ok1m(bpaC@kYwkYqDv>seABDG^$3=m}?s zda0)`U3FKIkXv5rTZR1yb7*=JHpn}A!krqQ4h7DN1h!T7UQQh`fn1c>A3L*`7Ni29 zjbCM#dq}~GaAqoR%l(fJm@Pxt_yAm8G=z`!HvzyF9N;(8UD#`l&Y!K6cU63%X6HLN z5W2dzGwYo%f04$}mz;x!{JvmAqHNi45J8|)N#@?6?46+WMiCah|5|kOUoZl)SlBVP z`ZT~0$&E;A%_0QHLpGU-Mt^z8j|sg@2>%Hoc{X#SBu^e!9$`%NzP`}^b_PB(3~6sa z-uf(LQ+iKmqZGn?R!z_)RgmaEe|Nqeir83dcVlEO72#V*0_=AAGDd?5Xj2x5&fLJJ zzA>CSXCEK9%#4IIhh-B=dm*y-BU@5VuZno1qfeXu*4=5DW7TMD@VMZRKmQ8QIJUIM zw)#^75NtTXY{BYyA0M4~9FF1KRk9xqC<73Lv;%ZP5fTFXp26eCGj^mCV&X`_%tB3=PA{S`(776aAfIWq;{&>@M>`!hkM5qObye%n-o>JJAmqBE zHIN}z^7O)29AoK&*FcW6BRe7jDlCc+_4NZ_w3voJ4JxzK&%)}po`Dt}o?o7HX@#_e zl;4PB+lG~T%=7eVv&>ErpqQBUOXAthLk$C)*Del^e6Q&{Fzfllc5;+{pB2h zyLV6d8Io9oOi0G8*>7M6&gkpqpvSry_f}R7LlwcojT9n7f)JO*Vq*=lXa6)nRH)CF z?qPd{tnb2`UzFC%##2Is2kVbJfUh7EnzSG>5#w)yw$tfDa0y~3Uh5d%mnr@MtyL!4 z!0Jqd#R)QlLr1)~_#T%eeB}##wjs8U^suvr2c#Nxf8yV`$Wv0T@zLkkVBRF8>=D6z zm0F_jg}vu_+S>qYV>6aOT6GK9L(?i9oaCyPOTaCGUP1Pg^Y~}Dya~W-iNCRi--8#a zbifxKV3o8r$lIz;@vJDexI)<)Am}Y&>x|Lejma#Xg^8g8Bgx_q z=J#{b@InmGZfnV8J;LyuZ#|W?J%nTp;t4MLf74L^6s;t18$FS`=(V^LF00Y_;3L$J zHwXa2JEFudsKNJ>cvIWUBlNpJA|&YN9(+3q03`*IL-1uTCi|M<=O&KLihf zh`@ucUx%5y7mmM!BKC8GdYU5NDa@vCof}U(zMI!ac=RpjN1xX0XEuyS?5+W{A)3rG z@I)A0nGcCtwV}Y!eX#hm;$L*lR0>+=&0h~LbavQ|Gz zp#>zcgqBQJU?>+M^;@p>-jod2XGaK<3vIu+P1p`2A1a1)Y}+dE594^E)vQb177Ta>)9@$adid4E9Rs z*rRhKcScL9>jbVBRP4u6;Dy+RFkSK7nzO)5@Pw!gF1=AA`U^ADgaTqPd0m;JSTKM7 z72s6WsdW(7?bswn3I!+fdcJ6SOrI^KWbv7k>a)xvlR*_Kk8{Q^5dEV%)7!960I~$? zdwTI2R72u)>3&A1#XO8););p;`K5L-Mv+C3(6^0#Rw`BO{B4`BO)v3{_jFB;4|aPw zo3pvpU-c>BbqQa@8(GejD4magc4#J{>hrDMdfVl#n=#X$9NMf4Cj`?N|R`q zuefz}G^}hZ|Dg5Ec?<>{E6U8w#$=~YE9{?6u;~_kpo?dp!4U7P9UaC^IiYm4oh}4S z6-Jt)%ts?RJIg%AvRe<*-^Oq+6ixlNto>dtYh+qmp4r=30RrkV0oaHXP_DlX+>Ji= zg)<0WPx_sJG`3Hc_+`4gK9ZR628>LkaBua?U++XKaWB83Yt9jYkuJLmekfcm8%H}Ms<&KgsP74`P+PTN)&`>FafR>-EQu2Hh$MZ8WINd)7#jh{#pYFdQ<2ZT=3| zA(95fbF{1~Y$HBoo~%6$>cZI$y$#QiTWyaCj%vT$^1Cg7eu8d3q>{nQaUf*tq@`jC z+SBvcep{Xidk*7OE2mW+tXuBNE2o_lF(Fsjr10pkjMFfk+^`Fww+i)97L4N%Ji3!1 zbJOlF26uM0-K#|Mr%4arrpexBh!17bxDpo?nq~FlB- z8rLG;CS1SgNwbl&u5TvG5;2vRh=TB=q6YXTkM6(m6DVutJSJjm;nnS4Wv|kMfxz5& ziiNewaLC4e-oNEwtnA3f1J=`55|2uu4v)4O1m5(f&(31Uv|v8-YzOc5JU;PWIL(xFU!>#r{bMn?)?|0%pv^RN#K)Fc%a*of)q}I6= z{p_zq74P|xx_h(vg(pp4WG0bLTT(^&sRb{7my_n5+JHuK)y};eGC!4WprKv8agXjcWnfCA|VjR@E_GdU9>uy8~V{PGLH2gR~a0{^v>-Wah!0<2KEjC&GgMmG?rx>R0KySB2+C_7%wok zu`#hk8~M(i&oQE6w{r@}f+1Kw__V&S?Y7v_8hm(Ro#Fd!zJFDqRLQDGHlc(KO|i`z zCrY7rb!f_Zw#1;$;idAYJfdCKXGp+gg%R!C?y$ln-A=m&a3kMy&T&j|XhsJ<{0@3r zv3m&x7uSd+>FNTE)ruH-O~PFa)N|Z15Y7x&Uq`Y+gkg~b=b^TmyKYAWHN~3Jb6HkQ zD+6^oA{ z`+H>SQFitpuv;*Ob-wX`?kQB2JVP@BNaIPnU{Wy_r4pz2hdhs`Jr=t_*;MCe2U5nP zD{M{-hsu7bZR2*svmESsX@i!wIHx$drzsge^W3&v&N5aXFrO-|XO1ITPY{t-^<5yi zPK@-bKR=T%_a5l2}36dON20kbClXUbZg+>7f^;$L5-Rou*3 zst1jRb&>I&BTl%cJ0BUoG5a%pp4S6#j)h+=ke#np?*y;jiyzI~v z_?#0AQWkU`qM_@?Z;`&jLY~2gn52E#iBxtUReRsQAR3RW&xaAr|E*vHnWf(=K=0G| zXb@wNxM^Q486UohFKK1w>MHX?+1I$e$O2<|0s900tVUMNK=V?xZV(!3j3tYr|KOOK z93>-#zih83d86wIjdMtgjN7Aff(!NE_MwvPq*Jte9~ej<8bnoymQh(l~ zKjEOq32#xsiMzd>UHe&d3`J z<+qRZ;)7k|_7Cs_lJBWU4S;;%I4c|oVK)fIy5EClDHAB^{Y+oUeH$w0gokmJX+C%D zMEmg)&w~n0X-&*$mec{(uwuIagBU)*G!BrUO^Xsg&dgANrro8nxscb9h!wZ6v{eYx zd5Ow`Z|XU|HAGVgc_3|t-)>6{pICoQX7lSCkI%c#vo$A%^%59UH(O_(4&8q(wj<2? z=THVn)?hE*)hkF4ofiR>20BI$LsqbhW!vyG+3T8w36*5*oN<+emQ+_%BEB84*D_Vf zf5i-x-O#WqM9j0#^7HIRyX=5+f6;Q$Xav^zwJLw4D18HEgd3w=Bydin=?UoYBHJD)y-IxFJEkQS^|9XsCD|(@Emdzz?Cz-LiI6O26L( z9MZrZqOQ^4gnP?rJ@q7|7=b-x2%c$9@zS)waKpCe4id-RfKYd5sVC7#_p*iVHhsWT zyf9Y-XbA`=!x&=1G=(2Yb@r%_-0+fDNB3tHrV8 zedVCJ`lbIqQi1y9bZA8hnU{YCL*b6omBb%Mu;TCPRhRB|MbWG9hzY!&g7NBty}&^8 z%7ePgHKQ8-)eQ)2y%Lt}8>v&~HnlMMp-Av&grv{8M-i*;_;!XL!fa)Q`HAv&=?Ejy zispVu^#F|Uh2PM&dJSf@*=PPmEoQ|N@-_O}<(?&EZf_Cwqk*ecLCqo8?lIe-Y7k~P z_0@ilVLf#H`i(Q35DFMZ5)_0)a8z2U^?S0afH_GWEV_&bWCgmvA7M0ZM zhnZKR;5ad65#S%(Ix)(lD~2oHqm-XtU#B7a^Pbt{>GtbVHa>p*s6?G&#w(MBfdc-Z z*$1-4av1gKMt~k)Xa=8T9XJuCr(^iN%O(Z~=XN%Sd#Y_m^pB{uWPLb6cU3p3txCbA zOSsVhSOEpCub_25jR#B}(w*zmb!BUiGG0y7|7h}=Lld6_JBbFng6BPv8E{H#hQ3+8 z9{g<$;>WhnP$*}Le_~fm&S;u3!n=6=+Ss)thettr2ZsLq@3vYJA60#_IfvpH+B4VGQp=|^W=bX{%>I1xZ8E_fjS#%`sPn|Uh zWYJS%AUyd7!~FK7BH1XvX)9VHH)rfOE$yCH`jG_ZaQIRLY-2RXE6Z=nvh*PJ2yvvH zr1ahw6Dbo3g5jRvgmuJivC5b>sy+Mg5MriiN<){IoFEY6V6M!@lJ^JTRtM1$u!^ml za#vx~4U(g519exovRbCG!|#Q$whI608i5I@0|O;=+HNUoeT_$!7z&K#!0B}oyby{6 zs8(j3Pqq#3VG+Ee=#v_JXGaX;WCOQd27~0xS_Wjh&+fef7xgc9L}B*9 zZGGa(X0J-`cCL^G4pdOJ{D3A3?LjSFhslQ(#m2f%y#Cc5S8oNN6?=P6edPRGYtc}F zl2ai~7EL$-NhQ?iZhaGw8f);etWx)V>eVuf$wE?H3z-|slRaSNnXfv}&c>w&BcX)H zJ2IVDyaqazDpz{9J;Cn$X??OPz4GbZ@-{5tsw|G6*wBaL;4J%GJ8=is14QHXV2m87 zP#mfhDft8|Tg5IF`W^sx-*?}%UUmC2DBlDt=gT#a`;GgI%Xf&>@+mk%uc=s=epo4n zgv*?WWO^-AUJ9_7ju`<`V&K^eYIkGdoxz)*pv$f+$Lvl!??e09rBxnP&jHzAzeoZT zzv)-w*i4cL-Qw0Fs1PD6%ugskI&0^lppqH3itw`?YQb)a-ON$Xdsg~Sdv(VB;*%$m z%0Va2RsH-iRUf08Ke{E}CD5R`GT>vqxP)JxQ+vK98X+LXt2`!ZU(k{k$2%;jO{bD! zzKgph z!(TWUYR9D)8_ERUhd>pTFw@HFAIw2J#VSd=3XV3pTsZCrA6ib9V3(zp2#A|f*`3q_ zaeSJ1PIBnOvfHFPFKcx$qmQ=}TXxRen-uTa$<5wV^s^cX6*spl8(;B)sE|FaW=N;k z9fKG3DgFoLJCzRj5JuA^ukprBF~vSOv$lB$_wDd|gG0;rH=J(%vrvpi2h{j|HDWrw zSC(^jpoE+m3WvJyx&!J6C3G(D&*mYyggY^Nw_DalcD>}tx|1wHPLii&B-Hp35@mfC zo=5jo$J*ov_e+NCYZ?JY&D)4^GfG15HI90@VLJI@xYjmZR#q#I+fA0u%?nsLUzQ@+ z{I9y(xDCF7G!)+_34rK*w9dnki^31`9epZ^p3yzWDQY_L{l*WfU9CPPs5FOO+w+=Y4wx(zq2xb4@%zi1Z5dFD38b6t{e`emq=r z1QeDB2xWZ3M3xMnI~XP!xhJ;-Z(Z-ns9A^ zVc>AZ1BpfQu%<-En9>cPpmoIS|N6J4NCvcpq42DTY|I18kL6#&?B!w?&&r-w5SJ-ca9c9DhQbLDNSVLImo_?15`|))~9qc*sc@dG(Lf{>#JzT6Uw4d<2-k+ zV;ZMc^lw0<3;JVMunz&543m%A$;rb0}FYbHB~S1bP2kAvq!UhHw=PrL|t zsdPgYk;GLuOtHtxY=ltxDg@oLRlAI-zN8sfuzIy+KLQzYG>WcXy}Bak)C242GJky& zke&sdL%WY;9X$2$G_;;GZ6ONt73N5}HiWKP{MJLhDXZ|d( zIjD$;2yR(U^S3ipeiI+HXHec|Wr@F%jWY(InSW*F@}7JEq`)gMgbs5qdC z^{?baYsFWrf}&0FlfjB+)U!0iYb{Y(gn9|dW1rWiZ{t!8Y5g>l$!T2`%}$HMR1)Io z`If6fwz?Ii>IERzM+kQH-Q&WpNVipQu0SsD{bNm%ezh? zH3v^;KcJodX7I1Mq6sYWKL_mhv=S&nCXd@ljJYYXwsWSxX)+)a*Xq0L&g#|DJFd^RJO~ph(R*G?&G`<)w16ayk6CZOTh;$Fs3Tkq`mT+k z-Renm1&u*Ha90xu2gltg@FmJ$ASxO)vfZRo(FuzlDn$+hX-v%w~SLqtnwgRG>SVnsyd=``;mDq#|0H-zX0Ce5cYd2mVN> ztReENesqKIfG(bFJ&}0puCo;uuaYh@=^&uR8UZ7esJ?a{H|B5qssAUCd2N6Yo>8K# z|BTB@za!Cg(hbAb@P}oHluNfWvij2){SDbwr{|YGokD1)pB83?-npQ%(dY%8mV@|# zB+F^X6ms9(Rw1{-K@TOj;(g$=zWK!uDafQVk^Xw|qwwa_cu3dkv^o}`JhGyWO(z2- zE$1|;a_r{Cf4@d@8%=Fn-TMD<~@)d_CYFino z@9Q6H#XnR7yHLSlHqJMlHCI+WkmG0_`gdeRFq_d>v>=MHk&$I%PCdysHZ#3))4+Sw z`#`@R&t0DvdS66a0$~Q?Rdsx4zOrhni-wcc)Gei=ckguzXf3;O!n|8I`S2yesyJk7 zst7aJrrF=vxPikq{0!wf+MGn=dTVl-Z2d)Tv#&@G%#WfXj`VG6Ln>Zq&UQWvZtye*tj#>JmFH5I|SOOHU zKq<`P^I4eYdb5(Ek(hTLpAM8e{#oL({4lrAH4eAl-1n7mm=&F0$G#oh+8%$O(om)b z_e*cQ|9-HHN%3g84WsF!=_I*7Qw5=oDEc7-ZH77z=@?pn;)>eieQ$EL1QtvT_OK{a zM6pBNC--lonyPLsM@u}Ka&N|P4;Mokr*zV(W0?B_;>ov&Bee_H)OnL&^3+}BB1%w7 zN41OW_W_5w#Qm^r=d(#V=8lgPVQ%#hR8Z z@xcGp7a&{LMEz|97QnzXV?7yj3d3kRs=PgtA7@x&oRib+EJcYT7O!nOYW!~FciH!{ zmN3ACEeqyk6|x2q1-l(naqCU?1L?%{GTQUJ=z6)$CCtxX4??*?Wu@#I0!c%HDRs$^ z6fBc2P_2@4?HP^muyU%DO4lnvyATGG2UAd3( ze*i)^<|w>ojl*Ot9y>d`Z2Xc^@pB&?n6V!P^1F-mV3f6mL3B5<0huyc(ikjZ{O2A! z%Bw%VWHlQ35+iNF0&;iX4)&rftjb0C!rGFk0$G1$ zt5p%u_xWYTxaFxi#fz|Ufw258%wj|HItGq4Hsai9oXNL13Xk4|%lg-3_vujGuTxCC zT+1N@G__mPV-YG{No(r+)w*D0|5T<5>R5@1(rjbIVBj~ zrB;JhRABfAV7YgTj&7qH9z1yT?U`ip1$^8;tpTZgYOJW$qkA&6x4!z`0SL#&C(xoi zg3nYB**T6|1?*HSvug$L*+>bB&8OSH!oEk2InQzE8Uz&;OoO=d zSW>=wKz6$koDGrM>^7>7R#b-UTy_$8K|`bC33&Yn5C-#A06UHh|Dqe{uU_E`O|)w7 ze+8*Gt%3IR{Ys{36~R}-up^}vlN%7s&gavrI(8zv*1hw$MN6iv+#YmqLlI}ysX!ot zIAuvc%#xO@8Ox=E(_535AD#Kn^C-z=8mMD?L#L2*kJ^0jTSx>Y1`aVCHoD6kD~wq@ z+6NlBCc|dxbP1Ja(L|iL%`X0hQ(6L=d5hXoiF|tRy{ELhN$0ySes896LYci>u1#_^ zKZ{}p&`Ng)p9`ZgeLXLtol2PGgtR1eD&Z>Uhi0T9y8!3NrII~z1tM9x_g$aO+wpR? zS8^*#xZf0-ui>`*cpRuMGJ|aA?JB(U!VcniYMN)0om};cNL0wu+<>c*E`mLj`2CY~ zrybuInLsTn_N#g@@HI?uSjJUWcC)0u6}dCvjj71o@}U{9n%a6TP0OQ7;+e$p$LZI0 zCPzKiEas9#Lh>0EDqpz$QloX_@CCTl*Q#z&oN7#GK%3SWNh)8og3ghJgSnIa_|H57 zgpEP#%>C8FvSoGvApB9t)fnrG-C5N5N;&T5YuF#loWcDBe|Elb;^@`=g=|Yp3{~p4 ztc7Z=O4qVLSz4^wnsHiG*tao%Uw}^2{@HFzPR9!AD;F4K4~uu_6RL0cIyCUrO}iCv zSKqdBKXNJ*(!^9`$8EwQ!U+>U1D>&p^KUf_R^hG&;=7T{Z`^e-)oI~Efr@oKs@sa2 z3U?s~*XGjuMv!?|Eb%tPO^Lhpmw(cYUvdU1Yv-OOug-*aK?G;%t2@R0fJdc_siXUu zv2fR%bq7tfJ6^guSWzn6^wmljJ%|~D#c(TM?_bw5`S?pJ?>kD+1Q%@K6p#SS@R8sl{9tZ80{2aVRYjDvL}#I z9KXd}Kkf%FtCoFl*o5;<|0OU`HEM3D>JP~m;iReOm1@LUbUuVR*}`{U!uQwK1pAxl zw)n^FTxYQGuqiZG-bC^PBy3@q?vyJ+h8V~6L2Ya=^TVul0{$kNk`Lh`Pc5jpukHI< zzfU3CC|Wm-WmSz#c$3U~CSm-^PvY@j4}YWw7Oq}&d#~NzT8Dry+<voE_KY_4tYPDDslPw*t*Sk!k z#%>_wYLpspK^RrSv+D{a)a~jg*VyxKMZgwtCs5Vy3``8YM`$~%FvS9NqYyY+1v+$T;hVf$M@UZA%idTVmG zxCqXX08S(=Anf4x@%Xd91Wu8w%^q3MPiy@HxdF6bOCzj+g)X>LP^GI+M@}ojmsLUa zr-~paYf6g_FB?bn!lH}%e6v}3)Si(=v`qF*1e8och`Bp${5{(OcB5=al;}bKwsnSA z9s<7lkryLO@4JwKp+NmBBv$Lg-w>LtIQHQRFkgx_!tv#^pkpTRh1?btyHffatj~(USU-%!G%DC^#(HjA{#?OZ!BDXTn=n9& zQ?;^vlTn9G`>~05;*)=`F>XwRk^Z872RjeHU0=TtjrZ+bP+YskIZBB?SpbQ9z|)4h zT~uRC>Ve9Vb@=b$$)os9XdhJqDP2Pl9TW#bLJcK;$O?U$$Z5oFhE` z-@|N#r)Djb-@9a>0-2hq z<;|Nxm)Ig}dlmc?3@H#4Ly{TyOS-9x3;zwB_@k&-wJAipUk+u%Htm-0zKSj+9Hd|o z41b){R>Ys~##bDgI>+#zwcz)jnPr7J+V&Y-O}jr|1|&GeeK5RQVoL8Oq&6fkWj9yp z3|A7$>b*b|*e{9fO4zzs`>J1XF?Y|g6;UG*vzm3F6$>uH$4i=G8R!b)6IZ>Y$3Br)3mK7_yBg?P&EKUU*j5G9bg zCUUI~K0hAS>kJit+0!M9oMKY7)U=b+7_NkR<1=jPf4a`^b`q#Ss5aQG3U&62#<1YJx&AeX4`Y^|PAs-06dl47>4 z**XO%jAVZ3um0&KZhyM zn^5=F#k(;+#a!ru+e~t^V~I=%-uvI(TnSF4)X2QKU6bg4pNFTn?{^g(B{}WcE`Y&^wv=ZyLpGVSb zw-4l$D7vpl=T;`_S!aWH@qmC;+hn>J5j_d5MIbIp4-nTLgqs~0a=aM_TK9-w&H7on zjkFnj2)D~5Yw>nh`|?!?4xaz(6M@fWgS&iKf%Dzv8;+OuNo({9lvy1N&hZl=#7y4` zZ-LvWRc>JTs4*CRU;ZlA=@v>ErG21VV6Q-+`=U09_m_gBNRMg!Jv|UG`KCspp5buY z$!VCz;7^$!a%Rc3DLuw8VZSCHbn7u?*ShCpr8n-JmI9g$NqrEIdM-2FctabfbR?jq z2b+u>*TKMZ-t#J34Qp`b6Q@fxwuO$Pp_UmRDl5w$>6I6&ToXeU_pNw7L-wfNspZ?O zeRc^BqVIR>_28TlO!tQ(sy(57QZ$bzG$m;JA+U@3xXz zj)-iTY1ZpTI@ahpDazz+c3g5UCmb6n+!1!(K={fv00}wZVN)JZI{)TL1M~#psnW#l zc}3^xECRe$rYKw28Bm9pG4Z5Y^(D>k7)3iK2o0>r%ijFHZ(CcwKV~P?t=Rokslp$N5#; zx7;g{OvjIqe6!%8vw}B$(dzjycE{@nn2m^WE(G(9ZBwQE*L`(hOie}JT^vrKlJED?Xy^Gyb3n#0rA)# zfe~}o2reYoJO2Zu;G&k(8V1P^lSu=t0L$%ZcY_FlDNu__8i43Ocp8#)?ySvvF5u4^ z=L>1qT(#q6#I*i;tDUkLi%bb{vseqB}^wIatMBfscHSS;Ls4rMW`fl@rOxvSET$gflB~45x2M(E;-s`W2J%mUgOZLi#;_~ zJektZQb{4qCfn)--=Nu;53%;(BkxwB)odqv_=bSRZcg9>jCTD6BSENExGPqF%bMXs z&4jxp!6B?WAK@CZJ4M-F!4kC~A-d}NJd%2~I>%vnQkHzY!u8YY1rq;AH#|%<@)$AY z$!5}!JR%lAA=C^|lcjB=9`*rwWrq_53ZH^(FmXJ86uB{UbCM$cZ z!essE#axG-B@R0sE#5H@Py7&n2g%pp&XlZIZG!^=2~!`ad%84I`>U?y`2G3u7p*1W zw5WLJMJAamB;=PtN=H#!Rg-^oj8h{FJV3+6F?HE1HSgFb>>)$PwxLG%rMreOTb9~n z+vvr7>tVPDjoi)gvG_C&#z=i^@?tT8CUQomFmTRw*lipZ*zwPM|89tdCUhmrVd7?E zr-t?a$JuwsQ{DgXb8?DfBpD%6Dmx+y;YcVedkb-Fl3B@Y-M5{+iR`^YR*Q^~y`mDb zlac*(oY@A95>|`kyjPJHJy87h-ft`M;!NE$~%RUr42Pyd|F6-!89v>szk$2)u zwpX@AS=wGi=Wua57auY~!Lz7<+>|8d4a3v3-F%UnonZ z8@CyVH?Su=aTJG!@v(AVvq8EZkYQh7(%_-Cj+|b$I>omgjoNrxO>6Z<3uF)6>A)sX z5D?a6+)AzC|H6by#d)Z42WC#oF{lwT=C0N?kuoWMLV>Fe0GfpM#5I6jaJY_UnW6cPsP5el6tP$kQ%(78!8%@lJj!tkFZNnxUM5*<6h|IvcfS?YyKZgsN3Y zdwR?F;h7(wJ2t74E9SC zcAC=cE;Mb?u2h?W87%U8HgHj^!Xac$g6V1B0w-&x!0bXy(vH&F9mAAx8Ko3*x0hJz$d@TKznq{Fy%2N zBy8h~4B;=-4h3p?a4SYy_k1fqb{-6D;x*8))-PE}v_1s=s{~!t0k_ZhT_Cm@04}1p zP=(kd?1#$BvI3}^fKG5bU~i@IH9w6uXTES5NkRP14K&cRL|j&zIQZ=iaC_@JY*O}8 zTvZcrB7>4Kj#?91P)U5c05B)em!xa@ayD9p@L#8vzVxfyehnEGY^&M_PC*J(=rO4D zB#Rx);m-6Bs?%zdrKS?P8$pQNyzNbL@RVaXNBAgXfu(NoAQ&rTG!+4(IH)w(QdOm0 z2gFRGHKsM=J%wAHsv;ej=r6yx{oB|CR|4MR=noczk1XO%A8WxYJD^~o#vkU2;&qJt z_}4Fd@Nc^J^*0iVQ4^=c+}5wIGL}#WZop4Hg{WJ%BXTul zxDqmL0g9D2O%cc2mM&Iam;6+Gst{)Nq)Pb>9}|?qXs2}J*G2ob#)L(FSRo@5L{2)SfV-{qT=U;2s(CIpZDKXvW8S37Ju`q7RViO`~nHHE_zVKU>9(6C-B9Cdi=#=5M)vJRf&w&s@+UFe%#>4C_MN>iM8D9&bz<&+|bI z%nOWzu2qokw6qz}Y89eio_Y0dMrVSCcJ z@KJf-nYtDSjT;Uhq(tRUtR~eqe|c6#C6ohGdRkrWGbWFkWu%;5CJaJg1AuId?=%AK zUHvbQpYZX{2$(Gfqri>wY#!@rN2sKxFinDoKI`dK{}< z9N0t*X zpXq_cVbZcya3FcE1%@f#Y`ID1sTBNX!XW}m+vLQdNy7wdsC0O0QhmJjkdS~~2X0w| zo2afH5kY|~{#)4#L>__!UoYAA>FNlnff!^G2Ij?2TJ`XB-Z$ zM52kQ9|HcX2BLNQ836R=X1cVDJ{CZ}lONUF_uk=jadUbrTP@$*0ptz-x~^%qo>sm2 zA?VEX^z_HG9*q+4Zzfc*)zw^q0wm=ae!?0IKQ#3{p!?L3Xa1LL0EMbNqh~YfkH0J~OHv4}3 zSPm6gaLIh67GT<8AF}pSfBiq+d(M@F9tn81RTuo-=6zkENc1bOU7UQx)(;{Fx83Hs zg#7%rBfEAgCxm*tg;UtRXZySxnxg06U{Z+N?fdowc%FTG>$~pVQaD_^{HToi$B&n2 z2!-k9PCBoN?aujfx;!6XOO|*$)yAlWN)C5wO8s)?3W&!m)FmV&x@1;!lFuxKZQouP z>S|p}hPm|CV``}_@%>T>Txh}?3t8)zx4&CaMJiO8P*CfhB6db*njeAa4-2e_^3=7) z5$J{DMN0g9AhCnQtXHwfmAs}!9vDiqH}u;z@C?F$l~7h7Q@?-`pv!f=cd+xah=}XV zmr}bry1?=1edhDoqK@0a&F^=Ii7f#DI*#k~Ku&5`sGm~F8G1YPmqUqdL#-;93_ZgT z)hF&YYFpS3E78D7ULyEGbyZKAUL~?k5HK$xs<6~Sc6NE!DIoCjts3cZ<=7|dPKc;6 zmv}?tZ2M(YAA0cXnphN#$hzGY!yeoH{S5qA-GmX8`E0ENMPGxALNoriy4LnTguYr= zt~eFnqIIi)t0`b;UU4E}gqbtMhnaFToiO-6G%0r~3$D301qAzvjJs%5J#TR%kK_~Nu)SUI!SPj)r%dAL-KN*N!(-m|OJh~mVWfPBYC6SVn=Jy|- zWB(C3eEF5u98}k4x)0%wg7LzhoPgWG%!=Kp)yfB#s)IRw^hH?-0!EY0GC%-J32uk| z{drpkHIBdwOkpJDmjY9-xo#|(s#$zy zMF8sLHV#1XmPcR9QMriWg{?Nw7s>d8IsfN@a9Ct)8Jcs>bN2H4u+h)Z#-==16@1?2 z!m-TDVcwM$Z1M)x{L|0tZm=nYfQRZ8>^_$b=i0xab+u_dh;6onSm-Q~$ zJN~-TWy+Y~;nwg~zdOWskOT@-ik)DO)fV5qd&{AQO+sZDW1hb{K(%_)%bziHm7wg{ z_6lj!N)x~sxb0;A4%EH_<#y``8M(qL&RKd-v&M74DyOLdhhbg#ZEo$ik52GKt?V>v zGf(i{?J2PxUBT!FgV_S341*b^ZqrsvgH*HUFAwAyMOEL?a=e>VN0GXAf7yc`e{4St zVyW*vDmTYcoRB$(S-4!9p3b(7U4s)u4NFlloJZtoN7NHedGSW5Y{a`mzr6le`w*xH zS*Na9MfyKWSco*% z48WSB=Sd41oYYC*N(>td94tW`=oNT~w~RlWC$pIL#IyH z$kwr6yQ+)aTE)AlUauVAn{7VwUf&GQ*E^0)N86!sdpy)EuK3@@%6b+p({WiA>m~^> zhXN&Rph)GB-InmUU}#_j^>E_1L;)7%2;l=q&iidS6mcZoKdl9Jn(!9Gu6-Z#o#Hs0 zk^HU03#;9)IbW5v!_Vsv+DEMEPrgGozx*0j5S?YMHYD$}p^#yePU#Y3_nNy-Dxne{ z%s@-{9$Ndu^_7#IuP^w0g319G(iA1WaL*-82j8`Acz0sC>-xYcelVUc?T>QxDOePZrz+vVKjE+kt z44|kL(>`g^w*Ujex@mcjCxE0R>y-1C4mP(-9Y$}@k2AvlG`wq2+QV9b{Ve&U%3$tU z;?z{k_F+vRCCGYX2(C)$iNt)-syid|%2Bo7Jm}XOjP}&^)aVjOaY>et=Q@352LIRN zQbRyTPtUhRl`r(BV^yI~XjL(apuL15&rQM5HZ-=stJlu153GK_0>y2ljGgcp9B#(k zuBY!02A*f?ry+GbH|BHps9@LjJyFfTiZaMLOcPvGo7g*2{`0Y{A+VY|?6Pt}C-w!8 zB=QdR(lwi66kVNLvQ~$UymHRaejAPdb{j3^YYJ9JFc)7q4Xu*tSvlResx#Huz>G`` zh2;A7U5|>2n$K1c**0I8J>ZMZFcO4&Vl4??U9340Ai<-K3{M~@O@^XARq3%%cx(QH zpt$|r-%0@6H>{TcA(bre6?*yUtyt*YFK*e>!%?7c%ehs6dYH%Odt9-7wiGo`6FQe_ zHIa+GY||LYL}>HkuuAphqM$NciQkcUw~F&PS4K4?K}pp*uNye)l}8?(%Z!y)?`}FIp-c8-^s9{NA)zqHs4%d-jiYH@)0@6;* z_UZK$pdpmLSz5y(E;aWWq)3rZ+k|?9hhEu z?2&EX3;?5%m2RLow^R2>t^xycq=PcG{^DXi+4trN;ORN>ww~KzlbpTGQN4BHIKE~w z;i-a2v)LM!3ja%|9em~R&8cm}NK}n@%!0qnrz^691(7-O;tF}%A^Z))9B$OAQLq?K z=6LOu^uK#Del~+?M?)u0p_%THq*17GB{ZNH*7~Ge6r3K)yw=WV@D8do`pw(Go@Pj6 ztqGqkH5WM@mk(d^_jfWs#Vn+q&ecLk6 z>r||g5=xbQSB8FhChRf%D*N{omxPhaA%N;?VE{apWa3ZczT zGp@K%S_8FctP5eQN0@|3KdVWKIB+4=a!20J8fJ%i5 zq+G}8uuuAC9g=tKBFkzb9?d}35VI#w{Et`1>fGYzMGwp0b^Oahd^>A|9VK|Qc@}Fc zcFr)q%G5s|(C|m%8B_m)kM;(z8wpk@!(DqD9YURW9eLYkU8I)e+rOozUq7>T69w>u zmg{@E?nsl1s~;6OAnlj%X; z_$jz1Z0q<>lo|>LKg@`HO!&daT-zAtn+D!V4Mit*7#pBv%&@ z*^c$z?>V>AAYDQRAe}KOkowleB>ft1h8{DRzWRbw3v6u8g6!BT9FFG@FX7gmjEd^vzu`^b(n_JahGgZ4!E4FB??-iUA7C$c1o=b1<_{#1$bfbg$d z`Snp1+fa1AFLqukm})FH;N{U+Fnafv+)TrBZds~?INyMypxjjD68%5F7-5KVf&|;{ z=lRH`J3}CI-0PUQ?(IKRXkf>*u+_7w9arR;@q)$kqT-Ks`~R{+9d|tnx;5LJwKmc; z6~jgsH@X@?_1roPIrrpy^OzKfAF)4MyWKgQ3)AMtnRblLSZ}<6?33fb>s45&ek)FL=6oZ ze%A>*{5{;9)5xvVgj+&{zi{yU^_$@UP$Mm^+o|;;Op2*cQ?VrvR7*WTVngrAPDxm( z^I@J3hQ3Nu$XMZEzG448dLvKM#HTkqOA}$8cdv-v3_5n7op1#|w-)=P6&K1O8#B)hY;du&>*Hhi1y6EK>lgY*)upKZah!Av~M@lxE z9bJM)`W<`zsHWC!n9aA+{>`3{r|tB-akF4SFG5bUbM9=}bM3wL)W)IH7gtV3%ip#B z$FmJqDjUh4CxhRWLw)fmxl>o3#~sbvA!y?I#7|NfvP@X0ECia__D4;m!r2zGXhUvo8zdS#-0;+MHIqNa|p)0F`#ntm5b|(e3Yt6#I3aHfRm3gVSEcU1XJWB?%;6<$cpi zwwDOJsq@H=PBaCjGyB9Ee;Xe7ZDTVOvD0{EDDB7n>;ia=eGG(FL~4;I@z+j|S_BuY znyVDcpXBFHw!ZXZMCcbI^Yw^f@4*SBBC4N(MIS<4XL}9K#ycT=kN}tbDv!d`o18U8 zg*aA={p}vRJ-;moLc@K;>#Z~nVaH9$Sv*)x91B3QBiDcH^FJh{fh2?7s{iTZ;%WkWmJmdZ{D z2$)#s7T|FZzbeFeWls_DY{%ihMo>SNyMY^OKr4QNJHYdJdI%||jX!hwd`NUiM?;aC zfn~lkkr90>98)B!XK-#Kq~M=G+b5 z3XQ9Hmh?p3=2hzb(pGtlgGQEpj>Ykzm*RbjkKb)cP#R=^WGYUvh?p=JXh8aPFr;h-ZG? z2P%$)5=^M^TmVi}CP%|X3QR-hEKB$Bi513&J^0C4en+Qoyl_#-G@A6T^w*6{ojP5X z@=P>bx6Txaw0}S2KRzD2hGejtEgl?j+Jz_->rj(e@}0PkURD001OpaQ_>;B|Fssf% z=}CKPy=jSh|N2S%oCzhtmv#lV9fvNmOromX^HgQD7gc{w)CVDl0b`}-4jeD0pNY~4 zAN$tv{`<`YY9Mg`cJZW0ZI5t-D{zW6feg=_0X0q-c#@5(@vwoodM_%=E;b4;OZSY` z_jv!uVc;cB<2tUAkAEeo-x?4f`c|Xq$dkA*IEf__+7v>8=I{LB_n)Q7Bew5TnVkk) zIGEw^^HVa(zF{N6h4mTEx-pWb;)}f>-#$_{WiMf=6ecM5x8np02p~4fYE)Lba6rACJT<`B53* zBxwh%%6jbkk4OF*-~HRx{@7%QoRRP1x0nJ7FG6S~c&+Iy(q zh7tbuMgQVz*p(ojmAS#LyBCh8q6t26P(izqsW_>yHgg*|ezp%jFGb!P37XfcMUCvFgxQDo3t z_t;Te-@As7;XB;_-@oGb|J#93!4=O@Zr5uP8VEu{U&!#DlC8x)+li+pGpxdvBf*HN zVefkS=s%t+Ldv7?Qic~hwqt=b3jD^Wol%WyOUH0`>0sw9odTwVANd=Xq-(xEKuuWk zqe1%nlm3f=LQZk3C>YY>0AbUAzrg>v^jJkKfLWA}Qh@o~XQ~`&Kpi<-q>`Ug!HiPSAs3+JeSO;kOF)Z(sNS{c9%^>?Z%| zBRPA%fAs%$p|_VzhJ6m^X4(gVOUVY}q9Y>5c74tLp>?b|UNI{>JxJ~RYqt3R+4t{H zY)st)*uz1ft$)h_dfr`#K#4uB0`6 z?*DMue)%V!tCW!Oy8`L_B9YP?XIG$fC`YJo7(kVt$}Rb}ejKV1K;Q4GfY$r5v&nAc z3!thx-a2YJa_hgGRz(#QP+gY;ioH`8aB}Oe2q4u4Y}5<__5^0~XdQqg4uAmBAh6_K zMi}BAf>L*@HK2^AO?W_{uBXh=HKYlJp`;VAig3S!D2%zgMOud2G{V0d-ux85^$>p6 ziV%_8_;8i54%YV1o!cJT1MJR?yiUL(3;=KAAkY=<*Aw5`+=zACNwRM^3{EWhu}Y~XvW@$2%-waD_jm};>6W}e!0X2;P)GqaMDViiEJNe(45GT4q)Ue zDSqY+1juy?5_syqNzWcrY3ROz$7HN_1AvaGrWwFxEtRi-Kz&Pdi*x_^nCt z{WkpN4FBxu)s7`3>_O|Nv#PYgjM%I@=V$i*X|^OK7b@H+rty*w7xl5Or|SSZnm~XE z-I*HP2t^;I;{~9Nv$EQP%@!(&nSG2yzo#ATGQ~L@Am>8V#P}EFLjIm;SBO`LrSO|++{iQ zZ%uY(M}zFs-~gb7bXu#DGh(Rg3t_*)C+4W^V}>?noNZw6aPvB-Dq3^{9|p$tV&4jY zCaxd=K?i-nxZa)YE4D1st#5+y>ZA#bfY8UMdtFVkVaorscD(s$Qh;bfQTm~74x?_s zZiZdA2DICKcSac2O>}`N{ToY}2-Zt!5Rgk9l>k-TTqzc4*L2`9`C{}A!O1(nPmu6m zgY2L8=f?spvLs-GfNtxhe9WmfuxRbOM&%-TpNW8s{hm9##?~M!o}*kK=L#&0ZN{{*CD8;T_2ISa+GdDFS*ck6#~QQNVW#RSY>O7n${{0fd#OK?TdoeY!+%at2n< z`8lm=4|^)7T>*5I{5$>Fd64LlsWk<0#%w29ey!~OVy6D-8YYwwAQ=B1QlQH|O6`G= zjg=!LE$XQvG>FNfPQ7d`l}f!UBcBSe)P5BoQlb+pCV)H=xkpLw&zMqLCCjNNN%?s@ zsm)!~q~_Ed9&L^u+mQv_LzQOjFm^bMfUwO31?_>apd83pj%buN&clrPK$ceFrLl`Z z(>n~xme4@-Jl!nF z{2z9ipCbW}jsxL)kOUr^a)h#|;*nV&&?KA1MQ#zXM$CXlil2In?PDT<42y1lHd+R) zwK}$ktnrC*Y7I_~VCyS@x}k-w#uCosQpJS5J%F}^adMXOU;=U4g*q>mm0Ej_gzHiW z&%mjkA_IyNYEO-PnIZ1_k1vI19pFG{jELU&Y+DMKq8!*Vn6zTQ^iTy5dIHjXL8|}o zV#UXJoM#5csk?#0#H>o$1PDSKdy+ZxpHj@!iEUo7((Ci`@|&q#zIf7pR4&z5611}X zCxMOYb*;2{9p_j%f(;*k!Vw%C3J#116ZWbc2Eb0?VwLf_kP%`8vZ=3z==%jk3s^_# zfbQsx;Z`!?DRN>aZD5U!b$gshq5=4@#hJl=glo_SaFX^2z*d`A0VAC;Cvvwh_V(0cWyZdl6>Vr{`z`X5Ca&`eC&e{?Wr#I!Ziru zIC`-iA)=l~-h z^nNyjbK)!-dXAy=JO&=QZXiYaaAWLr){)`KR=UQ{IT1_H0LJq+Wz6vdl_+%^a1 z8LK@J@Ux1i6Ss*mRYC|avN-=6r)G|aGPDs6Q^rb)_ewHJ0MmkSX5=?X8Ezo=c$fL@ zNy?_KKD;dCCZ1OHSpm=66btuXv>uVEaXxOT@lP}lG0|#Vg zCN@!5D+S@Qq9no9o^41=#w&LR<)2L+K=@y|%it8jz&o4n6$0R_c;7|GDgbjg_|fq< za23=aqMdwSXb5Pld<&MitCtW`p7P~9PR$Xz_D|oq0P<=`=}y$M4Rdi8lmonEBOpta zMaS0gA@>)z-iwwA=QzH0t>-^w8jehJfMb((j=YSAFiotBVmg%6k5b_{FgJ788)mnX zuqi-n)N{!y!=59MmG(xmj1a_Dag#xjF+sIlPFP&45K!+L)oK6*#T>pGrUB!Fdh*H} z_3$w-#8B~r?7FGL$*@$_l5?RbGaz7%&9e~y1RG@4wTkHVh8Uiw;+i}EseBnBVu)IN z56l3L5OPLvt)>@*YF&d7`Wp#RT$UNbIS5UasoASSk{x--(BPfj2WAlaW(MGz@>7EB zli>xpK@jLt&??J1+;VxcP(CzyGEjo-Pc?HhvpIYjsMY&-O}-BOnAwwv72v=eRmn53 zY(QL4i&T6?Uj0$p8&o#k>+zclEY4ux>wJS9zfAem>wA!BuIMQ3{U;ClZ<&W3%%*W7 zZC$0c>SzsZ^W?NRuS_87;W}`dB3rHrG%Y+&7_I<`g{u==V+5s5qrec{&W>gb7QvY% zStG&H?ty$P6E32_AO|SxxiV;8asr}DAu=}3eRIXE7?sI*ef(hRFvR33Wk~44HvGH+ zfO#_rS7sTaudQBekp!8v4GWTt!?YYAAIgcR(A{%6NwAEg1J=zgyKoop!Jfr`_O)1y)E+J<{a_tB7(fq(Prw$ z6!QNTgKt|LB7EG50*~+`VVaNG@Yv$#;cB1lq%=TcwzG9@JJAtY0N_%J1=XLkp9J{K zS`#!|19{02T+}W>gfq^xLuMy~`4Hj($o9xP9sqVM^P;RQ?QQ^`hM;94vw>&6#B5F5 zGMlRTE+5dH{?UR;PGIMlS0WGN-8T(}Dc=&|nTqj>^d_w;u)l2*hLh_LN&f(9BzYEY z<)Bvd`2(Oe#@rw$k?IFQRqIla|B`GJFj}N;vcO$7; z+i0UQ6HeLS3HhiKZ0dToSU5ZJkOO44j3)URhH-ImFiGMIs8bdtaw-FRMw)&y9yLw{ zL0M(VoNxt^Pu&1U5jYMEpB=m9sR)P6Uo3`KA{NB?xX2Kp@ioT_B8#e=Jm+g| z&4J($*N@=#D;Dz8S-ptob&Ak%^!w{x8KRrkmJ^D`xO+FBCK1TM*01*eLNJyv%O%!SQFK zgS9f_0Qzkf&3l=YLFfdgs(@5TI-;|oiz8ZnpYa0{dGkdO!V!CsXprl=)XP#PRPqSW zVdW5yopjo5cpGxB#|UTFn~b0)j`5}jNMbg!Q0mj1J7N2dkOAkms=|L2ilSeMDp^7>T(f(;mMCHBmF1Yq# z$lq&K70$HU0g@6(S5no!p!~6nDB>}tyWeS6|4$JKRsp}U;DI-S4t=?*Uh8$!N~;|R zQ=e>1nTq+SvlU*5NM|R>i-fL}>IHlWS*~K$qAXe*weoW|T!RcX8vpj#7LYyAW;(+O zE{9-Auq;|1Q5znFusTq~bPX*LKRH3vQOk@8DgH6K`vX|lgNs9-cxQq z&knA31&>i^cK3&Kt0oDj{v0yK^vPf$7b=yyuBS*@QN3~EMuo%#|YXF?2+F82K*>vr&>Z( zfzuJ$hz;wGG$k%nbJpf_`pZ^@Zc$6U0cr-@bF2XPq0K9p&-+h>$ayM4K9bwDd}T|! z`ApMuP+G4F6s-|rZXj>jv;}g^0z@2d{ZYX#EiDj^wi1Y9f6|5&2i^lOhS2n7aW*6z z2ZGNh>q|WQWW{-aog{1P8bbIEf$@y!%E(>-yx2Jz9YKxeL*QV8`P$u2im9aeL3w=-cQ;hea}pj$;5XW4GwN_-%H@VrmZAJpa_ zlg@5bwhWcUx{R&GuDm?a=xlOv?BhP?>g9p4O1{;&*_D_UppP`69C@&|0q)vb&$27a zJVP}-NO4Zrw%~*blApAvtFX|Aj5(>{MeypRlPU34$^w;cac{=Jri~D=k*SnBn3tNR zHlRjdYh^1gYQb`flIn_ve$~^WEc8V z8}M|=_OFvzYA@}0&aRP@rB?DeG7ah&RG)zAn+~UF*_jdQ>V~-!{rY(I z0I*@DoDV)K(268wCLo483QAUmo~Do^e01i2-w=xMffO$iTYq*D7gudd#48Wu(&@7o zmK7WX;-UL`i^skUE%GAJ<>%9<01*5?)pvf1PCajMA<=#`!Y*hRyu+&B%MEpEH z8^UZS2$eBER0+cDJe69r5ADuY*eW;{vo}ja55aweH->Rpt~V(GB|4uzaEs?%eiR9w zf+HmMT&-$D>mryy^xg4QC`li05O|;4O=f4&ofBJ0gR^Y#O(cf2~=KR{*3e{z6lVhbX<2Rv< zx|!H~cvO9S01_{r_l@Bv8&==%YG*T;yR}!rwt{!N<5lkgjXYZHJM#bYF2HO?g-3xb z!ni3g8G7JqXqb;zbjU^>cLVu|45C?L5|35iX~r_}&2*AOmQ{9{qhI9PK5vq{jh;X&A2n<0KVj#g%y z4Dw2+ba>~DvZKsPVWOqNwJWD2c~fze>v??$IwI?};u`rUmD~w~8a#U-Zs==|1HAk3 z+##sLO$K%qZ(xh#-B#JM2X9U{h`yKklj6vLmG=kWbKQ@*;{w=RtJa7ZD%JPiap|aK zT_u6zLuxJW8aQ%`#LqW$r5uV5<Fj7U&R_I=9ahGXiNO%n{JfG zq1usrrv9W?m1}$pn96D>(2o$wjN`l2JX9enm3t9%M54btTgNnn;hd)$qy~kgtnFkI zmj=P)(d$wR4bP%Gx4>T$;@0MuKLAOe(h>(v>-!SLD=F?WGo3e&Zx4PQZi_^E zxks7ofV|^>?jwJ<-u2O_A-A<`S|NwDYe3NLsT4=u7dwI^>sKMPhXDZ!_619 z79SNxm_);WALaaV5Ij$yF_f$Z?CM$jrb-Q;+%bsz-@n2*-^0rUgtAmV4d2`y<-=Xo z(~+w`RAkZF&;)(N(a)YK{vHAO&d|DfkCl^_d``K!Zxm!1-W0CHUfqsa9Lxc{ctaPzN( zkAuQl;gxq%g*Jmi^bwz-#ItWhG;KdL<41k@Uh*J?15kxm2&=IBhuy8e;W_`laTf~% z(7%KGIiJ_GefqIokei1(mc;xOmhDJ(%c4|iY4M1noKwR%5XlGv<;dHa7DQ6=yOH?ur*op7puE^8Alf+vy)1qU4J3LF z&Q0tfgtx9Yf_CBi{9O&x|J=3?q(PV13|hd4m;j5I@msVy`0$sy(5w?^m7?_nRuk_N zY$ewHVWXl`fh9FCY|sqMuvXhn#Fuv!w*?DLT^`}(&8zG%fQ%u00*!~hC%NSl?9hmh zzoigLKSYE+%Kok&G+rWFMJ!T*F)jikxhNpbk5`M$%C>(|2E>T~D z7&KAsLF8rBD*2%hG=Ktk!{klFuHBJ3$Vd_Ip***Z>-L}}6M0C|+jzv*|8J-8nN$cM z6E(UiLpiXQdoBaa{wSyk@^EPv@!ZC2Y-t>TVy51i?x9E5?9Z&_tK9Ge6^Y*A1EqCv zjm#LY4&#?bTy8BNSW;kp5?*lU z8F5uP)Jx>S-Zp`IF%09@x7jf=KsxE10aW7xr{u*g{$V2goSWz4+r-gOB&+w`cl@#X z5@e(@n=&dAa15As13Dyh7vrH)u+F_KbI`Jj2Jl}r$UXJ!2bG%?DL=Z=5ETWOb+Gr< z3wPyz(Z_)h8$#vyxql!KWy;w@tIBv9{STIE4?a3{VWN}cGfJ?a_ZWKw5IE-?FShLV zFBZJ-n5mx4E4Ds&a|H-)eZ8FA);~%oSwbhV51*Wu#Lav{b6?0qCKaYaEQb#jG6jugsko~IFS%y{*W z0l-BY$9rt%XAB?Kc<#?IX=0rhRP|77=Ha_3nm$lYI0Za+66TmxmW zzO?Az!z)Q3V^T$0#GzASHJ1gn#ecOZ_1E!9?~F*zF65@vdOHs9OkW@xVPm{*>3`Oy+39Q z@^mt^!TTlWGmEoum2BN@QXu6O;LtkpS*E`+gv|)Kv(Ov3w#~9EA=jOw*9@IC%N_Y% zp}#XceZBu!Weg>kb_d8=9%Gx7N=9P4M~FV-Mc0A>-~%@Pvj!%}XqF#M?8`H#MXkM? zeSpn^D;fzIY7|ILbPa_RecB`}?k};H&A4u{VCdHq*eMA@2Lrly1EAK0yIM4umBRtd zPJdaT`9u%P@IPNi7i>Im&WnLgUcUoK4_?o>6$S;LIm$IQl`a^9ur4%C?y2`CAxe{< zhwc-xf3`}he>#Nhd0CMQ<)FwHnF1P#CXa6snLs$6V*DZ~1-^DECUQgJ7JHN*F+#qN zjzDj<+H@)R$_;sc(Nm00GhMgL zE(L7c{YmGko;EoUC58OzsmY#KQpHsM2gF`O9MEXhTktGxM(Cm=r|v`B@vEgUP@vo2 zRczjVH2Pi|?D;74s#?No@9Hw}T4rP!uW7?*n-Tn+DkcR?SM4Jc-U&Cz2cuv#?6gBp zlN-w!mMv599XcOkHOuUhm0T>7WkP)mlt0A2%qqCCGOw5y^12N6E)rN755JK;4eB!{ zAKyOntn8C4wHc%F_#pNg!4em-c zI3~S+2~tj<=Qm+siI0k1HBVo3Qg#tYFr%qB)Nkd8LL9M{T=A+(9jS$JMN+SmsJM|U zD|CH?Yg5Reta=xUkWu%TrA^EoVHo6H>xvI08-=#XK~r=``UkMGCnq?Q`~_ECRRUvN z^!Z=qN}W=t=5xH7_&2+?_Y_dxHUTTEHF-O7od-NdJE@mJaVZjb)q1AW(OzjAOxbqO z7mEV7+apj>^oJ3DAO5eoQv`R(my*XxhETO=LAA#_yXP2~taic|7j(;{&!fOU< z=O!g-2C6|yd#&z^6;9dNPasFP@}}Xo3Ftf_?9(Ty=1h8)Dwmb%{7qGucX_$9m5u2I zx0_46R|w_sXt=B6a!Y9M>@9bjVjXA`bex9Q71s#`=$OcI4~Yu&x%_mBFSH|lNQZT( zK1b&OpYe_H`*)+3ru(CjX2Gd$_>&5O!3!%;P9`SQ$Q%9VG|uT>R3xgfS-4BqGx zj46>X3vy}~)sLqF>2Ar65FsT7=vrP`HU-?uZ;yF66}vM3zIow8yLDbCc>H}C4|Yx| zwbmm64~UoSL^uH++Iz0Vkk!( zwCyQ(wB(sJkqJL$jZB$Fz;<*X)fAgz(&x(3#9|&CJrM+ao2=E*Cod~_(IsbjN$q}_ zF%Qj%2d5mfUJA20HF#3leO|8KvKHQF{-6&Cea5g{1+zgIi;h_Zi&~AgJ~QKsxX1It zYLd&p)3w0k1!*8@pg^k(ZAysvxmrfIw&4>E0;+{3^1^;(tRQucAARk7uUE*T?#;E+ zps-ZSwqBBFfb-&mMzmR4>>0;>Fp5HoQAuGETML~Sf-Ju^l4@UyeFqHIXy!7pqEA2L zKuXNL_us-~*(tOs8_!X75F>#0sv$M^#r;+n3yl9HjVl91Q7iQ;V#47QZ~KfwuF2tF z%5MN(Lt4vXy;RSeAyf@iVz%cd6CLHb&dIN()@^j_yBBLc>yVn1QGc`-N!7up!SNjm@b=N`^5ZZ-Z02+ z1Wpvpfz+EkWJ2;VO&XkEzNX58-jSgKT3L*Ak;_mSN9;7ubyNPRA|bW1f=kaZ_Aov} zkM2HBk^2|g7Z2C!+76Zc*+}iN8%uLsy(> z7OP(h(^oQVbX_=ZRtYNNT}dM)>cwxKPKgT-Wyk4 z_+p|nGdNM9O~%ZjyCIZ=zVq^QU$F_25d;4+4>D#h2;x3+4iCD-~uohJ9h zWZ`SxWxQ_YT|Oa_shNK%ZgPAMLaMqO0uz*pdqOr$9YE)~=cPEEUXObm9Zs8>eT^Yc z84poW|1J9s(oRske1Cn!YT^zVucC{Kp0_r@?FoU`WQ>E<6U!wx| zTdvj%J$(emZ^l}aSpbbIBt^d=(0sXS<+;^hdFSb^^-uTTL84RF;Yq=HH7|KDzIjMWyB&W!_zpMF93hk9{q7%bG-d^>T1k~B3 zgP)h(m)^%J_CM&$(N!Xk^lDhh$w(u^U>qc;!`HO^g(lKk*^tzA7Yim^&Op3c%*`Wq zVkX?ob^xM414Kr)!UuJTpD>6wsSM$oZki#@B(qdfzu{=@wzTp#1;X^{Tx+N+ zk~oIUEo&HP@KXC3WVOZ3f`EylX#~!|p1(mh;SF zmJ6@rL|o_`1(e&?p!#E)q@V!T$?0q3%6#z5K4T_O(#sbD>bomKjIxX2Djm)Fm7x(H&DbAIoebmvF<(Z)hKEvwf$5-|jac{3fl;tH6 zBKUf@c1ZYvm`*5SF@2WVl}^_1I>nMBP*$?`79CzXWP(nSTR$1`zveR65Tw+j`NL&^ zM5*872}3Enx)*Oka(LV%MIn|h3yL0QJn>C@NWROao~2o$u9Wl|(eu3TPxPlc)>Dv{ zXkYueXKr^X%g5fMfs$+8Z z4oRMwOd{UJ;1GBX)EK7Ey`+=ZlC0H*A~7!#k?M_#{l$v~uA$jg(+ohucRSoG@I&Lmpe0cRJeoQJN4PlaYVh4(JBrR7cI{f4(N+s|5Uzc@$OXs2c}7hz8is2 zvy173_t#yXRHcuWE4@5q11#5XoQP76vC81v9rHLx&)=%>$fr-}G zSC)|((`vgh^)9}Rvp%R`WP)jYf`m&WLzQh#1uw*lY`4fl>#di`OoGMvE0WVmo5Q+p zCcM)ckhZul8*G9>FWPai%=rU^Y|MBZt%itR)z8=E=fBQ-L;v#H`}1}Opk&63+6YAZn?OTx6k6iPVH!A!4Qd@xcPCfrs2Z>m9wjF&F<%Jc#KehRFa=YDgc;xB=7!dre`-Cc+}{-!FQWdPEm4Y+~t6&wdi!ciV_`?6d?2_Ex!Md6eEXB zJ`P?on?RWqDeoY9W~b;X<-gw?NwEnGR(j(Gh>}vaJ3%}K*e3gOJkG&f4vUrwfR7AZ z@$aRIdKsUdO!ES0*>n~5pPn=(y#$-pc2ey<~9)YXlJcC$H2O9yRZQYzP?fvoiuB!Z~M6? z9^T(`K1ovi062*9wtB=8@#OHGaS-#{57EqQpZ^|=!bj<&elx-*P5Y9#H;t(LyfcRr zI%H1Z$uiG?!)^b}JUBtT3A%HEYIBcr^{dB z(yzLa)k%9X^`2QZ&}t`*xget~Z80j7G53kt<`ia&#q9FBF8Aa?-gCN#HP0OSgaZGM zz4wmidVl}NZBjy3NEBs-GDB8bku9T$?7erkimb@a-ehKGWTcRpnN9ZId;YHHd5`Zo z=Y8Ji`?-C8|9<|u?AQAeyNN43DtQIYZShwu%b zMdfidvG^j+?t-&Necy9nF=hbp`SIePqM?vOT+gcxtVFsnVCFe`BIkM#maZ{ir;LzH zKa!IzbKm@t`=72X)XT#hEzV zw4anefKFs3tMs(FWcA)+V6x{&H8I*6aemAFBSs?$s%yk;)Bb!K3=9v&5^zg4`LQ(M zip_v+Zn1Ykj^r*qy3hoSOh4A&yLS>`l%|GR&*;STg!4m@x4mA9F3EAK;nke1o(HhP zl+3!V5upm0AL6JUA0+u)WCv7V?xU8t%B`nmZ43lP zof@ieJK8PV3M@}UA43yfIs*iUg9o+OT?bWpe_X#t3=WVb50 z9e3zvl@9l}1$Xq~9uMDU0k_FZ!xM(q_hTn+pED=C{}W&q)<6}Ce!!p2@<7b|7VcLj z#WX(!2Nz6DMtO1bTU})r@ruHpI53rte!3u!5c44QDgXJyBYdNU1henc1asb&k7Mua zG;J1-q??X$v5R~#xw3_syEUxH!n812G=TaMREdTU2srBth-Qe@By?rm`4@lr46Sb41L2p5sH=MPm0I63&_TR znXO5!GJkea?m@jBMH(_~^FPX&e^+tST#%TsIYcz$4t&p3z#nN;tm7}}m?L&H@QiAN zT`v?nVw(SkAE}u1C^tnG&0S&6&j=-wFOi`klDZ^K_gI!-^$K#)zDHG~pi-%S-o>^+ zQj!`f$0cP6-+#k>JtDsVZtDH@<>F4Pcn#u-lv|pMcT-bx)mmE86tfj?P+m|@zK>az zdnPK8hR2;|D;_U>+(EO5p$Kr-7J?BN`rETOi~1B0)1<%H;s1(3|AvFM%7`40lNHaf zCr$D5cW_x<05V&#&4Q(7XIr=|9(Ym3iNi)-oHI!FmPb%sE@3dq@odZS!PZJHpRy5# zhPpirwsH!#5IVVd;mX|~201oM?Cg&tS!=U;B*p#R%a3JgTGD37><{0NxX!Gl%4$&H zIf0UOx8}tBA8N+mSH!}zxEi2=H-IvoQTBsFp_Cub(c!_yuocQqlpG4w1)Ji+ z!<7fP_Y_BuMM5X_t5hkI(e#cz@5IZR3vQql>g90@j{0ha-#FGfJB+Gr!P?IHET<2g zMKa1Q%=T|4WPJx*PH+7a0)*B>H4pYYdNVRm{S=uO>>cO)FJ=&#%HJxt6d!w2q~H2_ zReG?Ib3!?S>G6oMuIx-Yw?=8ys&rS93Wc5&y8D5c$V+#cFOq96X@Kg{+b6nI0rXF?)5c_U6%5bj zEvW(YBU!(E9T|(H&L0%a@PTd35@fwF1&@!1p->|A*J38zY4yAs5@Fh6dg7@wCT9RZ ziD$>(etv&sA_wW1w;?y+ZmjpS-k2FE61h~LIqQdfljfgqJ%&#K^T^RRBk5sxi}+I2J^RMS0Yp-2&tB$)Yd|(v^;7{X8XGC*K623 zo{Lix>l~+6zuPak7?&DMzIY+6LUIcsc<+J?Deh z7w&Y4DC?|IJ$nJJ&{()&$1kD~k6n7UiOs3VA6lekib7Lb@Trc7CpyGKujGVYML4v! zc5!Ol7cJTjuT+)cMTkGvTXmpItjQ}QzDcd#;PCm3nk+5D9;p{=89$MEq@z`7?6T*r z_2op>4t6jvOxAkZSHkQ_57R>~GTNP|zxn3wdPsYZehn}HF9bbWjG8A~YEhgvH0+*FL!jxE=A?4!+(MYIx#v;_@LFArjo>V~Us$#MD13GP}=(!0TZt)$AF`&}R3b)3x{B@CiHK zYQin+E?xk|*E}D)iUd`V__AX@&I+-*eCDh9tsaI7D52CDI8>(#i31kXQd5>1akF}k zCWP=;a%8VA5`1*oG1-FIS1~a7a=6P6Ah4YFGKSU&tz%CH`*{U%XzEl8ED-Xb+=zci z6g|;9=v>GC;8#0KD79qPo7XKC2PXZ;XoaMsy+0sB1-XY&LEExqw5L1b)xqdbY<}m|h)OcC?i*D!M-Q>V=+~e*OrUx(; zW<*%heb^r@MhCO5`TkOb{dY@4#eSQLPeq`+;m@(pXQl9L#B)k5yWS^lE(lH+Ti;h1 zCPy|FU0#lMiSA{$< z<$P{0@KstsgAz3KI^tycoiyvmpZychKKPvFf31o{ul#p zXUgEo)}G3-dv!=cvF&P&1+atD}Tu7QjVQ#4NXja*i3bR3WTWCtLhPhIgMOTK)@$-}f zdJpPJlQJ{% zq263|j-IrKL@(r8g3}1KG!^Coov8Rn1Q$bo@g*YnrGujNa5qQamI6WCQeBGdWjG5j zvN73o6>syG#K9=n42#jqvMq0M0+p__`&>F7i{!)tIcYp?~uPC zFv1;UTrz3Yc(Z&|h6^pPC^Q>PqmBFV4|{(-aB%k}4D!o$1XcF+RpNT(wWu);68ns3 zpt1^O;>GQ1jJn*5OCT3pO<3o;t>NLEb>P?~=Q6D^aP!G9tgHi@1?7{_J!j@Y;=3*a#0 z1s%u8habx3?i4R2%y~?Bq|&l9b3k8~y!)y*xW`ijaiJQ~kgpVYIoY}CD=SW@Jn`ae zna(T2BQu)mNaijyQpxUJoU&9p3K?on@!?qDGr?w2$&EIr4{|(3XjnMjhRTk?N5>;mL zN0R4N{h_Y;L@4jMU@mW&`s-c=VX;=+dx%pPJcv4Jm_Y=Tc!7#duhr@WySfbE_%nYY zcp{Nn#WVnH)!b_}sw;|=*VIIwS0=ChXtkg(t`T6+nca`QG;vUyOqn-XiP_QudY zndUL=zqOP>_-u;A-rK5Pbx1-)60X0G@hZL6H19X0kD8T*TDsGPuaMZ^R)|akmq6^< z>0WJeuC9wpWiUHiHqk3~2F5M5+>~5x1kHnwh3n*K58pegq=2J`D?I37YF{H008)B| zPwFk*Q6j}_>`@hl9;S{hy6ZxqN@_D0F0RH&|FjpY`RHB1*KpOri`s<;fgP9V2!hlV1kXcSSL>~48fJ6v%LGpSPImv!+Ul80%IySSK(`{uV- z@0kQBR7GMv?e@Iq{vvuBx8q=kt9HL}e z77$@hFY{qdR*-_we-T0b<0PZ0#aUkYo}jfFee|0!PJxEP6J#A3pA1zqgW$j#!CK<9 z9J{x_si_{Iff7mGTLS$c@tk7jMNAyAR$x+6a=d3s_yREaT{u4YQg2V1v$IvNd#iuc zc?EC7a1x8y_zZFhuziht2SDzA*C6v9!Z{%xqhNo^D+Gq(w7r!RT>9evq?jiR1X44Q z1@UIkOT_I%eYXlgDd z6re?NDt@dc#z|XN?CQYf3uxPM&+_qSPtGO^rpDO?6Enq07EuSx-U>?z3le{CFNUfL z^y%lPoB?EP7iYO_p;#CSHGaT+rY>K`Nw%E!80t(shpnI3%x4MYoJR5luIW3OaW+yD z6`&yX_<5oIXO=<(jLmDfn&Vjl^PC4f%X(!q6v89&WY%)D5?nXG6QioE#&^3f?r($E z{@XH&a;eGVGv#`Yi+Otg$&lMbc%pfQyWOkuA9}_4>8(5GK+sTnZ*a1gP488*oKc3V z#YX#6Ep4swb9gO%RgRWN4@X|h{3{C}mHS!gs0qO>yk%lGD;D*YlM9BYu0{P(E`Q8Z z|5ItI&KTly_%64aeGX9-Z+dzr+Dn!bG2dl!iKXb<2B9O43Y%IXP4u3`y~l;E&~^Ek z<1#2}zh!n`{|O-ZhzY-aj9*C-;Zt4xcw8z1^#PD1FyPF=t1hDw9v0opEASjRRA%fu*pYo?bHQXY95{2BfQ87XdM?d5@1jJq|G z|Lp}NVW~WN^0N7SPv+a-E}pjr$HM6la`3WDU+)*7q8F($P&n;%-jKY^JnDaOGZl^q zw9A@>sUBaz0rOpp6B?S*H3y@a@$)s=WuW5|s677{sqDgE&`hwIo|ffigPKtx(%ie? zQbsEm|A%;$PE`^03aF8rPbZ#%Vy=~IRr&AvI9ybAZT(kFbE4q5piNScK6xiuMvZB` z@AcT->Jc1DD`Xq2P77dLDi;x=xbe82Hs9zOsyJ)iG+_202V+JYf|7C`3FR{^DFPJI z2B5wr2R$O@WsqEi-lSaQR%L=aSnu>ChBcw*;O~`Xe*8fXan#0sUO*I3OT=awZ7R4w zLVbc4KJ3r6{Yx=c9SE}!U?FrY^=1A=2;?7;K_)2_k>3CU6CNBoFt z-jb@-z|RSNn?))ueTCy|RdOi)$ZLFG>K!PKkRxZo<`ELX@R|h;xDL zNP{3GBUfdJ8zN$ol!CnT99Bn+(F=|prRI9y20MdQDUiAil8AT)gJCH5Bz2}?_C(U; zw}4@D0Ibgo;OpPc0f61jVR1SYMNr?}KN1G>(PX8TiH)zp(tZzpf*3vrc)alY-Jn_9 z9otLyc~j*67~DYf8u<*D71pM~&ZY}FX-34HS>w!@n3dzyOOk~0L&od{|^m)ZFl@~0~? zhzrxSDfGz{8xeFk2Dz#jqQ7I__|{T6ttA@caC$U;_8y!XSUDs(Q~u%=p8+9jhhl#D z1Ly+=1EnmgGRQRTLJttx3=$rltD=p>+h~uFP4fAo1>03Kx0K8n4%;l9WTVL;AgzxeuPUVi&UVq6U11r-X{JW@mI0O7)u9Al64OugHsWB_B084qt zaRa?Dyj?0L&7YW2$EPdY8K69@g^Ew?5O;)}zLjG9eQy15s4}QA_0NC7z1;ow{H%Jl z(?g4BsZaPd&+U#;5df9kWW2Yh>y+fBYQ(KnagV|=5ISBc3D-}wOLMZH7&{NO<+ zTgk>6roe+*QS@LS76gl-1Ujf1aL!ZGjy```X}1g#tTG4Kx=8rOvlLR>xIC~+4tDyA zyEuNxRi~I~;EF$DS?}zWbO7oKJ7#%0FHucwEy^43&(lZ$aSTsCM#WQ}k+LJ8El>E{ zY~m0t4BXi$${9gKyXZy|eb+h#%v=R{&GNJHh#Djgt{Rc`h>kNLwuh2Z;qD$hfsieM z*z`HT)OirTlou>g-br*T1vJ56l`!IFz@<~n_@E|M&%;U-S#-KT0|Xx)=uT1(#nFwt zsV3XN=T&bp2v#*Ce6oS`Kq*>@U`|j#*c(+nWzhIk7|Z22Vl8pIAO^9FBHC{pi!MI> z3B}oTB$Uf;{ztt*95`s-x2vSBbWIwnbo=EXQ1p4E^&e$!dY~-uk;Ou|2>}Kt-2$!hVllZI3KMGEHLyOP19xlb(=y_2EJLC< zPZm=V_!J1lBK4BvmD6J&4zAqFAlWBdj&6DMQRkZ(LF8DNXn=dvnCn)%vQI(EoT32> z=G_bxu-6#*)EwvPX0LQ2Q#=2tYeT-YN%b@(QyJyTBL2T^W6a%S%}oxJJ!ED99n)=S z<%vba-dv?L6~eS;Sf3xzeJk@(y`HZ!q3G%fj0zR6=Rmlb_WLtyxht##Mvx!jAmmYX z`jweL67@OwR=9An{7Ln3_&}D?{*XRiFaPuIlh`gv5dk=lO2|tPiMh+(DwRN6u0};S zmej$ZFQS#a3LzNU}BQ}{k9Jpf z;}!{0T&jfc=vzpq#W_vBspsDTskr+26N{F0P`h1_y+*4~Uxma{%$8`Q|07@HPlXZA zvW(Kj*#@<`zH)0?GDIew30prDtnt^xx7qQ}pbH;%n7X z*1iVJj@**6{EX|;R2Ne8L8dBRuMAyZu!VnBZY44zxe@a=h_G6*Mi}QG#eBZR=;8?2 zO8E6cWAGmAUWGdQeY5fXBZg=>5jUomFUbA%V|SZ%zJ}PZ6d1M4y&qV?g6#_4+y^4$GyW{>f3H7h`6M-Z$ zK zyghVBc?KpYdTDhDU=(}Fj=OLjm(FCUBpWYIktUGcCjBun4%`ivgUd4T;2et>Ny`!+ z5i9vJhoVZ_HYsK@WAj7uDB_T`T>Xs}0NBkM=`b7e)IMJSX2DkmjMpq+yCr*zMJfmU z2!n0lpeP*cJsxh}!se1mOps~IATP6~Y7mLYxR+J<=D>7g1c7Q%F`05hwg)Iv}$Uj6VezT9K&mpbfEKx=kG_c3Lw=#0|3c zSM(=9I+_ziR)+^6G}ld)+ls8!rAaNgBZshJ%siu4@6Zlr*%g7d4gR9D|>v7khXwrk)+tikUS^tm99zww@FV(sP4^BscjKs{Wc*DTBv+y5FS8%4apOl-d$^&T^@rKa|9n}(>^R}9 znnU*v=Tm3FTg(n3C%M;53jbwD+CLj)3B))?=)n4qAT{CL^ULr#BaNQ{6>#>?7`9+I z0JAqPQX5`@D4cqmXNd}haaA7rRH?uJjQ$UOSNF_|+kEKqiTqzK`F~z3UdOkPL&g1Y z0iHqfB)K$0+^@L^7CzC>~9{RW;Qeg)bW3cs8nSD9B)Uu5(L^a>$F5w z14jeisvC6t;)7rQc02t0=l`1@{pa7|NkoIv`tYA~DroS}$AR@HLpq9;an%Y*-vD5^ z9>lb@W8t^d+0{DmIo#_YSWEBzpK+i6y!ij>lY}{uYO?E2$=mrKLA8JNv;OwKp%pq||Kl5)CD0HM91ePJ#F^+{1<_8K z?GWb^s^jLa)&)+332soKcK*|i`(L)>Ulca~{QXIF3AN`h{Bv!EoM9ja`m65y&k<8C zk~Gr;%J*a6^-ij_vCqR$7q+^>4$c501%f0W^m|kOh1KhC9}TIpyb9q(KF9yv*x>)` z|MFK4<|Uk@eY@!&j`hFeH~+k5{%!mt*^w|sxuh5SzrOW~`eP*LLxX;|QSjnEa^EhvIe)T8IaV`fUTQYgBT z4w2Nz1BlFM18K4LZwh(2-59uiV*v=h@q#Ox8XiX*aeE-x!&(38Q0aNXKY|8ck1h>> zSsbw#RUB`&%78Ac9f~@Rg|@hxIhY2JlLe@pHh6g6^+in{2A#5PAW1v$HLrX>`l;K$_SE7 zl!2jx#p}NDN*yw-hb9LT^lD;A8p99$?YGeY;||zko?i_ji3eA^F_bJu<9Zs=kW&Fj zQxNZT$O9w}_O~Ja8(6_#|LNYX6U68hMb!rKfuS!mlw-kGJGFz^<$W@-!ug$GZ?uJ9u#jywiEzpRA|MwAvBb z_uvOfH=AT(#1s#JnXzg9SHi>iWMog))Qx3fd9ZB3g9&rv9D{Z`!*Okl(YwIT*xt@AHC8c-7m zaexO9m@<;|jMdrve2{S+9soP)2+%tO{PNZ}AZR{^I`%zIe(wFBFKE4jy8gVU?U0L( z_>k~q@BVtZoQI>@8iob|S~=#$kfa!cNSPBpH9@i&*4{@jS8EH>8*p2UL>p?@eqmG3 z&#rA?-IuagM1h1#3<5tG?MFg_cqEHtgzHrygVAQ*rR6f0(GMD>5~D8d6Tl<94`7dB zZ+*Xq#Mvad09$c>c5A8)YFM)eo`TiJy>pOk5@Gi$ry6S3p5Qeu@;q#-|8vS%fR-{*gw}#0v8mISg+}!G!8v}Io-(??73=orak{?1?Yw#ES-~}ta>lPT*V(-6FJXPQ1zf0F6bozjYK%GI& z%F@-;9l!2Lk={emxihjkh%4Am-GmnVXBNl1F&~f?bzXs#Z3nlMvo7BVa71*jKKo?# z9BzOdQG6~b${fdBc;Nawi}%v-g}6-nIYDvBe(0HTLBvG}7OZJ6;m!UkiZ~h~!`Y5_ zQ}9bY@4TzmOppo*${{gmwa1$QRHKbTnP)t0TAhEg+X}m)y1mJxVXn>ktP9O{2Cx;U ziy8t2X7Us`66u!XH3l$8#X*qmlNoj zKUx|yE8}&%UVhI~kazVT!`uJ4Pc~EPz+T=489fim`!nUJV;|F(ut%{?!$*g^ZM~9& zWUE%Je>vZdOj|Kc@MF(CxDhjyvzH$9#j^Y{jjC+86Zt#$(*r{e=S{t*=`g}$F#Av( z?yK%N>qc^Mw<|;}2-0UfT3I^!88RjiW$l}*Cdhf5x~xqfXLbL!aNMCklCj}X0hdYH z+BDDy&%>|$g8WA=o>99n-bp{&OEu!phPrlo?DnQ>buFlnBa}uz4gDr^9+RF&$Ok)2 z{ThEV*L13rT%n2^D6NqG2Vnyb_F~6Lx8FLt;YjApQx_nS;U@}>nMfc3lKZ;v>rp)g z`Fvy;TE>an7?um~0R~T($;k~L{T4u0PR$qoMg|z7FZug(LQ)O6D0$1QU7S;cZf^Ne zJeWEs?w33xcoMlB|ByGX&Ya|>ggA0EreV=_a@DIIUQ!vX;x-%5mYC~}%6${Q?b;RF zeDLl$b&wHx`*Or+OzZY77|;E>{2K3K$b~gM8%g?9<9YK{#$pmCUcTA`muu zSumK~W9wy_J`_YtGu?JS-Y3pHnooU;ocBfMo`=LF&GjUSuA2|%7csP7-Gy<=ULMeu zPCkZzn34RzW);dxM`Bp&JW;bxb=x~VAHzLjmS-u>=Foi27sGe9S7_A-cX}dF+|at7 zN-8hMOp)TC0>WCjbHa;~kpWtaibl+_5$=mW3oH{Lg<4tS4J&M@`HS|aqSAl?o2^Ie zzIDibxN@bkAP%SQdb+uZ+$(c(l68LUN?Rn**W$OK&%Z1C;J7#}+Jt-^+}&QXtO~}h z2tNA@5RBJfi=4@&Kc8{DapxZSsO!tSUWe}UqH~3V!kX*JG23S9aFNcSC4RgAmMs(^ z$S1aXDYcB8u-Vk3l|!fQ&L6I4VObo+lpsR$GT#G9?f`=?AN_}StENmjvXG19eTWYl zJXU&}l#wxPrt;Ep#;H(dio3UBq+Ny>jva;pDK|JWUwGjKz~bw63|L~;bc%#@)dK*nCN>hP>MCvBpU-V1y5GG{`KTNzXgqeT# z_WOkB!@E;&FMWh9N|*b*U-jV`!5TPWt&-uc820?2{feg;WQ!cnzhIS0J)hbV=cnsJ(K%JiIeP2741 z+fVS@==wjEu}j+`#{QF)yEzjLx6FoC++P^I#i;k^h$6AojDL-`D<~#^3mW*KoJk?Z z!G@{*&7>7cg;wNhUJmLaH0C%uj$?B?N7uj~Q}s2=TpVL_XnZs9veh4}(vVR!q>QBc zG@^+_uW*42!9L)jP&}8ZN|ubWTSJCiVuJh+af>zdh+(v5Fn{bsNzcAZfQ#&|xYyG` zO@uP$c(H5tVh~W;fttdPi|f!I3zB zAC&+q&DW*5;=V|&p7(^ycFcK>T)(Zw@`j3b`Z)gGT4$?T7+S7Hvy$a+FP^axK=Ua)C0c&JGQ8*_<_|gyN-SGLe3!x_gkmfJfpSU-AuwvDiHKj zbV87fPrSIk1q>smml|$!x6mTJT*8$t^h3BYDdQB$`?{4lO#&%6kjS=JopP&RUmQvd ze?I>Jas9sub^xJGV^k~Dc9D(0am19h0{8Z9Tq)(<+d)#v;)xb8Y5V*}6~s{*aLT_k#aD2X0CQul z5!A-!@B37DZA(|O@XVkFLQ(@no_;$=AE&+j&DMhHBcDezJ%1|{HTv7mn&v_#=-m^H z6knyhEGRg|VrCDP6XNCQ3=1M)r*`04?JN@@JGHnW(DaY(_7T%<`^8@Q*bz{l83RyT z>n31cM&$vyjhUbNA@^U0bHd>Xy3yROpgiPOze3`T%_C`G%hE{iJrR^C~jiP%G3Ggp8E3co`xh16=M9jAJ3%q@fZGcNY2w&o~y^*t6P{x9%lg3R`yYk2YvA8r(#@mEjx4GA)Z2$uATwZ|PWqdDEdCQoj);Nvv zSp@MX{x&Jr=8dOfR;3LD2GPYSNK${%_@2io5Rsrvi!+_6n6FXsW95+AX&c|!4do44 zc!tNZ5Qqa1lYKoeDXYfiwkmRaTpQYl^9`+#2|drP?{o-yNX#Y`DB+_!^J741{afhT zgQx(mqDQY~!}YL@(=KD0iwl+9ROc!#;iGu|z$;#T!*LoZ?E3^l!K#D>VZFXc)*3fn z8O8hUQ{(Aypi>0x1pnBjKqTsI>&dSZq`;q91VaR+HZQ+HZ)9?(!{Nv>7N*qNoTUDT zZZ{+CC%6MXKy=3UZ$ElZ*6A z{Fm~g#2Z!kD~1G$Q>V*oX9l0QaQ&gN@^T|Y_)KKBgOxwDc|7on$T2Ayrc@xwcjboT z%8))1RGccPl^=g6uhMR*zkQbk{`$bd!skACvmk;-Dl@@y_{P57pogtA{SNvcd_(QN3=R{QrT8`&^1*#RPUS$X^aXk z8YRoJfax~nL^uzh2bWLEl9WDPj`|V8{z#BMT%lfF9Xbs8q~A))ZzCnX_C20?OnK*r zKm`gT%hLH7B#z8T&j{1mh?qENmry40Gk|IE2KQdd<^>CPrItq<$I0GJ&pdGl~ve7!Jd!05~<-!ho3LQkkY(7_wpvd z*TAN%Kdvi4Pmb90L{jCq%aE>%KO|d31W%yUq>mjMO_wigfu~qT+Y3jD3~<7W_Dh%Zcs<cXeo+x6K#C z&l#0L*O}ka6&I2q1!)YYX@A@4zP&1C(0KlwWk3gWh_E>cVrH>U{MmDOUn`Mk9Hv9X zx=ANia81q{XT}={ElG4HQnhw|Tzy;_^xL`C>>%Ci`2?bSanp@ue z464MgJ(ee4Zst`AM@#T3Ou{QRF6^uYi7}00tk(rTh2cot3w%jsG{Tf@Ux?Zdd zS-S82#?1nvrM5ZoRv;%8_hT{+h7;A6gr=&d^Uunc9m()Z`p!g zWqCr;U6Y+%2nvdqJ_Wt>`OuG+$ta%;b0*1LKY7y|nOegQI-J@S!_=~%T&5POSt$d< zIz^hqLlYD26VXE!0|Uxe%Qi)#!%22$NC>Grb*~%}G4_b?YbrMk9g*+_%#YgSXeYq^ z3;!l;2YFp*w6I{eb^RR zf$^5b^^|Y09ATsZ@vS}xc}o?hRkcfmVGMci48Q`2bNTV@V>(9cmsBmvq@JJAqtl5{o|QPAc01TnW0AuIcSk`ooZ@bCj&) zMSCcp{h~`if|7jtYWs&&6mGYQRsV#wnn=(>vTRqUQk(4V92-*=H2Rjbz+6KUp^_!% z?>9b+t-Wz2G0NyNAy&)6?}vIo>Sy-0*8?v*3OrS>&<}Ddt_(`+5&nXY7T+HVD1^K( zN`z4ikC>uu0`Y{5T%Fx?E?InMb9+D)p6VH&jCBnHG=^b zv7NV6vC^>=$2}aTD^vY#uNe`K6-(EsTbY0PX|{^YWwO(C-gX6_t%>P z(#)I4=SzBI1CAfNw3QvaB028Pc<{f;Jq0&v*-dc^OJV&LZ>86ih)j(!!T;O@ygBSOFyse z_kz6Uy(!To&)4@>-U}KS-Xfm_uZI!c!wn{_XTE2tus!E>$kK_Lfp?7tSFNw>ujom=h`!jW{U5gankR zq(AG*Tg!Lyt5*H>k-4Hj7*Jj;23(SpIyL@@jVmPTx_m37xFGW51{=&hW>tUb8pE_C zsO_rY>`L&<;2a8KG342%+?<-I)7o{goP?}JrQgLgX^%CAA_V08X zool}Q4SR1r4Pm(P^Sgl4R$3X=ROfA7{qkb-o|A`6b5sZ34>Bd$j>R8}I5=MQL7o_D zK1T;jtFMILUJJo3+}}|A)@>t3Ug&4@rp`MHMXd2{I-t??r+4P{(E8mu(`OE>zfE|a z8@jHmNZf_*Au2C0706+8|8i?0YR2w9zcFRoHa5XJg^UMU;?qG8)k zvb1qVpg-6`W5vXA-SC|?6BpL~V{4%cKb8V*>P1V*J8m5l$OhF{;Z5}Dn$a$b$lu@A zwY+fO90l9qf)PL0&B6ApFz&syJ9RtRuL$wkZ|~d}5WT%}dF>p=19dz7I0e?~^)FJ7 zACwR%46m1m5o~R&-d;~C2-9qji^_dOFO6X%+V=1qP7cS(Ga9?EsJU&B%djB%n!q`Y z;T;#c(AY+pEMIo6zZJNSuv%82lhe z)s{5ckJ}=a;@fIBb)Me2NqTa55G{4zTAhEYxdiX7`Z6RETl3t*yw4oR*4EZlSC+%d zVymd`yw8K734?33)kDj6X?O2W3)9eYthRcyrgvO>WX4nEa5oD2O?E!zIvH(0wt0zJ z*TMmvfLAQ#BSl^M(r-(961rQMuFR*^+P94UxEDt-5fDfXVL3_!YL`AWmPgpQy{cL$ zk?a3HS{K`}x9?ajTXT7Ye}?oIiuDin4Ql%ghk@t5V$Wx!nDfG~`p0BWv^Jy3l~~?= zdbxcxEU_YiyraHRw1cytfy=a9*-k6(6vOXJnr6aTrb zwZ%&l8M49GclIU)q{P(y;+y$8UMQ9qvbf=|2i?LA)LqMM>MEGRSDPl1oV6=fIf{*s zVm)!inoLb7bog}3l+3GVBl%T>5cK}K*Rg_s$w^UjH1Lq3K0S0Rp1O5#N(Hk7x>Wnu znk+9}y@RK{=H~3?oOd2M)9C6Hso{i6k|2;wy-yJO(SMMpu_XiTDi-&nGSZWza@*re z)_s8C5qfy0U-vQaxuS)?#4z_g_uE_EuhKymP)R2liu(7dKgRj^V5UTH@~cuTczOwk zqO=CQuJb~*q9A$og87ox0ryinlwJ*XdqoKbP>H zTHCDX`%O(OIpYJSVaw_bag|)0rDbiNJWoQQ%{R>`74=D-{3)KA{EcqEjP46DaKtHG zPFojDM_1s-7NK(6M@Ca=R@}HRh(#B|Pe3R-IuRcW?gQJIp;2L+nTI$5H+{m;#<805 zI3#bz(N$lwNX35!K=sDS%;m3`*Beu^?uC%!cik26&ihlq>M&KoOeD#miKCV!L)uHw{@;*fnJnw*Ic@@r|kFF z#%~){`!1<}g*pNlrY#Dtnkb^7EK zB}t!Nu8wc&q1OGZ+3|`mhTvunUL3}7;5d1c?g|#BNx8sAiuvi(hH*M_^VS(UUrQam z-y-y-;Me6z=eIRqMti&Hmu4-%Q4>j|UHWdYGeM-Q=%PjF8ga1HGuK!9#H(RzY~&x? zoSOVG)Ic#9BxZ%-E$w@rzV)-4C>ynVZJf?W{&J2BP zNAZc3{Fr6H9XegzwP9b$=U~n0h@iDP=Wc`J4B@}Q^OSn_lRjt#ex3dYarVCW#HbS+ zCmW}77<`kJfqNU+Up@(8sOb@#{G3>xNN6T~Nljd1ioy0;>C5qe;HuF&zK8~n|4MN% z7sFNj82Ke|^O!9>(cgI|Hhe*d8O%V0gK`enK4E{Q2L##l!V``@WhM0uvxn3mHPuD) z()w`u3irJ&2(ylOL27S9ykP6;$RqT?N+Jo}(crZQS^d#c(e_E|c-Tq{wydcg%ORNeCE1gK(#? z$#r7vk_hlCJ^bW7>lnGPRP^weE>-W_6~q~$_m%{gjP-i?ZW@cTxf^YwC+u<%nhM<_ zPhxMbfOdY^VOw(8aQN=r#654P%v-S+f8ON4Cu%_EC`aq~GcIqBTtcN@;okZ}IsCJu z`+T86qT8;+hxz?0hu&*Z2^z_rTR;5?kUT$pVBd{*1C43s3W{2o?kuAF7}}a$$7H8F zm-f))uH=O4bJ6=QBW2mKI3pp<$JgFZo*4bKO1eFXk;z<)pdn)qx~du`BFhffnh+^M zq?*nfC%(01lbpOrH=-;d5^Zbz?0wuWLUijIL&#>7VO=6|>u&a~td$Y0d`+gcp;^!S z)epU)F_Pe_*d#Omvyxja7o**4dGP}ST1fk{-Tqx2{_8&W@(T2v`|wFFj_AW>5orqY zIMQRHu<<}KCFx==BGH#wMQT-3l2?dvs=e#mLl#;tT=OIk+b9{|r}IWXm{Q3%`z|_% zihU)7>UDr;CN1BMG8oiq@Y5g=F;0=lMjMbQYj{!8{g7i;;hxP1mgOb8p8Q4k{ot1F zu&uH~^XKjT`s!rmIP|`1-DY{E{r<oIMe3|tHqaP??T0R&dDKA(pc56aMT#-c)LQxKa^PAzk9R}8`6es-nzzw~XOR;R z?=I|@9a-9YoJ1bI_*wZMr(=>3ZAkDOd5<%7a?uM~PX*crY^*!PF;9>0hS=hu*+6WS zUS>n=#>PFPCDOGrZ(PQ(pXAdE zKocs`)b$JC<7ymAINH2iAFIs89z^!0V4Zv#Kd}B7ETM0`CW0Klt@)y??%z(YAINzY zYl9Q&BxukWKj6@HX$XCgL>RGVa)LCRHBsAmcg?4I+{Tw* zrWPJ9UnOApvT&W4u1BTi;D^xq!|8&H587t&&Lj`%bVR%lGaLDvO3$^(uCtxCv|+ru z;=6|kyj%TM#@5gYx*{vl5+k^hlvo2?PWb&8RzS(~wzF*c{XX~g-tE^~)SchPSUhsO zTn^(clvSEwgJ0Ib2^9Kdq3XyCU5Vwj5OGSHrAH)NLZ>~OkA^bJfJOeceEZQQIqpF{ zulYF93hhf_Zyplm%d??Np3x3SY$iRsu18QBk%jTdiJei3@436*6v~tb3WvOQosz-A zTj!4Si`Q*Q&q-$aqQ2o-+txX<7Ji6H>Ru?pi`D;()5q^64vHT35U8hg3|NPrH{zR9 z1`gFYL}nATDe8x*`{Se=H?rl|bEL}Pc(#014c{6p$ZC?d5Ga(8C5dp6uixlS0dOFF6^vkHbQ-IGH{|-O zV6zUs(aPHw38(Ur_hMjr;dt5e5S6yeLdI2$ih%NFvmU=pX4~F15|+sUuZeSW3|Ojq znp2csAGbv%M#nR*Zw4OCX|md4G{l(Kp6mv!6nikp>8-$M$eTJ=z=bwizNa!Y*5Rpr zE7@%vDU`5-%bZK_u!=LjOh|R*AFy>8XJMiQYDqHp470OP_$-8zehvlin#u#xpMAGh+_m z5E?OFmjlc}+X>*}>jkKZq2fZ{l2eUFp3=|h1Z`fC@0)n+VBuKOv}jRw%SEu=DL}S@ zawyyYWpa>E!_P5b@6u_6BTj1Kfv1F{BY}`zDQ3j^}Q*bsuF8tMWb@I;GD@`;Mf=Hw!jd>c}e=r=@-MueXAL<+Zbl_ z^KLN*d&!S$1``83uI~lVu}Sm={fL z<3bybiI(p*ZT1G=V*Wf<)>hp|DEG!*oXOUs9#xPY>X`ru>C{1g@gl+5k5l4@=#hVCQF_^-eDiF=;0?;3@en?sZjB4Ocx?54qRBLrJ+YpY zsOZ%9o=_QiNwjl4o=}V5RRvP@m()oYp146f{tGLIiR|Rs@IfDL*v}D3yS4y(v#?=UF5*AsV1ZfH$rzWY(p77o7e^XJJUz+tePjnpQ( z>)6q@<~ADp;xAVm?_R6>zfN>eX2%o(`3;xpMCEB@K*UWcSWRU+cfj?_id1Fcjs_#vo5N zH^~-hBYdh>yt8T~S8O`8p>p(;Y6Q7K z^4MDkxY8b%6E@+@qEF0DL|EUZRSPE%jS6Vn8e4y1upVq>1R$|T@Sa@lf_(|?icbkD zVFZ4-{))~4=;Nm2@^EmzX<(DrR)ZYT&7Ff!WX6GWU><*b6RZ8_H3Yu;SAl|xd?8%> zpzbK>sPkx$Q4sDPjsG3=*E-7V6|z!n*j^v_x=8ya}e&i_ugx- zb*<~V7Doed;GtU+w2lf!&wVpJ;+En-Q;5tY%CeZBF7(s}D0sQpCVq(zg=E#g>WdW+ z`YO-W*EAU@*1*lNFHXbrsApg#6Gx2E6?w7})>9mnq}p{3gm$C3>ui*(oS{J_>>8eA`w+=4MqRA@>N9MqBvSS9IElURhR@ zlO8&~iiqIoeK4tC0uBk~eee9=Eeg*3={J#&KGEl>#XH1h<_PfXMp>Lr@G;Q`FW^48 zja{$*kncTW=QRmE5clbJM>04Fsefx_yP4^O1Iu;6IJ`elLCZq@SuoYPcfYt+eBl1* z&)_{U>CaD3n!Wu-4^jOTQ7%fz*p<^2iLqd{^=uUD;?aXv>A*19$~1E0feN!5j_B2c zteaGUx3A!HMcZ*&o_?dQ+TBy3fl9qI1-F6Z#_Mb9XTmgSJ>pNAJcSS> zqoXbrod31uK-v-riL~ybj?PdHZ3eG_N<&DV*@(aF^T$UMpwUY?Qfe!@0xUk%-ii$w zOrAWaXUz%Bhl_uaKCxJLMbFH10Ln3$6f_2MFWvrXB4nsYzg&ZcVZ4E!WK@M#5g~W% zozta4`N59KZQ(S&Y)BFY zZ}I6!yusA#vWa(Q21Y1R;Ixb8?ibU%+%ge0FU$Ax>HYQPr290BBQfi8LfcpdV;vW3 z>r^A>7qeAj7EP6ZNAIqvyzI=oku`xv&-Z`e#2<9I4l|y#nNW54YDZB_!ts5K83a1J z7P>1nIxd}aku9mIHuq&t)qi5hB;T_PXu@~5otp%-u>|VM!Vxw`KJ*F9$3*=l$|C=-7xr5H!vpO2TVjp|HL{fro0($oIL15+j zzHXV1usv3Es@1>Ccl6Z)ZWPcT^j^S2WF^1yq5NLUr68+f;?rIaR2IMpot=Vd!$eN? z;*VMKT>0HqVJCEeEZD;5^CNpeOs-p;z62s3wlBlxxq3%tE@R!Q(EVR$5(-88eC`EG zh=<-cyzfd(EvGBHry&EeH;KmVpwaR|4QjdElwaHCJCVSxRM#HdYMN}Ap zV`t6bORVJ}OgNI(RnUF|1MPLx8G$OzsnhLP%SvuH zIv}TzU%D6QZB=n_=5XPXw)Q-N{t6Y2L^cM!S4yx5t@N~H^L*bFao+sExf#jx~jK60Pgi9vAu)gi)X6coKlICV3s7&u+8=U@}U0S^O zDiv@pWlei<@7{VyK~x5GDEblK)HHNr3!WhTq|9;V#O&|D3greQ^S!pjz4?s=Hw4AX z%fLGc@~rPX$vO`pFZ(9|nBJ^yx*N)v+a~JP%pUX7KidF>D^}q0W{^d9>zDkZ>bh#@ z)$orf+h%8|!&iEn{Y4M2sczSP=*+z0hpgt-5*m5J^BTz$(uP+Ok{s5L#@tr#^-1mi*X%a84{H0fEjS*pnl9#@%> zl;m~G^@cl^9f8^BLJ4Uqp$8;a$kO_*l-XcrPyY zymjP;H;cvhaTPW7jiCjiDG8kYz91sa8OMX~cZuE3Q(g&QRc)M4`+bLD`ZlM@NTL@! zy4bK^d7NfMBH1kYPT5l5_yC>VqkqEfC*>iJ&QRKVqtebap4>3+XfX{gQbC_qT zW4l~|6Hvg^%?$x%l%h!K5WIy*yk00ApB9)AY9^I*f9Y)Qlv6V(8Rdx9;4iqmI=mT6 z*j`AtxM)4m>g;{Nz{a6eZvCdsE5z$Q)j@E72##>vDW{u6@L;){$u!7YO~%6PpvIHm z4bERaeM;I$0zzvRS;TYht>urSI}PZf5aBYvr5EdA?oUMa86(8XjxN^v%ryU}z4*WR z+3+ICl#UOfP1OzUpIxgpJ|EiT@(nmTkXOJe-G=G9yqC93`NH@s%{`k+Bai%*Ct!-9 z57wpkP~LUwt>0IZhZ_3;Vf7$#{cfm%l}Mf(8)UV@Ii-MYhkOK!jmz?19I)a(1VgC=@s=4a;!ck{{>$mfmf<_PzHXI@8HgDJ!KS|b^m;|tG+i|WdHZ<)toyer zo#>%kxmy^k<*!%XqlnT~=M}g^D~EekrPAfGfBne1kju=}^^;pOuR^vZs$(&XcWURVKI( zG+{S{mZ{5?-)fGl)N#07yxQ|=m2oXs{&aq;jGlzbHApa(3Af14+$=d^|6**l=AE{n zdmzexa6RQ?(ITz0RrSjVPK}6D&13Pxn&{angx9Bf4-IN%0Wol^vR_q_#N@K7$DU%G zh&QAJB)e?o)>T{pK7O!!zQYXuryYYo{r}vVnLFbkw$z3jab51 zE`p90g|k{6bWqGPwE{Gg7)!a*2Q69f)gB+!4hsN6o@8pr91A%Yl(gd26zGI-lm!;U zK6l*n>&6kcB}6Bc;9*XTsj^dSg$Hck!oA%^C~XB#5ZTR?r)wO{+rD<^{A4>_^MWt> zVUW4}0wtxy#ZLKjk=-wvCCHF>H@!5Yj7DpO6ZZEPT!%E)TIobc<19756_vne-Wd^8 zlq4^|tUuOY@*{u8O8nH9>B6e$z+b0>W0$|ODLN}+Edg)m^3agSw6%)L@C!rTT?|JW z+-8y`vY|y$3u~=A1JltWCfH0PFM)#RQt8c8O~|^F2l0xNx>2(IrJ?$^mQ{ymjat)@ zs;0xV!2zvdNyy6WaH`CWH8+!(@m>#n<0AlX5q}2myHjz^)Yf-~(AM+i=-uw02#f@% z|6B&QdoE>3pb!H74V@!Nfdh|lSpimy6oF3O1mk53@WQjG@M;tmm?WClG1I^M5bdk8 zwJ+BOPX6fu{Hg5z9Y(yo!f!mr@x@z!!$uC`B?fDUz72xWGsOe2JMZo3?TkR--1pL$ zl!hsJ?z^ozt=x?B=3RG@Qj=9x7bBmTd{2)@(gud7XhZ8YkILM10hI38Uq;Y%BM$C! zOO4i^YiXnZ5&9tJUQbory7P=l_NtSZ5O%qUFhzc}!PeNPUkaEpyoX8(b@>k+3Jv@I zRRKrC7vR3mI7>|sC%-_*CEPd_xFE5PeGQelxo5H0S(SV&e{t7-tYaSTt2@Z=iZNn- zs*$gp@1uT1;=EJ@uO{Y=$_@5>k8rn&8Ss5^PvQl+_)zpr!&%OV)qwv~&E1misWK|Z zy^Qn~)y}K7|K}2ogvT|#`&klFcsRRKs^KO*L2DysMauo zpjHfZ;qj+gZB1SHI&cGrMK?$G8G}s>e+3gDRc%qo%)Q{e? zK1O$+Q+Sn~`A|P|>m4z3tlf1ExKGJ{t>^B#Cu1Wc?kNM~=sdT|#H@e7g}x}-`TKgh z!i65;xDbF}7Me)o~Tr zTZbQ#0dr=2E(j=klOZivXR0j`N*==YMSk&m`J@FDRd zEddHmeFA~K!LJF?*=BwcDpBg;>{$Ss$TWld*g1X1i#$FEGkOV?57vw?W#kzhc=xHK)i14?{d%#EJLz`qCTywG6V72rsm# zmw^w$c*0w5=~44tLf4jsZ7h$_xRV(5a3sF;lwt9Dl?+m2IHchFsDE7Gn3l6sw$!@) z69<$5EZgpSspjeg!APm2=XEvhywtb(;s~KAEadz z#T@CT0nyoMSM4R7BOjDmTLdc+%9;u|ZQ{#zO0?1&0*nb*ZW-}y)TIK+5DEs<<7oKgB`J8CAMhIvm$w}U&CnW8Hpv#gF`|~zYMNf5> zj`!54UU9reBc~z8#_@^9nhTE~8Xni&Ej1HRubf!~6@!(-Bj07hq-F?9Eb!8j)-uhC zRS6!XgV@2tmWVhd-&)!TR@4oOKXtRW7Bu;3;_YZ}YHt;yPjIBcBIeAW66rANZmX&{ zdsuCLn9OOTqnEmV!0F$f>}kO zZrIlZ62km5a%pD#=u1vbLarMK!Hjuzi|@)uiLgli`b6Ef8ZilP@eAmjN;c>0Dy!=^ z-uw`qlWVnUcVE}QlYWTaLawv5&Uh_Ct|MR!XRZkp!u$tJs!sCBF7amYvu4L%G}T1a z^4_;-QJ$KCx6tLQl~Pz91tPBDl}|p+QDiiyj&^9{rs)|iy{GbM98@M4cRgzr9&(}YBK<^KK@VDwsVEuv_;JwB+ZEh@Meo>1 z$1!FKq$c;vKVz)Y&z3*CSo=)G!vq^ zUH%kN5^O*b-`x3SpZcjE($GD6m4m@c7~dT)l(l!i`BhOmGxfq!OYpf_8Ie02WGRGe zn62#2(o`6sH&Z%wHHNYE_)A0R<05nwYvk`up2uUP4R z-Yl%Dq5e%!xVwAubg($2?&+E1LW@Yj>km=xd;HaHnyn0n$vo~c4lt-sA4@YuOCDU= zIe;Ub7pAzt9=gBXiqB0m8FJln*3+LJF*$y7>v;cTdCu}&h$DHa!x_^)xvQWC{pqUD z-7^mN>!j-w&Qptj=xFPtLE&x!)htLaRGqi9yCk~m7F59FHB3c8o*Z%GTf744ToBcY zg;OotgY_g^!;Vn(K8L&Pjq}QGX~tEjMC~|%X)6%V`X>Y)!dt#lI&%soc9-=D229Tq zRBPkU#cmx@rl3i1ajP;QtLaZ)Gt?W4RIaP7EJF6-XN0}ckMMp!^lll`MZQ{Mi4}j3 zc@M$7dAyW!z<)m#HsdKl=9*OObA=?F2g(9rv^y8~-wAbcg^ojF!K zq&;~!F0;(3UD+$rSh(P7<=P~n*_D0M)lK4ns=t9q+vP*K#X0FBdI~#D`N7Pam2OWapRkspvcHZ;+=sE8?sz7l3D zdp)v0{D&2gR!0Kw(%g#lmcsB!g}omEmYbA5XvK;~^m;61MLa=BK+3VBB1XJV8LO9m z_XLM8DfJFj4brPGV(6%g+~CYw=3@}^r|ZC~T1rp8VX2MR4weu}D;nJreQYlM_ygG! z%HAlx6Ep3LccCsVKg)ij$vgYr{npD8mPc#kJ&pYoVv79Q(|!azbZrJXcUY)z!%_@? zJaN6aQz!w%S3i`nac*G04r8te*z&*$Q@w%R&af#b)B0I&! z^s`rJ3nl640;SU-jtLrxXEdQ(0NQV$`Aa<(NYWX3(%zVow@tvs5Wb@+-5*^kT(}KT zbBOdviTi$p)Xz-4XQG3m(chr%0OD9V^@H%ERPtm4pXWq<2^X11_%15P&n#LQh(1A9 z)T_o`JvXpmeYdng5)=D99_`*@&Y08M+VEQGTB7xsl3Z2?yT!_p9&GFxOHSn9=^Cl9 zW@+&i0SP+45ZKBs`3o6Y2l8r0<{aM6I0RYr%Li(6%+J_KPeR+4GKrlncLLK~OQ7s+pAJOogPPuYB-tY&Wn-D0)s@ESF=l)Z6od&x>BTt)5Xl zs#du>wNx9rS+jrSzC^>R(}x;tC5%_6GM|Kh{K>7)$h*pH`j!?(BY`;VJ7~ALgq7`r zVk3EuP>f-pXl@g@>c5_1iI+4=Pll9$Hd;f^{MbCpHvB{LE#6!zRWGyT3EV1(A?7Yy zZ&DXOFaQ|hzsYOC?={B3L$%0BQvoskj0d@L)*uQe`Os%uZ2#Q{ZpK_hBkt}cHt%$bL zf`*eBCC@h1Nt{S|c4NRM-y`$j=&Z?YiI4Kj3(fcCPq3d*8y%gQ!G6BQ&X4|PX+@=7 z7_Ja#Nw0Sk|73Yv_fkI6B*lzv=U`PtEr1(};G6yw9GdnB*hBmCTNbja@>o!QA8?r!wK|Qy83N%J>JKjn{?@&#wd+AsPU!qVwFFOps*@S zoB(i373%tA*!DcpGnHnU+qNW(b(uv?KL#iJz8cyPHUn&tY|Xg&&7a;m51_m8 z-K_w{MVcImLevWIa?e z_@ge5ENP_7X#KU!|}TjL$rZjc)Po3!s# zDzdI1--xje_6{}jCqa)KaB(QR1sa*eMKv-$ zF3&TV@{*_<3l4rP4}E_8Qe;bZe+B6G=<;7?Y)C>;*-*`swXmWgUKn!N(ctg2mlkiy zAJTLy@6jt`!aBqO{6d))02Ra3(Vbh!{mvg=Vb&VaTquD5%?s+)vpNRfC)O(ECckF7)E^#*?RxM+w7Z|HSMlvaoco| zc(d!%49D53uzk#W@%1(n!Ik59Zcl!0Tk)-kMVp9#iN81PFwoFCoiHgB+%$^7SRhXc zwRjTMIOrIEKabRH$=Fyk1BrUt73I!Rgsw8-hgClct-wd2)Mq_P+I8ZJp+N`7c?n_u zy!r0l;4jBPSa3R#Chptni_b!oXFkMbz^_m&_~q~UfdL4zb|ONFVR^onXNiwB z-gY)1Z49>Oq<&nY(Tt%cSzKKMJ%4f_|Ni9N=oh!#7<)4P4hZ1K_k4y@v8Yit1u*(> zZJK2^e4YECXMyE<6O0+@En^<9Z2G{Juh#a(mUXKO`SO-IUO{t$Go^2CeVqZe7Q!m< zICzUgRN(@=FiIrkiKhS}wRBGra`Y4v`PBykwP)5z|Do{fq^I+=T-F(lT#1WBy=O}b z4(MHdFOj5AzuuI~t?&yRAcjYN4Z=MH6!23UD=fTVJAF79S4Nj z1Teo$*)j6xSdGo-__uHBkJ@5iJiz#v{*3gP+RNme(5U38EDy{RPP5KfPT;U(_xr#1 zG65atmx>0Ud$tjL@$+>VwKAE{Px-wzYUvh(odHZyG?PHnnHSXtRc$}T25BfXtR5{P zZT!eo0%2U$oT=Zt2kCR3cg{$EbpwEK`XND!akq=seYC|zUSh}Bze#s|>z?|26R`c2 zS9e@!6(WQag}arPU$k{={!Rd7fyzn3o70|c&fG55EsHj6Ms@W|xKGmJIL!$J83#YFLqh|q9Lwtx=>NW+F<=n$Ay?~}qhOpZ-^?7m70 zov85GC!}TP#6lZsKCM9Z@k#lKbXaZrH~rWXAF_01CnMH$i6f>iD22>ovX)&D?~M^c!?;|<Qt{_@_OWQh$f%J_+**u%Vet%SKFYy z=SSMe+=XD-*|#%Ma+O9^1x=gk3+2_@rvLp131~1;9wc9wrWZ*Xg;1)ytb%9P^(`gx zimcL#p*=mWVG0@v9WFfZw2%M$xGv;gxF1h!Y7$jrSw6+6${ z|BP!?K-R}m{vtwz!@iaC*T8CVRbg>OhM;xM%%V$qSnWTwI=LB3fR}gxAlUJtCW)%? zgD%}P0^nf*YKHMlv)Qq~p`H;FKz&F3+c)Sx@;w(kuY-?Uk88tG|1AyT74~3;bq7#L zYNXtB%kEF6xi|6xtkB32qyY5#+wXF5&;oCkd!@*q15JvOug^^p);1|M zQ$AKxwF7hx<{E?ffM%cunskdjwHc6}dZa1M=&!R`zAdqR`xfO#b`%UqZI;%_() zI84ntayrunafEL*?Uk{8){V=O-;tm;{Sg`_bG-g1#DJwczuD7ys-X@}!(mgIfBF@Y zQkdObt{@ouDXXY4eSTlhg{C6aZKwDjKl?BkG-w{;RdIpzN0JI%bg%k-HAo0sQ6tj> zi8=C#yiom${GBK{x}J&euZYkWuTQGVgHaSmbGNj@%k#;m!TAxO8m6HGa-iyca2r9u zICpgi*ROu$y%VU`@4s6Is1^Cw2OS>9KO!%1SqP#!V7lc452*VUHx@u>-&lbZ)NbV| z`O4h*7W-DKW?~{(y?{(!`A>QVw#$E>dJGyb(DvX-0%@r(==v=QHl4lHQUA+N4hs?w zbC)+B(2U`MKi|nv(hUoN1bmQeHo9F0JSf|MQ( z`Nc57q;_t^WxJ4qSZ^Lo)|uwTi~$sAi^@W>|MNxu^I?2wy->Rz!SaW#dSrl!65&}4 zkXB7u4Cm?n!#|-+gFtKZL?wG~_&uv)`rr5DMJJAe`bwks_umRYc_Az_RbAe(RlX5+iW{-ImNd%=)1Q)-dRZ6S}l{i?IIh??wCD zGc>13O3p2(IrY~*{oDWh+p1yEc^%+f(r#h@NmzVnynu*ZtN^GdsPIrj3M-3ZU5KG8 zqmty>UL%uD*T+0AM;lfnPg!CO71ga;u-bt9a0^ZO`(IAce}CYC5))SmLFqC+JpPY8 z`j<8TKOdUY;>1x*kT48mdRe+{tLp;Emo9KrAp%B-ub>!HU}75W z8c;?s8*P>RzaNbX^=N0C+o;3x|N0K#(T*%JJ1=aKj`!6SxBvft`o9+g`v2NGsJ}h$ z(~O*dU1&=7pKp?q-|)qh!>k4BwKMVj!{PtGzV!covHyOajckzYjVsV6B>?W9S?R3Go=l)6% zld-AM_47nPCf=HQeU!~rbv?iDE=U6mU33APXWOR0ybBf4lf8w+O~-OdfTvp=8Ewf0 z7?(a}g>nJ95Z%lKx<}e_FiJ@5a+TX7C=s>cg@7Kf?X>aqp=d$YKWxqWlAgRAXc8ra z52m1~8BT=Hwj+sI<#fQj_Rtn1)<;bVK#SIke>xKN`=aSLK5F)$5Lh>S^%6jb#1aYw z1j7OZQgSZWvRTqK4`Vy!8W7zAm_Dn*>T&@%^02XM5!oqN=Ju~^^547XFGmEVr)PTO z0a>QEP+K;JA&o%Qpxd|!^{{Cem?JL)Hyv=4CtcXQ*)74SS6%2br;Mg7spa|_)A5D6>nwLBy?ftqUIgDwClrbXCEU*1=$yjl+7 z_6H$a{PJqwW9Y=>pRVa%63-^yI?(sPph`A)oTiKk$LP7V@XyIEo(9HTv?}jRvR)mK z?KXhEnx0~E@gtPuEgB$sb&( zT`V~O`5C{T!2Z@AwVAwHf}4TgR#M$Duz`@Mnx}Zs8BR-m4X*aJgkQkeF#rsl6p#FF z1dNvmQ2a2UQx=`@uhihcsd@6fvgiH^>M3EX<@JZyrHu^P51C zqAb5%%4da-LX$o}s{7&RahSMxUqlQZzDYgo7IC!r``YnTSv=4*4Ha#SJpUN!**QUQNgaa0t-{S97X z86Vh@pr*&(2%xcfvZpc#+!m}r9FluPJjj7UXptvXO|*TNL%%N-ip(3`6ow}ZHb}QpjTShXObsas`PG0N`3Zr z>$v*;2k|Lz61V{HtnBer-2vn8+*Hj9zw42%0rKsh`Lon~^+nG;=esCJOwXZ3gE-CF zb#$MC3H~85e%Id@i$n9rh^4kRROUA){?|14-={1G!wWE1#E3JE02oJTR#aoCYW&r& z^`__!F_@q@E|T|_78m_tOGZ5)$?5_<)J>Gwl?!4wa@)e)4{XdavOim4KBC!o-UCh-r0$(GNJxG zfX;>0=?5AK-D_*PeESJaIx()ApPvfe`7)uxNxN=-VI>}FaI{0eO5lqgdRQU2a)pUG zm$tQk*V2N1>ybL9)fmI!r)e;*lVU>kJ>hKV^YaT-5*S#!^IBjXkIdX@z3P6n%A*6c zs%>yC2Iofs$nj}51#nJQssb#TsgV8h;%7fgfD!o2w|^>2BB7|V#`pdxI9lur@6e+k zutO0bOsqY7$6%TCjA7)d9=h%%@*#vX4>WH81A{2lBW-s zsV(by-L(N{VvQB?oL|U(F&4*gK)8nUDR4y?-Qxj=X|T zIso3rYG>;GpWt@HM`kxO#4r6m1bPWykdQn?nPEw$kOEV^atPIQ1X#x#gWP-xbb)MUEEr+ue6v7p-vv z{7$=mR=*5_X{~oXIkw;?Ye!oGsK*_k^ljb?W-sdgz78m8cqn?_oFiy9_yJ5`B$Sdj zoyQc@CjsN85R?W?f71oDO#&2C7Nr-qIX21A!UUy^9|A!10a)a!&|r-1Le1|ev5PP< zFse(QtF(};b~+&2M>&9EP*tFK?TSCserCu@bspj5mp45Gu*geGs;}&e&?KII;*Kb+ z=P(W_>F20#`kx7!3pHB0`7I$mKx!fcvp01t1z9fle_vb1JpjoK)~($f&l@e7^<)Rs z*G>vWq*I+nJt#K2?K1n~=N53`G#(Qo6q+t3&i0;rjte&ivRLcP8l}3G^~ygy9{ZE^ zqtfx&?0>FJ3ZV!fOASC-j-v{7KaHT`kbYG|RKi0b?hf))oX!G$l~%Yq$Hq7+XXz+s zx|Ts@^#pq0oDMlU#KmACmE#Mts*Isz%SiK!8~zY7 zUPM{dsK|#Ipcr>>89flmPk{qNJ;PotfEU_Or6`t)O#DYm2*b!@sP(w=TR)VwSR-EH zq;E4A&eRG@3-nDdXD;#AuV2Pd9*xhroz7EXLr6l-KdQj|HR5yXeCo)S*jyJ5Jj^<3B3btrw+U@l!$FmTix4Xk)$`0SZ!!uNbCr zS6rmT{z^2N13Q8jdEOIDlGB^<=D3Ef@fjpZyJPFbE6--2@r`Vws@tYX4kx%U2=0Tv z#vNmYH>Zr?@~6Nw1cudI7rhS9o6wBgk11YFRGOmV8ukF)%?M=#5Vi#O_&Nn zOU$&T$A!9jVLd9@Y10NbT7ekGtLIrsyO*YZfrA8Z-2997-S$7$=n;s6TWygC7?QQ3 zSrnB#cqi-mgws$bEDf>m@w;w!qzoIg9!xQ z%lbN5>O(pB8{_x~?!R{UavMRc$@kv}2}qL}0o8!$C-b5sz(=%*B7Xtw%)u+GNndMk z#A)4+H!$u zsyF-Ny8m)m|NY^<6daXSIQ{BjX+zxzsMBu+U~ux3GLWy~l)WW6@@ejsxR@T&^E^OR z|Lpr$@Umn=6?hi8rri%ptK?9%GCj<^U#!ZmMZv|0)COfY&g~N~)3I-jE-%3P9QWjC zg7rCa#AM#t7fd_(W1o`1P%^3Y(GNtt`>r77hurgFNia^g0+HGvoSz_49&ogHfb7cm zpvzJ_?wPa-CWay4?ncW=gFyNX#%K2OSwbWuu^EZc%%zz4mJ8v8s; zAbp`~-9c8qn@*N4i$XvTh%tHrMVcS4@X4ZU4^y0ecBH>^vi$@uNI9(fFJozkpUFX> zkR$&MBM>Vh+$Q!XT|Pn(76=uAPc~7nsh6vPSq0;lC%1le&4L)bC3x}EW&UaxxM_FK zuLI~LZuW%QHN(1f3fSkD&+9MEg7!eU#XpciTrj+Hza`}L8T9--K(rABfD=&NSDG?) z01|8a8iM+;7my0?W&oQlW(m=lX!Fubn*v(_Jkl3DkF-tR6<0}t1$M>+J|{13n0aG1 zh{zKMA}qjFK@k0d@2MxfZ~kOe)pU1lrjUHr_CE(h8TP)4=4^^_x zFvATwfMJUIEne+8W}S+gPnDFMttAG;8>Q4#gwjswN56zsg+$^=q~81X1vUqwc1;>; z0p1qi(@Cljdgc9zow9G+<*bWp*6@n!e?4;o7GG^3e3M^2ZaN6F#4HmB`M2xAQLS6r zo$tBDbkW=@+HQQNap2y*qD8wB+1OW5+@%0T%02WzhI++7(hopn{^CtVKVP4(Y`W(+ z3d*;htYIJrG*^jvI*F<`W>45rh;-$d5B(+d;z#PPdD$`+;{xv29+LyqxQ$d&4~Cy6 zhA)^Cqr7l`P{fxa8JGFJaXl2kF0o4r6$eK2%lZ|``37K}Lj#GI%h|(dMJpS6?Nlf* zM|LbhvFR4aCnIY3Mc2RiqW$n-e+kr{akwRoU^~s_UOnMj@ps{mFv{`A*~ z0JR4Vi(A3=t@~QlvU~jbg zeS`)L#+`U0WCUF7j#8nClR3BkA&=%6CaFf{A02Ozapt8Rax5qTFBI^3C0bpRD z3`|4krd{gTQNxF!7@w(HGeHg9|Dc=+gLs4{(E^OjsC?(V*+G}&i)1H@x~+TySRDZ< z^?d*e@A^U~JgZc;OiRnU%=i(me-xs92?G|DO=0oH=Fnhx7=#p1gyfN+LeYA4=;6Tc z$>hSS#3Dl0Dp0Q&faC0NG>*cjPn6HumAt8L?}5un7qp{_?EF+N9x6$656h<30vC~1 ziQi{KC4l&$T>BYMK(6`J=-G_ks;KH>iPF51`;rdf zQd<%)d*$e{a&^*qG19N1W)SWY{(OgxMLO#cxPAiTG;<8 z5VL%JmwY>OPi4pt$Crf(W}{tlG<_Urd1H>*Lww`yywk0#FU~kf5g`q}is+t5OC*X< zH@6CRRR9BLtUzM8dh&Ljb&JSq-nG2e);+`Y;#^PINqmi&4on@G62nvtO!|9&8x4G) zUScF@#?;clpceqMpk)Si16S9~^pYDcZjQKW0U2F}JY81Q&^Sd0XW}`T*X=UZq*ta^=jPAS6pHt-@NJa zY7j85H-5M)K1Ycp^wv{0Uk=fxqAU9&!S=7Y$|OXu)<8q8U*1pJ(<$l?8|>e=`JZAH zQeYBdn8DQ@TCf?HGQ8Gqr`X3ylQqS^I+xIAt z7CBz*PS*W+fYKc_RKmnyeXzD5H~_lsk$u|Nw-}dReE^9mp>%Dbo{%=mQ+o*{scnGfm2BHrsp?3Gb#a0QImjFwc1Xp>Hd< z?CyoAg-;y+bBy_^Yn9JdWctF(nsDJS{IcaDTp+|+0sWBlLR_U6DNqocVwE|P1_tkA zgcAWU)FncnQE;vWFlID^=j9Z*g=)q1ov{%)QA&vK|*Ijx}C*q7Xd;^a5b9B7!Zlk{t8*g#fF< z`q;D7$rOZq^C{zuu+|~>lkzX6Y!4Ta1 zdxte5NVpGsNHN44L%V$VF+SFc1_XV-kd{8jp<2UZKz@=$ouq_TtF9Zt$tZaZghGmT z)qUb;diGw@qJQ=REdvxyMGmp4&%h%@qLY5d$Oqh&{33)efPJvqNxEz}K7maHj!NPrQ6}blStaI2ZMktYdY?g1 zg3jdyseKH_&Q?RK`Xs;R0VGR%8y;`1RfT|mNCpP3H&dw5LlCI~fIInBRhoKRP&>6x z#U8QFK|RR{L%mdYt3H>{gdtVc7mh`++ikBpswO8;sz&wQvYkTr2wM_w2^zLP=47F7 zm@r5wmw1-3B*YH2K}Ed;tn@?Nk|B3Mu4)kXvCvGeZt;ssQ4`2u45BZu#b7_bg_B-8 zv2Tl`D~MDpY%|>9Mwi9z&RvSM1_0kt;Gw+BZr-eA!?DVaSzsr}h4y)&B+ok;>A><^ zHziM!ncGq@Rh3hHd;fbyhZ7;L04(P{*H7r6Shp$jT?EaSADbz>58m&yVHe*Z!inQW z$y~FoI4uXfFXuoG$P4?nUb1sZ{~OuGODr2tiM51|W@m+64U(}H>B{6pu6t6E3BU#l1 zzXXG>4~CJg7Bc}F3N-#ni=%Vo-d{keGMtdYX(AzJpnUMoVY!+tg?lA zgg$Jvbfq$L(P|H*%9O&gG@|ctb)+BG;{yVHtTGjf5N1yMmG-9KrSr=CT8&!jAAi&% zh2FOOtD|fQLMwP!sBl0Gy`QLszE1nqayvh`(aG_?U7k&J!I2xqt`cY1zqk@fstKeB z)_HZcSr$zPPdmSp;FRev81~-F4*eZbvSY!B@mVm1_tDwk#a^fjE10->5>xAw&dUW< zgST6~)e^J~ZlZ-(gc=>LlNGMunD|F#*AXtV3bDA}3lsb)-gAlh7K=q`eCD{^G`7pQ z)1<05BW9jnLzdB}O#^HzJ)TTM3)43WE(Nc-;?6~q8KHfYt@)^#bQW517OoX}39hAi zuq+!?e&%HgT0hE!EV1BLgR^4>jy$QPr;!WZ0RR<;od!JM8hE_V8hlC=A*LX9{*0^W z9>^G+yPTLO6eygx|+HtuY6==9jNC>PIn70 zSX!BYFZ4I5ReiISm@v4FS8A{S5D5N_+5W-9kCZXL++~>}eC~btN4b@2a$?!(1l$6i zdrFzzA8dC{y}>G7dtZvKJP zFzcOlz6U=S>Q#esa_lKdCv`pr7Z-f6I&~T$Z`YRb<{w-VdcO-e1{t?SJ2Ujf_;Db0 zpE;MFK@@#9nC*9T`a|KXKnz>5T*GAVE$Sg{%;Zx~F|~^4@%;xIb`rKAwt}xyCw0;k zz&FR3gp?AZas;b6hf>cUtPrEiVGKP<6&y4KCDs-!uk^6mqxvcRm)`JF=`lZ{PfY+U z7HM{l7aE8=9HLMf2qa;w4JD&7ShG=I8&1m$v4q?>U*SO{o>aS*befm4lQ&Bs^GVH=};;^wu)j zJ-?6({Y8`}9j|#5uR=LU{K#n8=5f|v=m@2#pBFy<()HWqauLT;s{>_esKG0wi(6t7 z`7d}C`d6Uw7ACwLzJl-qO3Yasxg~a_kHkF6Q*B~12BSM!u>_4?XM&ZuDt1#JcR%u1 z&X}APRp&0UJ2K1-+;$c&@wJx)c?in+o(v>47WIlJJd~Kod5}trA|;zEKcvs^V#_Z4 zhI`HvS1fv*h|przM=^-EcQG`SSKK1GRg^2<$p1ln0Wl+yLJ1d@z&CpkH&4UJeq~Pa zKqPiaiI`b5oG&=#gdD-}hzhG`J>-?I9bt!(lQc*e@A%J-GK0p}!aX{NO#&~~|y4*2{= ztQE5|6)w-B6|z-S8?hcr`XWAq@R*0kNYA6V;RJ6bOi-(63x*R9YNbwAMpUP+$b4Eh z`6|U+H?TD@_#OFX+&0ONj?<@R)IN?$O@WdY`@f#DB*%oXnulZSoPc6R295kCla{N& z@sgca^s8jUUDG#4(}LfA2+MR4V^}x9+H`$(EX396FjhBk849!N zaH-gF!XCe9xhPxhwr78#@cgTgRpdtxRj|5G4_a&#KW@f*Soa+u_};4Iuiy7Gi?*og%5CXq(I=Haweptxy^dx1D6W}sCwUCIw&f!e5NzAMa- zgyHl7n79sXEjhPXWKtRD&~s&OOnaq1H0tHOmVqy)qD}fWHE(7F|Df+~^@aOBwX@vS zlcd$Bghj{TK6P!F(#oO^jH_EDAbgGak@Su z;#~tKh{D7J-VAS)&$L({YQ>WrLA*$lcv@;VjFR8cF5ciXA~D9*eL;r7f)E86Zf!m3 zJ&BzrT;;7b+R_4r7;WJ4!}m4zpsW7v(bEO2lw=C+X@PFzDp0VjVoHK3WbS))Qfes| zzzLr8(zaY=gvo%|v8ArHu5{ou&d4*?+QsCd%&b-Mwdf6xZbvNlWXHt*_lEYa`MLPn z`7ve{W>dGU$rlIH)AS(AsJd{Y`Tb+|#s=3>o<^^Jr77NPh5NjdfYL@^?x)=2(tUO4 z;VWF_oK%pEQltIXsRnuL6Ov|n60J{gMnpUUeSy-bRWpH`5-lN=|7dl@-c?5|z;`N3 zl<@a+Zq{eTSPG9%?2*Z`D?1`rsN)>ywDnV|*R=>SlmOC57zdIlimdk{-DU88N0D(| z6w#YxjStzjm{uX=m_2(w@J+;T10d`D7JuRQ{Z4N|Kz*5M49&C03nh+XRV%v^R{e(nEl}3Wd~;qPOqJ{~W1?hS0eY%Gx443RAE$E7q>BEq7#rW$ z1{MMwrdSrNt+;ajhF3&5=)N4xnZ9#{a617{_zZ344B(>%^2gIpS5%By(Bk@uIe9nnPo=Oa&?lOUoft{+ybIC~O1LMD zX>~*5*V~B+BgcaZ9snPpc4=Cg>|{s_q*3aPB9eqgfTy!Lm$z{d5O{3sTJW-QglDE zprlMvDe3O|p4YX{Icu%GJ=ea@kMGz1wO9y~H=buaW8C8&_boGBQv8GVGL9RlFFtl_ zZKQlwI`W9Q^89C=kn?m)$j{LIo_9B}=Dwi3)GkJv<0RS4f70D?Ir3pX9bc-kG@inE zoGN=j349HI;c@GjtpV+^v$7dGn$0dk6E}$r_R96sym7(7pPDR9bv8tPqPryYe*Z(r zLvM$UfIUD*ZM*oTnGh8zbU8Rr{{i(Ju+EVc?c?f*l%3+jr*e&Z=A4qdq1=cq?jTHh>=yAr|iYM{z;_pt(w ze?<7R(lh;O?wKs@9Jf_Hk=x;JcdyaM-iGtj-?EA+1F~V?0i}WnSR%}N>rGU1A-G#z zFc!$lTU+p6#rA1^-qkcGUd}KQf_$99p!`x!P_^p@d}L_d#fq1W?Y-n-=3W>6dmA@Q3>SFNoofLzxEVgN@15p?S}F zE}|RVOz|TXqSCOr0`YR$O^sUf@k=wFyw=|I&IBp?kQt?rDnnIGMOfwh%6gyqgf)}Z z3eNe%A8e7n%~7k644do?dS5NpY^j#VQlSo*P-E`ji9!8zHe8VDJp1<`QB_gAZS zOD@#@!(PS*6j<|KX9a~^@2j=~JkWjOoQMEyKQo$gX;co1HeK4C4=~I=JFE#U)4bby z&E!Q53$cm_3EIq{2cm{B+2C~IPlrxdr%3w9j;(hAQ~T5Ykegkj{f;-^S}XZfb~RVa zSOp+j^O(O{m^t;17;e(M9=u?{!^KjyO{~*zx*}?E4A*Z%rk*oTm-hfs_+>~1_<)o4 z`s1+u`9twBr_ml_`uA;{O%@a6$v-yYd-v6!o}L71e>xd|YK-jkW(9rJKX>>4_@5Ty zpsAL9-15r-)^&4)N4F-pFRM79N$;!q}Z}oLj-gab3G&H%9CckwLx; z?;-@f?p!9^TaY{-rMB}nZr0$Lkcv+jT^0)9-Lqn$&WJ0f@W`@l^;ycOo6fvwtATuKR& z3J@H3MsmhM9GxokGJ6vv(w6L|@PRZg)b{Fy{w*Dz*TImwoc;r*FTU@Ab?#4gSEdnc zL72V-FPDGI%sCJWv3kH;8u6_4=3xIq6YUGyPM(-(7d=}m#E$Pd{)^Y;KUls;V3Iw! zPZsob;h9i9U}($W#tp`N7UgaPTVMZ909NJ^v2!rJZDvp7_TyE1g#LnH{*b+9X225P zchLVc2wu?xTltAIk@S@ZNmn+Efy79i=Y%A=TDv;Zb8vHRCAlxVO#)}qf<$KeaZc)a z{ukdxQ96zbNgV#U4?Df?=B`jct+&6? zR(Gyq2_+ZA6#-4GfVC|NrC!hwx=uMQu7-^}-}XISIhxpMFnB=hfht)V-&5<@_~{#y z&e3WaBUd&LY>KK=-ZGJ%SbU7PCgU!J+BaXI;nH5G;zWYIVQKt)@z&Bo;~_mW#TMD?G3oAl~&jB z-q_rmn9mYNcy19WU>~+D#Jea>fAYO+%CjAl{5|jh=Ng$&Y07gC2Xd=^DD!53;RE@+3P6kmy^!;W|h}zuaDT=sm!e?83o%9m3d8(d9wUp}6JT6>%ggmch;O>_HwM7|ZXzLqMC z-+Oug#!5D^m(K!*#x)``-dv>!nRDf`_SVORUxZAjgQ7(o#{uE+!L$s+wJb~aGGt`H zdPM|j^o<%H2mKUkuz4c!;l!%ID>m19?-Xl79I@s|C>cPNY=QJ&?;+y(D8G3}|lV4A$x2 z8^2$bA1=>)BX>&Bb{VMSIn!ai6Kp<}L90f1=G$FO-5L9K$vf6WckJ(*@nUd2@kU%` z(M1|cHS`_4omQBn>W1?C5#EG5#oDJEkifB?T~S_Xssr9sTqLvbPv*7Jr>05~E@Y(D z7vEl#Gyg0^OqiEK8?w=Q*Uwu9$C-EW=GP=!gg-PCneqM()SeudK(+GG4#W$-0U%%1 zBRTiaZzK*#PfTv@@?B%$Rt+O(;T7tq!^P(6&1uT?WGTjVe2Q;tWl_mve+|2LJGjz@ zem`R?aT;qX!Qg><8Q63?BHBpJm3|x4I5o824ygO8h;FWwhy8U-gJA0wYhYJ8Omyl zqMAG6IaIGj4QW`Yn?wjWWk;R-X2@e+{ve;tq6Gj&%qcz($s_SVH3}}?DY>W|E+x@I zu83^7|CdgZ+2_-Gzc$|YTW5bYOnj}f$b21FJrtk zvlxs0yliJ#C+c639cf@_=^*?s2esauo`4ojAdJb?odRgy=sJfyw~ds|ak-^Xi-b~; zx(7@0^`QnAt%q2n9s?pqK53%y5)-%m_5TC09{m`fJ|6tV6gNBb=xYpf^_;#?y zZ#dRZ=olNLChg65couR#p#u^jBsg5*#Xam8lBVp-%oi!h4`@4N9wy*DF!fA8NERhe zswHf9u+QQOC@+!7*1Wg^{}ArGdxg~@u-*bIK~dRBh9VPGqNnBKBJXdevo)>)GpvBB zEs?l&8Rk3YSTzp@@jkkpd{d)zy*YiI>}%#r964FbwAJkx+ay360)V=xyYL0E0f$Fd zw_<`%6~`ri+&SWlalc{g{vc3D&WDMd>~87cVdssN+^IP2_A!WS0r!!g`7hQBaaSYDru20{z71>y)X- z`*#Tt{l+P=+&3ij?!;!G^x`MP-_AymdZ>p4Ej$tvl-Zz_T(B!?j76!@fZ*G@HQ`RAYaNdU(D46b86CR%keg7C`RM0y1uyf ze!+|;Z>%8ix$^Y#3trvPLzx(k9}y>2uS@%^oE8KOePVQT9E^2{f|bPz=ALktJ(9}E ztawGO0vdWAMek&8h7$K72QIY)=Dn8^1=0+bK*~sYlgr-Fz^RkX>r9x#eys7sv&im7 zJ>I$KE-wmEU}vim6E39b9A@D<&ub62MCeD9M6k26e+|!(Dvcz6Tcy@gzF>jbgOd8*fXihqPH0H7p|O-?4bcsE+HP5!$~kaoQH^UKqI zt&d%nj6%%20TnRQ{$72IY5;xnmg-&GL#weO^#U83ZE%U76mrI?RVgNugH+Z-XYj8Q zd-4RU@qghLDXYlotfwxdyQ-Mg1!=f`BV792+9Fz*JM!UskV5^%ns(KXZW~YUVUE6f z=7(-aFYe3S+PhuQ#P@ir5780-w7q)_kjk%UI)LgI9pIT%_;s=)G(<*I@l)Uq{Y%1- zn-v55RiY@Kx-|Hk@3=cC+oiukD{RJ>{Q+(&89<#C}<%lDu; z#RMGWgRxg~r#;pJ>~tB=m}dB3{kn}dZ%2{4p&!o$Aw|IaJlV&9`JK$Zc&uVu}#Cy6j3PV%0zcv7e1h|Zkb2`%?_eNlzlSSaNUd4lY9o8H z(d*Jf!t~m8d7V8l(-fXQ3?^A^8(w3?JITb5={mJdi@dC?I$ZcZbpt>`-}?`STSxMx z-%o=v7>_ZjV~y@ILN0|1g2%OJmyd6r7WC=0xp3Wjd}EtVKy)e*;a?(NkZ}opK-D7p zqfpc0gnAl9CTrPJ!t(Bx|Ks%=Mn9msJHvo!u;-Fr%@5g-&w>S(|3bkYj4zU73Q_sl z+?<(_f3^-V`P4(!?{RTf8CE9-R$%N(&!gS#!Dw%)PJhd{g9{{LCC|3Eq_a*>hrxMy zDe3`^G$5L75w#XFJljSIlhh|y1D?;)9d1GI`bE8)a*8*FB0SZxW|2_ z_3RFq&b-M|cuA1UwDhVm{1@*O4GmQ8y&XuoYo<;!opIM$tEDcJ+Tod)`qyJTZy7B^ zhqtm%yMUk3AJUpx@fuk5q-dFw`;;2m=bD?#g-xn)-(;H~yiq5&Hd3kWcp0h{^5jh{JD zpF7sToE#@%rQbuQsE+siUu8%SC+ZOC3Y?(ObfCtLh9n)w9IY*(CW8ZjnL9JRw0NyP3kBxKkS9yJ)e9onZZM|&|0O#cY4fN`P+ z9~I*5Cr#b(v+-Xep(%wSnbW+1RV4r4(WR^mvj2F#cX$Xwz3p;C_t>A?s>V< zM)K=Zu=xoJIH}*>+-iT>=ckX#6aifVf8l?HHj!X*z1Vv2z&4cV(g%S@jFR(~)B=DG zdtz-lj>?qWFe1Ak7jY|L({97XLT&E~KWOYM+`QPvI`a*VoVYFs`YAusZJsC;e**v- zW!3wZnCKpV!0CuOPCLam4K;i2k=$D1WetOoZP+_Yfh_s-n|B8Lz*~etQlNyh>V|MN z;N|J!TiAXlqAIvjk+|izfk?-0`e&nq_hNcBq2U@|>qrs}A z>)dHwb$T^z`E)a|*@7vxTl1r!=KWmv7!B6@vEw+mv|=4UML3gN*c>JDvmzCBbBbc# zr8JNfcQG0%=3&atr<^qnpO&4Trz7^r_6{@GT_U$n#HdVtS8F8feadT)RT9VBXN`hD zFnf~K{DMZ!wE>=2RK!fJ0(GhrvAztg61L05p_ln8T~7~x5^iY)UT+vn|9OlS8%xew z4=T+P7_FW48elT^9PX%G;Tz4~I*s+s;jq;en-0Q8BYT#|>)d?Tt5ng|V;X@YmZr92 z#YzFBKSu=8MJCq!iTtLqKVa!sG|O)%FZZ9up|8RfrtRO1q%RVu3Xe(cIjDs~aUT^3Pz4)eCVW-AcGG>Sv?S7I{TV zRO2`ti!yIPijS|q3YduwA-P-f{vu+tVdw5fMM?queNpG;5pPnd6P9CXiU1T)nR#c) zU4(bmi$pmlY~aKzV_8SC-C}17bP7>LamxefA@bf2%5o;{QO&23!jkWYF{SlAqg#@E zOjq4Qu(orY4VyQfaPjZkQ}J$vx*2a_Wa4g}{2Wg_`8g?3Jx?o2N#q(qI=z#z?pIJY z?5iRl8J?cb|9LlH;6XM=9AXOa(>qt0lrjid=n=DRYa-dq*9L^5RRs>k5E)gCqA%e4 z@|*k;gOo{?s)kF1M2jull9s$Eb0!~(3F?F82j*>Ps|;~A#IkRbBTM_fHDz=?=%?-q zWMo(P6?llEcUZMZ52=!Of23eFrY{DFIE-n|7jr->zQF8n5Nf=_7Jrxuo?z?Ka_@~?%ZerTvL&G4?nv< zQOw-ZEE3-*Ztq>KN#a7=dA0s**o=>QBCNMMkFZ&RT!GgjqF_+JZy+<8eZlr#p}daD zo6n$b8?$IB3N@-ct+t&Leq)N0wz(2mEwWqn2<1~_n#ep|Y+CSit=^W@b}ZD@c);Zq zr;7r)Ce}!Qc@j+-zRcCIYI}?6oTR1|%O;FWM~^am)!1;O#oQQyz5@n5TC&{b<<#Z$ zmZqP&t_Y@6Xv#g@asxb#g2m6Kd*4}XI!wpvhaQ(}$-|(^w zdMQ=z4Kj`tt+K~Sf9#)(?o-Ryy}irVq9sV~O>WWe2+D%O3`xm~xKqmLNI8UcT(xJx zDHHsdjGcM)5O4gsrg)2nguE?Lx#;w4Hrwf5+)>eY>(!=iA z<7^I>QYBCR)wehlY2oWjYfii!uE~A=%~eb5YvV-n6fQDFqSxoXxt+?f+D<9?TJJY* z)fJsfzC+uU5ZGp^Mlb0xhOXD-9jDkY_%i+@)clG$!J{0}txCMld{D7_C9vY$C>^xM zc6S<0zDP3ba-ON4xU)A}a)6l=PG?(Hf}5v!>L7Sp)^8HqEMZIDjl$PRZ~eJYFnIcO zDZpwp;HV&Pm8oxiaUt0o-qErKPw5WOnOHb|i{?R_7n&{Z5BGwR64ePB7t2{G){0@|_R<-!Gf$)vrb)XmrqUbgajG z0QLg1YufeUbUB)O^*F;YIuJt54Ci{UZU5?mn9vC{esyUR$bBS>sr{ov*?rKTtsrkQZ<5mCFxs7=M9 z!q-))*y*x*O&O||r5JVjF%wVy0OE2{0rw=&DQ0et9M^pFk2Z`8`apYryPbMwhC>*2 z(w>2f-22YomP$RbO(UEct167A(23}YLQRC#c#zliB0AmmR8Ma|OWR2K9<5br{RpXEn{f#`A!BMTB;OgkXLnAM1Jm=DdH|!StHfMcN3FdCIcz#InA@* z#im-xHbusiaTER2hJ%f3Xx_ zYfb&9D?c(UBRCb^Pia)1G1WYOuDxshW@=x@aD?qISxmQMaPIwxMsm_+~yV)o* zVurq>ML|9V#W8ani_|QQ{jhGD88ZCFF^_?X0GyY_poZ^G*J?}a>=Ym8SNJSb9Q3=v_nSH#x-!D2E-ux6tA4`N;9VIPxb8b; ze}u}n=x3XW1e55-jqw50_#ekZy(fl3PSeHunJpyAsT4jSS(W_TJN$>lOmz3Sp`3XI4a34l#1fkY|HzoNImd`b;m4O1;~XfNGYs62`^3!irY+O zI4GxjD3HT%y))gxg12_R<^F!yq`O8Z%E!HTg)61L-~*S(S}` zB=KV33+2#RH+{VyvbAOIOSLt*SBs3$L3CTj?Naj1N6HmN6Php5B&|OdtM;gR6j5n6 zz#aG1#nqpLML&t(4|2xtDfQzT&fw>@Qxqw$o?_eeS$xB*QSjQYf{Eo0rGhFL@5zso zl~e3JO^oqzo!It)MgTIH`=wV??aLP%vV5`@H)3L3H02tmVi}zPaJh)+E=0!kvMhQH zP>u>wPCSY^=A5b%UW>Ue(l$ux(=RN;93E5M&HO)yni=Z_hz&N6+b3qea9BR}T^#<1 zOo4utoSr}-Zl-7306ozEbSsDYv~ZC)>(E=GJF0rs&0@ZWv2SnD;fqstMtuZhD-7Ns zk8}L<#+PX8DKSHjF%`6b9tSH~;}YS%e&QjSc5NB~OBXN8m*dI(=E897>w%cRaY!qRmljw(21 z=qvTkEDvn&ljEJ%lA~0yjud$J3b`4moIAlqzv>7SWBgUd>O*f?G*z!xW3N_^*_v+q zH3QZbf&Go*F)WhwuQDOEO3ce5bZ_QUh zC^&y;`_5&SB4k2L>{xI`Ws?YZX}O@}0F=|1cZg>lbHme5C2k0>oN()? z?f+Uxkghn7tR||R^0W=T?`zj=3Wye`&0O_kJ!ZOOdTj9SoCH6?`7@OUZ;2bLwWKlz zQ3s-Ef%>lg#7U>E~1fpQ=L+^k%I$fN8xdTYB-ic+J!)74(Om^W*-9 zWyi)NoPBl7)0`QZBp68!_Vfu9p6J=J*vNb46M=B!ia)2cG~A*W`g`N{Y5@(i3Qwv7 zL)i9ZJD(R?e}F0V4B_3|m%VJ8vpTYWTfeklK)|5mXi+IyTCM+_@fe~gzcUGeiioSYS74@gQn%0#^`YO4UITD{ed|t1z=u5Ub)9=>VA+ts( z?2pJ1WBW@Yw%8ps`x6>S-^jrQ$4a#8!oFb0c;6zusK%K8{`k`V;(D^jVSub@X#nf| z6>x6;z`}aN|=yrbdmVeaCW08-juL6;j=t89ib*^ub3piFip6g&9%*jTcuVvLSAay6=z>PzMRhD zeuG6Pj`K#6c;5Os+QxP|U%ASLHug^zRH!adeCr<*} z@GDV{6JpAeOAn&#);8BF-%c{O+$Ip7z)9lY%jJ_3VYbkUU90KBb`#kc%zE;5FQHT) z@0sz#C)ZNVI=_A0Zinfrn<*9?1ftp5k^kgq*9g-fI-=+vdD!%zRpvxw+zn9C)8Q zi59ordRJ?4yj6Tv&_!G6$(WKqcyyk_(b&uEsyStGwGrVAX~8-5BRM8qBTOTbtxVpk z@KDty5qcG5u<58Anr_=jEK{Jv!HEt>p9}s7FJB=|XDvgl!5SmaG4b^vybCZb$EtgB zDc?A5rb z8fji?2||MUI@dzun?DTHf~^*0a;Q8APsQ`;KINHQFZ4N{vE@@eHVb(^cFqN+npE$Fyj@yN0qOpq z;gUFxXzK--n5BB+POGA`c4owYRjlyH)Z_X7NBkwA=$X@j)_#E%)T_!L40K28eU*yB<{yr@TkcjYc)=l|9u3=#6q@5JqW-OZ7D@7|vCNH!5d8m98fR z&^ygXy$_$>7+;V;&PL5hi}2%nv6$=h5pClQ1CS$GLF$t`tG7^h;cGate85H3pdJND ztSH2acVSo50x6ykx&4@aCT@a=y17W|@InkmoW*xVtK9R$tnc4uhSB@F9_0WT31hsaCrE#fI%;WqE(TfRp!Yo zvtUL(32|!6m*?@darehMXvf+cVInnMOqCOd#?16B1>5)Kh8?>A?RA3uv>;>QuP7A` zslpf8kmOkZMpN!qz}_CHMlrt2>vh-~?F**)1&yM zi2~zmQi>Dz+K&#Yl&wDCD<_&2A6dD>{YmIg;5Gy%cFl_xMipm zK&}3#eGBT4mJi0$KvT<}S9?J$F2?UQSyMsIfL2WoP&UF-p)SGajt|i4SeoQ&ICST~ zwUgW*5Zk>6@j9l?EXbVMa26|)_sRoK#dbg4(lYi3uRB}~KwJmugLG16n3ot1Xy_~F ziDm-GEUr(Nx794>7W81N8$#j-wDcIrZR*aEBTSCAn5&-_Dfo<)G9qrbC6^lFRVUwO zOsU)oip5Q`=uT}{sL=GETF5*YERYx?w&st9)w2w!(d^6KV&v_25D8>_S7BNZxmL(Y z)+AC?yPdkC*TsP}syYKobOVjGIl|Uyi`+@;V-|Kw!zJsbCA%xJiUu{u0r7kXi=xs;rJMnrQh@^NlDIZljyErEw!ruRe0p$T`Z>GbCo z9%TMpRZfdqNO+ILLu)Y<*6O}7dbt1TSOsb4$(p%z*%a0BhsB0^s<{S)60rl;W3UVU z{QSAeVX9)AA5E-wM;hnv%thZLOd%*(rC^q(fyNQ{>D-(WxO_}m$r&o#`@G6n;M zgdDNg$BWxNP;noRH;elvU{S4L1>^I#2fPE-bJr@oK2=|C?tGg8>O$?@o4<|l3Ue+xj72yd-J&imQg zOU!IKB0U5eTf}FH$i_THo+co|vQMye%ffeCQ;gAL{9DqVcOE9TT=qUAut^n{im|0iWT3+cp_nfZqi zz*PLZ(D<*CcxlZ_6MH zYccd&B9KTbf^M__!p^5ucMWm|@Q#TwG4<{VLC>YdIXL&c35=(9X=ZN42Um6DPoYCn z+`dTTCII42F~no(L0zay>KL<=LpmtQ8LM(X=MBVyMDv7q2Q||o7@X&T48x<>mJ%m(VnK?QwpBmo&e8>)Oh$ofNReBk8zhaS8$-l z=h8@k`R^`d^KO~dA!xgs5Q^`7p+z*&Zm3@x2QylXMHQq{sZzR_DMWKaZ=C+cjl{7z z1MSE7mz?#54)4A{wFGyQHXbo{>o27`aWXB|VsS<@54i&3zMlpD z4BW5(1(?~cB~RS1TZi5>)i1zV?g~804_r1U6p4g^u+*6T+-1{)yWVSVS!2FVA%A^R z|J~pCr!TITwrppepTq`tY3_m1&`O{W!u5?3E%L`jaxY?CRyn#3kN)H1k~MP-Ze_EUW^asJacry)yR1+pYqYgfCB?7(;YJU=n zJK&$V`Cu@!p77;a7Ch)TTMDrNQU(_W%9~waW<&5%6A+;;WUr1s-;KIxf%(@b`6tNh zFKG(wdQb~c4w)KT0%OQ}z6vyphPnXb+IYSv zO|W#W1V}7a0znCaG$)*Y^RHcG*9(S3v2p`-4XJ>~yhr~(z!~m)(7i=pp5y-G-n=fD zfO+;Lpj}Y`bzSFpqP&&8fgqA0gp6&Lms9<_e^B{#OS-UI=L31lA=pDp?p0=q&F69N z5NULO?5F&@X1!DA5Aq+$?@9ml!TkB0_-j=B`x^{SVWPnvR&IC{ne^|jtiOGD@NH7! zpgWd>9y0&OYiLo1K473cdFC$&`oH^-z;ZI{1TR2^tcDU4cmJo0;2wb9U^Uyk*RMDD ze|Qqfydcj?8?9UC;z^M`BcCM z{AVfrXDMJ}{bwmaXV!nB0*HkFSqlGI3ja7M{<9SR|1AZOzXKsIBa{95kpy6uW8S*f zKK%RSq)OpeDroifPz2Pnf6LJRD<5ddy#%BR@64gjmQkHx3_>3WMMo%3)J9x+)&eAh zw95g5#&rTQk|29sKbIZ22v`DTFb;sc&U0r2|Mol}r*)y{B6}JKvizX4njbvv%{{OfR)8xg`HS-BZ?^K0m);^*-$l-GANwKV@}h9={5{rsZqZ++bZa5-1`ict3 zSS3Lf>Jh~^p~zXaz#~oJt#*3fJ0k$u+=_7W%Q*p}P4JMD*u$kVCx81YKj0_L+Qr%6 zP_YJN$Vbxxrh{>P2AF=c>Y&HL#JTb8d4-+IZ`Vu9>?MY{KH!U?1L@CNeZZqpr0E0_ zA_gW0?{6dmj`lkvFmW&kRswqm)((*0Zgc=Dl5xJo*bb|Zn6`#!TY)lBFr>gd2dp&g z2VJwSm~ zA9zEQJ?zSVB*#lKB#WU_#&bCEYuy7^3Vla_DpW!c33viNMxs!a+@|eRuO$M^SbW1O z0l$|JuX%~%pzR|)70*dQv&iK!FaITY>fd1eKgWTaoZKh6Dc=#@e?!Us^fT=0!H}Yv z%7(UwD^MGc2lcDd_;CXOxu_~JN-gG#yCC-|)E)5-EQmc1q;t(v%b-i01NxU-Gj~vc zAkqaiMkPAr`47_?024ZurpluW%#0h03YU-g1)wW|q-t%P$^Q})=I@!kI@B8}{?lds zwMq$AyaH(etlsL2@rSmNy4cg=Y=9aO`MD-J#z)O9Y<}*uAE$om<7RMYNR8q@u zunKy#_EV9EKHy>^^sPm-6o!%Y3ukRz0t1%AhuhuB zV!>@<^E@g~t36&qO1PtDp#+_5riYy#!_O)p&HofpPe%aI*>?hkb3>qvpIHf7DhUwb zQtUgP8VfnR`s=f0)wvMV>8exH1f);U!?SM{&a(+7mkRE`2X|xA=ld)#S zwG8SNge@R$HFQGK+gku{DK6`8Tmy43_a;53prb@(+SKz*3zY?SpHmVZK6&mq=xW|8 zzT9%_{rBHrE;t`R^H$kf_II(wO#0H`s2{X4Y_ItxY&QeoD=gy-D6ffu1#!*{NpSO9 zb%@k}bL0+RoGkkQ0NTdiC6h$&05ch6I#4rEaW<*cT?L(uo_ff{%o}L7n+5?5I#eQ5Fn6a)p5+YF+P^bg@BuooZZ3%14iga2-ou9=90toHET}-U)_n3$cEWeFQ zk@NT9Q#O4xyBr*P`J%#blXdXkY1?9HGJ&*bV0&YLlOsSKU4J}hPJ|vW<7gx#`d100 zvl0_@42n#5!#*_{bper?BD_eS59E{T0EOO9HT@Y}|MsI}-2KD_oEc)CcFl z$;%PtJ3iAXM-JCA*$BC@Q>poB1@xRewktk+11VbZwdoQ)$yHl5HlR+fs1I_8uBYUV zdQ{uVZG&m#0vR-~fTUPmM>7I!uLQ748^pF#`W%KSb-c0o0)X2|{0S5deC(%y7J1X+ zrt;wKOdSBDqv{NJ>%gK7WP_k!-x0I!1g4_*K&Dg61MvH;>>)|#Q7~h{WZNZh@4pVw ze*$=tjX*q(Teh|T{qn^#0D}!~Rpple&|ek)HAXd!q${s41C{q~^VviUfegE~q&7Kq(IocK8vT zc^lG0XVPO`NHjMB(7t>z+A{!aXsKHZxG-62a;21NJaPe==J6_m6V=f6gQBd{Jo3Eb z7}%+y`~VzmZ@ude3GN;DI$6&jHO?|}*<#g`e7mFYy!BxixL|rGhfS7B&ApG}0 zHTMJ*Z;)(1yndH41j}U;;wvV;-i(J)Fw!suGf*FYYfJ4`)aPmyr9PR)^-Y5WB|BM5 zN_p*QGB z^er3p@bu=rYU$kxhN|=v!6a*}vc_pA;7bg`Y=^o?x&;iRRE?3D&7fbDI$h*33B=q+ zVCFUJZQH5YNU*ox=^}uBgI)fzSq+TFAJ{ZdU-ug-kEeO}w$pHOD{*qSdsl6~6y{jN zC~XEQFpnW;0^odzi0ZqQDU8|6*`b8DiaL|G-TaReUZ=q-f+fe6{al)A>Vs}xA45F1efW_j~i*H9-cc;a^@+?^irGm1+Z#1 znXt=ghM0Py48$&tS(^_8?Pz}+_RTQyF18p*y%A9Q-Od6%u8b?zjjd!*5Ud)2#RBsM zG8GIcvfg1hC>Ae<@iuKGRIsy^zPT$}vY+bDtfYgYNpKK6uZISl)_d(HnZ0blc}z~? zmq3#X=F-X-NFVS%s$mR%h_|+s2y-5q1NKjFEb~l|`tDgxf%*M8Z0duSHyA}L)}Rhq z9U`}Qoi@i7tPvW2z&N01NOE(?YTW=Nf@7m|K+BE;3Vpuor0j__uRx|$&jXI944`IPAZ(g-?$*&gcd%7H8V~xL7jar zU#&{%re))GR!BLsVvU9rZyxgDgFM#Jo;Bkmt4$L+3SS0FOm>U6>PMQ&T{=%hstg3_ zhrs!vK`42$5$j+xbGy!Cv!_YNy127`;+Q7bJ2V~^%=o2*0}{36(yRP?$N$eefI9=) z7X>BX_5S?bKgYY^%$D=B_4B>_5m2~I?%0qbbc#Xyt=S0V9E>S3a;$j0Xz&oj*j2B; zf{pH6ZoZ}QcH0W*H!pS{qL9q#&UY0Pb88_T)Hy)Yv$4gt5CDThl_2Zje+i9G)S{zD z3S^_;B7BKE&;uL^Y0=WXkPFGtL-^Y0k>EJibL2Av1E7p=@79mSL6wZ%I0X1j| z(zovf!PRIIn3)ld`SezCZqN6y!L9O-42D{DffvUZM0+s44yN&Zh9-BAAYJ*@t6L{E zOHkH@d;cKK`lg!rA?=rv959AHg&=_S7sQ{seHoNpyk`0Qx=QwvUv(+BT(qQGx~N3U(Xn2QMGi# zZ`bL3&uQ)nm)}E0DI*Ct$&>FyY2CW3&7?tlpuzciR5z_=zjz5;rAzu#DBJa+!o$ac)%HY2J`XvjR!4iX(A@ff+A@gO`k z=&k}IKzK+9dex#;6wJBDc1yz*H`I|7fN*&aDf zLSH36TLgQsDs#7V>$#fW>l=Ov-8y)Uwq2x|8;?C)+wFPT{Er5*IPr?@lzQ2u_3$o1 zQ;Sb2w!z*00LJD8J6Mm=2poT7;F2BYHMGopU--+cd6g!~&Ax}BW(7`6Yj130<<8{_&yQb$gjfXu>oS+T3`lg3LucSvxdun+CqjYVz z}Jrx{nE zEY830fV>;hy(HBNGauSVAe;Apml9wEFVcXvm;1X*k-@)9{BP1lJIKPxIKJhS4Kxp@ zgpyp;NQwfMJac(m{EWw<&5s77uY8YK0t4Zp}XP9Qx&qbRhT9n<#DF8wT$g+N4 zaZs>zfkJP{H|x0tMRC;aR>O|sc(?n}`jsR@!G;|&{2I2QK_o^)j(p2ZX8D{Qdhz%s zaSvh>SlUuc6((Z4l~!FWM#1E&HfY-y@DGs?58kqt!+HTl8!P?etN=_UU{P~C_KCh3 zvUF~;?7+G+xdqPUaHL%wD7HEw$2XJH^abFeWJ-cx1IW`!kS*1{${0qy+hjx|pq>`k^)twb<2>+)B5hL@RcR&YQC3@P`BRd45Ms)H^Z z;s^MSIV^bpzSXBJ5AwNz&0-;@-(>gz#aL=djR}!Cx-x_*y}NafNW2=axw8>S<<=t$ z2?IL2q0AP^j>!a6_bs>%ijAZF2ldO)haF5ip9P!W`j5dfcS@3^5>J*BVZKcuZCdQ5 z+LB0l0nuq_7u0o=0g=HQuysuOQr8#T00Z+w?bn|1giDaJCKtvsv<1_GN8xfFn4q`+=wOIwg4ifc$Rs0IMW^~H8S;ZTOHhHVzhM=Q$^!f_74@M3;9WVn z1)D%9@oLVQB4@=2MSfvHg^nK+jA3vni7+Iyj-pA-w3A@`CP4Upw=`ds*ZvfwXIv1z6vnQU_}0Qx^LK{{ zCf0zlnd{RZX=~Pt2fzEFU^2!H@C4p^Hcqr;Oom91LIzf0Jrf|m>;&A2$utyjg~spD z(#C}YlNr?V&=@#^J0bcMSn;qJh8ZnXwcUD6;4G8o&J=UtVgZ@w+UO)tAlA@QwosG>PKA+vzkQtdeXC5+Oe%v<)Gs_X zV|R-QZS1QYExmoY{J=qFQ?hS@0ukxXfay_xQYVnVm`KY~)}JUfYI^d&;9vvFQaGiy z9?}$qzdJqD?^mu?TQ;`R6`5N1b{<2uoKtNvgj>x)Ceg+GWImYye-#{n_U3l#x!9UZ z)Upbalo~vTE@-pw!4)x7D=}&?^&w-dQGjNq&LO6iD;XeVmmZYG#d?4!c7bo}K>1K5 zU~41*-^5yE&sV0{;M-%1x{y~Pli3QGvRsIkN(n&VrXT^YYqz+Kz10OAo#(@8BdQjo zz}e8`Y|iv7FJq{&8MtVXZ#DZ@fd^U>*R$+Z;g*5PwuYwwWlv!aCH|V$qreg)4L#a4 z0s%}-ixy4>;J8Us)?eN2eh*b%3|^eahrp*oyBrLf#VT8LELDT-erGtV`#B7+mhqU} zsi;a3!Dl^0)&dN9Btf93{HeW8AN;h9I?zuTNaXrdA;Cd1>CUt?U1Sm4RZ}5J4xfWU z;~ZI^mz?ly^gXn`iqsectNb@d;0k2cB4&Z!BSZNZci)#DD3|@2Q)W>?>plU;>C9z8ak$e$1U0xXQw9;q9JTj)3X#?f&>Z zDu=14JpE+7>{Q%9})#sF`!o=cjhhW_8mh`AVSuBfYXh63GqLM%kBNc z^_fSSIS^7t<@SO$8^8g|Y+K)E`LBbOf7v<8p24qB4l4UO2*2MQ^(`MXrXQ24vfy1* z)OiKq>N-4PTQ&Za1Sr`5s{8?S*5maFWUAf;s=-km`(IegN$mp-3|W5#wsXEKDT;9ur}Qj3tJKkG6}{{O$B~YM|eTCp@g+83V16u%5nf zpt$+LGL%5zXG6F0Zw_rh`EtJoXg9UTB-nG z%!+B>2S(#k^*izrK&2xQ?7sq#&ozOZ%4cAkZ!=%8$@be)$BX#sDp)q@Hhd=MN94(t)+yn%jMOdZ`toy%}u`04++c!bteHHFFNr2)OVM66FLps zk72mY;4q<#g#);B&8{|P`4GxeIyw{K%$G+`5|1PtunF6$deDr!Al9r*+PTel-^L3s zuJW~fxOP6+0I0RT2)-DfL!+~{z!Q0W;NTs`-jxQ$^^VzLW8wmkK-TaSVK-|y0d*L* z|F{3Z8+-4>9<9b5Q(MGmeC`kY_s{Na|0PBtT$614(Typ+w}9tIy4mh$`J!Td9@t2{ z&9duBEz`nZQ$eSAan^ZFxSH`4c(P$W)4XlKwUv`{!gAe~fyN5o07olo!+jgB0PE`T zS3b@we*hEJN)-jogGQj9;_L%c&zwJR0Ggb+&*NqJ517u%BO1c5dCvqk%{;%YHe2xe z|FdbIfLH1kc(|=9`v*)jcT-qS?|fn*G+}D#ZbtLmzm9Q}r)tlfc&lMvj&4&*Gw+m3|yWYl`_%w*SbkO z0i07-GvC`K3OaXeT8jGXX)^x%752R`c8o6#h}oOh^hU_XeE-wj{Oai9zd}c*eD-y? zU*Yxc;+@-?`4xwyPgMiQToqTVYTp7Tho_UyP4{Pxs`52Z*Xt7SY^ZBzw}A@5 zxP)7$h{r@&y$Zc=%?6}YWzicY?L`rX`0e_UfU0yEA5hr^eBzFL@r zqX0~p#-Wm-@joo4@Ti*eMc`y>>_1>@eVV)Nt3Y4^Da+jCq|4F8^6U$@$fEU65Y7)UtGsO8OgqnKV19`h4sSMl^e%t8H+w_O%;xH8CtM9mMtcKWuOr`qwFpwl$kS-+@c zWCQPL)&fqRUOpWRoXWk#_5oD)`v&+rBRpI3BJ4)yQ`1X(5TJZ*n;w-2$Gla|(Dh;B~!EUN{qh z!UqE(frhUJyJq;c;%K)B%mB`AzuNh*O?p<2BWSR{4Y)l*GTp$s{}ZUpY1%O9>KkAJ zEQ!q9`ScWU(P^4`+h*L;iD#66VYK;0k~Qwd2Dsb-Elt>jBQiF;%0k}Gj@x_4V_?ww zKj9k($AMY4*Kf_lSw7*&5HM>D763OFF&+#-+TjY5!eSq$)Tj%93vWkbVKfXt3$RDi o!DtwarUP(|Gny9$NqO+6{uA$2gD17ka~XiZ)78&qol`;+0Iw(9EC2ui literal 0 HcmV?d00001 diff --git a/rfc/rfc-106/rfc-106.md b/rfc/rfc-106/rfc-106.md new file mode 100644 index 0000000000000..d344ccf579f9d --- /dev/null +++ b/rfc/rfc-106/rfc-106.md @@ -0,0 +1,260 @@ + +# RFC-106: Record Level and Secondary Index Support for Flink Writers + +## Proposers + +- @danny0405 + +## Approvers + - @geserdugarov + - @vinothchandar + - @cshuo + +## Status + +GH Discussion: https://github.com/apache/hudi/discussions/17452 + +> Please keep the status updated in `rfc/README.md`. + +## Abstract + +Apache Hudi provides multiple indexing strategies to efficiently locate records during upsert operations. +The **Record Level Index (RLI)** is a global index stored in Hudi's **Metadata Table (MDT)** that maps each record key to its +exact file group location, enabling O(1) lookups. The **Secondary Index (SI)** extends this capability to non-record-key, non-unique-key columns. +Currently, Spark reads/writes support RLI & SI while Flink does not, creating feature disparity between the two engines for Hudi table reads and writes. + +This RFC proposes adding RLI and SI support for Flink streaming writes. Throughout this document, the term **"index"** refers broadly to both RLI and SI; +when discussing behavior specific to one type, the terms "RLI" or "SI" will be used explicitly. + +The goals of this RFC are: + +- Provide reliable and performant write support for RLI/SI using Flink APIs +- Ensure cross-engine compatibility so that Flink can access and utilize indexes written by Spark, and vice versa +- Support global RLI for cross-partition upserts, as well as partition-level RLI for large fact tables +- Enable asynchronous compaction for MDT when indexing is enabled, either within the writer pipeline or via background table services +- Implement smart caching of index data for low-latency access during streaming writes +- Document scale and performance limits for write throughput supported by indexing (based on empirical benchmarks) +- Design the implementation to be extensible for arbitrary secondary indexing on different columns + +## Background + +Apache Hudi uses indexes to determine the location of existing records when processing upserts. Without an efficient index, Hudi would +need to scan the entire table to find whether a record already exists and where it is located. Different index types offer different +trade-offs between write performance, read performance, and resource consumption. + +Currently, Flink Hudi sink does not support RLI or SI, while Hudi Spark datasource does and proven at [massive production scale](https://hudi.apache.org/blog/2023/11/01/record-level-index/). +This inconsistency causes friction for users who migrate tables from Spark to Flink streaming. When migrating, users must switch +the index type from RLI/SI to either `bucket` (a hash-based partitioning scheme) or `flink_state` (which uses Flink's state backend to +maintain record-to-location mappings). This migration overhead complicates production deployments. + +Another key motivation is to provide scalable, efficient support for **cross-partition updates**—scenarios where a record's partition path changes between writes. +Currently, the only option for handling cross-partition updates in Flink is the `flink_state` index, which maintains a global view of all record locations. However, this approach has significant drawbacks: +it consumes substantial memory (proportional to the table size) and cannot be shared across different workloads or job restarts without state migration. + +## High Level Design + +The high-level design introduces the following components: + +- **MDT-based Index backend**: A new index implementation that can replace the current `flink_state` index, storing record-to-location mappings in the MDT rather than in Flink's state backend +- **Index cache with invalidation**: An in-memory cache to accelerate RLI lookups, along with a cache invalidation mechanism to maintain consistency with the committed state of the table +- **New Flink Index operator**: A separate Flink operator (`IndexWrite`) responsible for writing RLI/SI payloads to the MDT +- **Synchronous MDT writes**: The MDT's RLI and SI files are written synchronously with the data table files within the same commit boundary; the metadata is then sent to the coordinator for a final commit to the MDT (after the `FILES` partition update is computed) +- **Asynchronous MDT compaction**: MDT compaction is performed asynchronously, reusing the existing data file compaction pipeline to minimize task slot consumption + +![Index Write Flow](./index-write-flow.png) + +### Detailed Design + +### The Index Access + +In Hudi's Flink integration, the `BucketAssigner` operator is responsible for determining where each incoming record should be written. +It must identify whether each record is an insert (new record), update (existing record), or delete. To make this determination, the operator needs to look up whether +the record key already exists in the table and, if so, where it is located. + +With index support, the `BucketAssigner` operator will use the index metadata stored in the MDT as its backend. It will probe the index +with incoming record keys to determine the appropriate operation type (insert, update, or delete). In this design, the index serves the same role +that the `flink_state` index currently serves. Since the existing `BucketAssigner` already supports both **global** and **non-global** index types, +the global RLI will be used for **global** index configurations, while partitioned RLI will be used for **non-global** configurations. + +To optimize index access patterns and avoid caching all index shards in every `BucketAssigner` task, the input records will be shuffled +by `hash(record_key) % num_index_shards`. This uses the same hashing algorithm as the MDT's index partitioner, ensuring that +each `BucketAssigner` task only needs to read from a subset of index shards. + +#### Index Cache + +Streaming workloads require low-latency processing of each record to achieve high throughput. Thus, each record lookup against the index +should complete really fast. Reading a RLI entry each time for each record will incur 10+ms of latency per record and seriously affect throughput. + +**New index mappings cache:** Additionally, a separate memory cache is needed for index mappings created during the current checkpoint. +These mappings are not yet committed to the Hudi table and are therefore invisible to MDT queries. This cache must not be cleared until the +corresponding checkpoint/instant is committed to Hudi, which indicates that the index payloads have also been committed. This ensures multiple +records for the same record key (e,g insert to a key, followed by an update within the same commit boundary) are routed consistently to same +file group, preserving the 1:1 mapping from record key to file group. + +The cache stores `key -> location` mappings at the record level, the items are evicted by checkpoint level when the checkpoints are committed to Hudi. +(Note that the MDT reader also maintains its own native file-level cache.) + +The actual index writes occur in the `IndexWrite` operator and the location from the cache will be propagated downstream from the `BucketAssigner` operator, where cache lookups and MDT queries to determine record locations. +The cache is updated for new records and location changes, while the MDT is queried only for existing key locations. + +The cache update flow is as follows: + +1. Probe the cache for the key. If found, update the cache entry if the location has changed. +2. If the key is not in the cache, fall back to querying the MDT. If the key exists in the MDT, add it to the cache with its location. +3. If the key does not exist in the MDT either, add the new key and its assigned location to the cache. + +#### Index Access Consistency On Fail Cases + +Hudi uses a two-phase commit protocol where each Flink checkpoint corresponds to a Hudi completed instant. During a checkpoint, Flink completes the data writes and collects the Hudi commit metadata. +Once the checkpoint acknowledgment event is received, Flink knows the checkpoint completed successfully, and the corresponding Hudi instant can be committed. +However, the acknowledgment message is sent asynchronously on a best-effort basis and may be lost in corner cases. A Hudi instant cannot be committed without receiving this acknowledgment. + +During job restarts or task failover, there are scenarios where a Flink checkpoint succeeds but the corresponding Hudi instant remains uncommitted due to the two-phase commit mechanism: + +1. **Missing acknowledgment**: The acknowledgment message is lost entirely. +2. **Job restart during commit**: The acknowledgment is received, but the job restarts during the instant commit process: + 1. All write metadata from a checkpoint is collected in the coordinator and is ready for committing. + 2. The acknowledgment message is received. + 3. The coordinator begins committing the instant with the collected write metadata. + 4. The job restarts or crashes. + 5. The instant remains uncommitted even though the checkpoint succeeded. +3. **Task failover before acknowledgment**: A task fails over from a checkpoint before its acknowledgment is received: + 1. All write metadata from checkpoint `ckp_n` is collected in the coordinator and is ready for committing. + 2. The checkpoint succeeds, but the acknowledgment has not been received yet. + 3. A task fails and recovers from `ckp_n`. + 4. The instant remains uncommitted even though the checkpoint succeeded. + 5. The acknowledgment message is eventually received. + 6. During the gap between steps 4 and 5, the `BucketAssigner` must access the uncommitted instant because the checkpoint was successful and the data is valid. +4. **Task failover after acknowledgment but before commit completion**: A task fails over after the acknowledgment is received but before the instant commit completes: + 1. All write metadata from checkpoint `ckp_n` is collected in the coordinator and is ready for committing. + 2. The acknowledgment message is received. + 3. The coordinator begins committing the instant with the collected write metadata. + 4. A task fails and recovers from `ckp_n`. + 5. The instant remains uncommitted even though the checkpoint succeeded. + 6. Step 3 eventually completes and the instant is committed. + 7. During the gap between steps 4 and 6, the `BucketAssigner` must access the uncommitted instant because the checkpoint was successful and the data is valid. + +For data table (DT) metadata, the pipeline will recommit the instant using the recovered table metadata. However, since the `BucketAssigner` operator is upstream of the `StreamWrite` operator, there is a time gap before these inflight instants can be recommitted. +The gap between Flink checkpoint and Hudi instant commit will incur consistency issue on index read view, it is fixed by cases: + +- on task failover: the coordinator would recommit the pending instants with successful Flink checkpoints; +- on job restart: trigger an explicit job failover from coordinator after the recovered pending instant been recommitted. + +#### Add Event Time Ordering Value for RLI Payload +For cross-partition updates or deletes, we can not update the RLI directly based on the existing key-location mappings. Currently, the RLI payload only has the key to location mappings without actual ordering value. +we need to merge the data records to see if the incoming record is a valid update or delete(for valid, it means greater ordering value), that is the behavior for Spark RLI write path, but it is too costly for streaming. + +For e.g, for two records `r1:{key: k1, orderingValue: 2, partition: par1}` and `r2:{key: k1, orderingValue: 1, partition: par2}`, `r2` comes behind `r1` in a different commit, +comparison of just key existence is not enough, we need to also compare the ordering value to see that `r2` is not a valid update and +not sending the retraction record(delete record) into partition `par1` for payload delete. + +The suggested solution is to store the ordering value into the RLI payload, so that we can compare the ordering value when there is a match of exiting key lookup, to make the decision whether the incoming record is a valid +upserts or not. + +The query execution follows this order: first access the in-memory cache, then query the MDT index: + +![The RLI Access Pattern](./rli-access-pattern.png) + +### Shuffling Index Records + +In the `StreamWrite` operator, index records are derived from incoming data records and sent to the `IndexWrite` operator in a streaming +fashion. These index records are shuffled by `hash(record_key) % num_index_shards`, using the same hashing algorithm as the MDT's +index partitioner. This shuffling strategy is critical for avoiding a combinatorial explosion of files written to the MDT partition. +Without it, the number of files would be `N * M`, where `N` is the number of index partition buckets and `M` is the number of data table +buckets involved in the current write. + +To ensure that each data record and its corresponding index record always belong to the same commit/checkpoint, we leverage +Flink's barrier alignment mechanism. In Flink, checkpoint barriers flow together with records through the pipeline (see [how-does-state-snapshotting-work](https://nightlies.apache.org/flink/flink-docs-master/docs/learn-flink/fault_tolerance/#how-does-state-snapshotting-work)). +When the `StreamWrite` operator receives a record, it emits both the data record and its corresponding index record within a single `#processElement` call. +This ensures that the two records are never separated by a checkpoint barrier. + +For example: + +```text +// irN means an index record, drN means a data record +e.g: [r4 r3 r2 r1 ] => BucketAssignor => [ ir4 dr4 ir3 dr3 ir2 dr2 ir1 dr1] +``` + +The barrier propagation algorithm prevents the checkpoint barrier from being placed between an index record and its corresponding data record. +A placement like `[ ir4 dr4 ir3 dr3 ir2 dr2 ir1 dr1]` cannot occur. + +### The Index Write + +In the `IndexWrite` operator, index records are buffered and then written to the MDT when triggered by a Flink checkpoint. +The write status metadata is then sent to the `coordinator`. This metadata includes two parts: + +- **A**: The written data file paths +- **B**: The written MDT file paths (specifically those under the index partitions) + +#### Committing MDT (including Index Partitions) + +When committing to the data table, the MDT is committed first with the index write metadata (the MDT index partition file handles). +The `RLI` and `SI` partition file handles are committed together in the `FILES` partition. + +During a Flink checkpoint, each index-writing and data-writing task flushes all its records to the index and data files respectively. +This ensures that the index and data files are always consistent. Both are committed together from the Coordinator as a single Hudi commit, +following the current commit protocol. + +To maintain exactly-once semantics during job recovery, the write status metadata must be stored in multiple locations: the `StreamWrite` operator, the `IndexWrite` operator, and the `coordinator`. +This follows the same pattern as the current approach for maintaining data table metadata. + +### The Compaction + +To minimize task slot consumption, the implementation reuses the existing data file compaction sub-pipeline for MDT compaction. +This asynchronous compaction is automatically enabled when indexing is active. + +![Index Compaction Flow](./index-compaction-flow.png) + +## Implementation Plan + +The umbrella issue: [Support record index for Flink writer](https://github.com/apache/hudi/issues/17647) + +- [Add RLI access cache to support efficient lookup](https://github.com/apache/hudi/issues/17697) +- [Adapter the BucketAssign function with the MDT backed index](https://github.com/apache/hudi/issues/17699) +- [Support mini-batch access to the MDT index for bucket assign function](https://github.com/apache/hudi/issues/17842) +- [Add basic infra to bookeep the mappings between checkpoint id to instant](https://github.com/apache/hudi/issues/17700) +- [Add a new index write function](https://github.com/apache/hudi/issues/17701) +- [Integrate the MDT compaction with existing compaction sub-pipeline and offline job](https://github.com/apache/hudi/issues/17702) + +## Rollout/Adoption Plan + + - What impact (if any) will there be on existing users? + - No impact because this is a new feature. Existing Flink users can continue using their current index types, and + adoption of RLI/SI is optional. + +## Test Plan + +1. Verify that all write and read scenarios work correctly with indexing enabled. +2. Validate that asynchronous compaction works correctly with mixed DT/MDT workloads. +3. Add new tests to cover job recovery scenarios with uncommitted DT/MDT commits. +4. Conduct benchmarks to determine the upper throughput threshold at which indexing is recommended for streaming workloads. + +## Appendix + +### The Job/Task failover + +To maintain exactly-once semantics, the implementation includes infrastructure to support job and task recovery: + +1. In the `coordinator`, historical uncommitted metadata is persisted in the checkpoint state. +2. In the `StreamWrite` operator, the current write metadata list is persisted in the checkpoint state. + +When the job is restarted, the following steps are triggered to recover uncommitted instants: + +1. The `StreamWrite` operator checks for pending instants in the checkpoint state and resends an event to the `coordinator` to collect the uncommitted metadata. +2. The `coordinator` recovers its write metadata list from the checkpoint state. +3. The `coordinator` recommits the uncommitted instants using the combined metadata from steps 1 and 2. diff --git a/rfc/rfc-106/rli-access-pattern.png b/rfc/rfc-106/rli-access-pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..00888f9c5a4a93af740e21a448b2533357b602bc GIT binary patch literal 96912 zcmeFZcUV(hvpx(60zm|ofPxeYN>!99HKIswA)!|h>7CF+QOY9%qSBEjHKBJ1RS-~+ zPUuBMTIjt7emgwpJ?A{?`~LlX*LAoq5=!>kYt77>x#yl)`<0rCEH#J`L_|bHEid;N zNXQ==sHn zi3GZ@w^Cj_OsBZc>lbpJP5A;vYE8=4b&kK-*c8Z#Ik^lJiYWsOrDfSDDM))uI1>VD zHpLQ7I^wGNtNF#8&9l6_Hglclb`GnW>}u-wllv?!(GM6&DA@dn{{0IjWo|^b&oJ-5 z|HSF_hJKI`8j}C`A;?D(Qs}FVVrNoX2odqWeIenLJoBHY^a?~P$KKZ7dEq||LrRgp zckVx~jN&F2ge{8Wu9)n9m<=f{Tf`6k|0Ey^wuo#Zf7r*GrK|sbQNU^e0d@TcIRi`n zKLP!pfd1{a_&*E!|JW9Dk#|}B^OpAqU5Mq&KsCon@|&5w_UKa4>3?dN&tJ~mFSv*v zrNKp)+f+Y9Sgv}!JoN_GH)i~U14^+WYY`GSaQ3p&-(0LTD_Fn8^4`p+(s2Cy5>^)R zQg0QOZ6~+bzAS=xS)%o5SX91FS1yXP=6b0DGzocw;P{?hWQ%A^ZxsK7Y5JePP}sdt z^E&fMYZtq5h-UuxS35z^*8O-U*_1xPFY7Ky_`}J3BSa9uwJAzu6zOeSEe^WBdv^#g z#pg5G1C6(-Y;s?4vV4x!Du-lRI&BWADoX18n61Tl+hogVFpOrx_pR(OA|;!BD1`zW z&9fJR;H2=(Ms9-GAYOpWA4zKw`h9pJl4W`|iVRS;&D496$cZM5?4docKv9G|x3IF+ z`DpsOzMmENVoTJa)9gD7sphM466ZWVo+-W8BT%(pGoqc>M86o%Q;3tumCc!R96Mi& z-8Xh+vP)(#SMAQM?*!+mFwFhn-kJp@H8`w@JJS1{enuFY^*K8Dn>4_MW7SCI-;)s z9>Cba7Q6u=Bb&~j@2_HqXRRlzz+$bbAGfxbtSvN|>(bGciAKXbGSJ*qx%c6^ zsJ1qY;x|q83&Tv5SZI4xy({i6H47p&qaE#)Hz+r*c;MP--=uC=Cr+T;CSAc|J)qsS{CQb5b zewQ!@q=B3%@dK0K!KFd-jLrvSYV?wcb-7I9FS1DNAS4OTsq!{ZR(+OX(l8BrBcucd#7e;bzy zX|0sFLM`$*{fg~&v%Rg9(Ff6i52G-|HoWOj5ybj3zd_&6%uw3Xx!#A~yF{2OBQRBT zRg^x(AEufNOjSM0=ZB#O@?j1WR{(U^i&QP|K6a^b^UeocfrOqM)+0kD-+40sN)yT* zOP31k@CmJS07fMSMrEx0s(=0WQE5T?9$0}%_Q`LuQBSuam{pUB8U~44uhj%AAY!Um0@g=-f zlg9WHN%s@oa>|0!nSwb0jrj|QX*~H|V@?;;Xc3#X{QR0S>eNxM*%oiI)6!(Zu9Q8S zjmh6s;I6Eq-n`FJ?l^Mm${asIS&)?b1TjoYI!6CN42C3g>0IiaZ{Qz?&ciMstk4Gm zK@eH0(4oyUgM9h-M8kZ@w-I5~RQJRAx4V(n`wDdyX17k$8ZHInzz|VA6?(!wewioR zZGZA(iRxxgbciPhcKM14jEeEOOGHBxsw<5IYRzw#xQsjNCXeqQ&Yg#kEA;+7pq3v5 z?X=2l9)5}z{_*mRP7TqYQ*c*KFEjYqb5Du~tk6TtaogV=G9utnv?x{lMnl+VKp=?j z)cvIt0}+_(@Yr$Z4=bi3Mqn!hVig&X#An}|anfqa3o6?=QQhWsLOjAeo;xt+Qp&hS z2qUm^V8feDN_DdQzCsJQA0HadLLdDKQX?iAb^|V4X&v{0Im@3gYqfKLAwT6PK1D45 zyf)<>Q2o9(Rh2L+h;JDfvZ2!K*rmmEryc2+x}#e@Y{bcQ@>7BAG9o#Lky5MUSR z@3)*N{~)DX{)sf&U%Tus56*j1$IiE88d_3BzW0-Szs`rAe2Bzz3Mm`HRI=KtPB#G^ z!C=|g5>8>%KexpU*IM3N?##q^%&LYUh}z$Yw%h#PL`HgJw%sF4bkl<4`O~}0&;y(5 zK1cPVUn~juwM_?tSYkL8-HhT-Qj?QZhJpImqXKdGGqU4W#x0=l-^op7G{Z53*d zNOHnizIzVthUUi_bA2@a_?QHjr1~(DL>+O%GSAK-(K_e8Rb%8Wy~Mx6H3&QpO`Yqo z{KGORezK(msdNmoe%~?{|m3#DT;w37v+-c{#vD!u#ijew(x(3tmM4y4eNCMCW@b6I9^Miv_@ zerC=~z7i%uZ;9UyBgu{;58SXJCq) zO!`dB6FmVBNAzncyuiNQfVY-4gP^?~anwXlt&wJLytXlMVGC(?`2Ji^g-b#&yysBa=^HoU+ftyUegMZ~vZ9qo&DnfX z=^>VNl8OR~KC<@(o*DP+n8Ji#?DMp*yM#24f5so zZnzs~cvwX->NlTZH+T)_v()Y2ii@A*6)xTltkYCjyGdT%6==vmdb$BzLIL;ETIme? z!}On%-VcPiuA69Qq$uQBI=$yr@hVGcGfV@JLsUn}TH8dcQ67j}QfhK{y@8`|hqqI> zPl2+dq(Q#AVr56bAmq-j%hmrPCm7 ziQ0uv3#vZ64%9BUyZn5(#y#6?b32~%J1Az3pZTopaJj||bg0qz>+GB}zMT4RlLkB> z%X1B{||@qN7HMISiVgVbM=86-C!W3NNj4ckeEgxbL-#jnV^j9=fr`^Rp%oxY)Su@I{udiBGe&pRcEGIGyQ1G%)FeziERlyS2IOc zOQ`iX7m?Y&@k9im5VCO|l~Zu~PyGY15=t3h0>gJ39v4!3kUpYUv21!80L{SIg`Z-9 z2*PH(pTG3yYeT>kP1$`aEb~bUWa{)!I6rvq9hZHyW=hZ%Do^d9&RLx?D*HYLuovxM zCDiMBzy}&sTr)`9S2|rne>5a31Er}K^9}n4`@`~fTQ@xR^(TbV%*p$=!Dj`kj&B@d zFrNa5vBk_} zLlC~d!UgSAs7mDK-?YMSqWDP$V+Px!-$CIpNt4K(RNKmHX(4Pm zel6a`niiMAE3KK>z-O6UJbImflP(A#-oodQGw46XG}%3b5=0ib@^1O`c3g6RS9xJapB3`oTls(SBan9_ zjFYc|PX7hD2*fJ+>|)hF?hP&_wk7GilCv83?yws=%Iv|(pEy175#6F|mb$`39L3UC)eebA}Q^?lwje++(}aX=T0xA8R?-gbMAc%-Wo?=KFiYvdHYsIytJuow2d7H*1=1sER{L0X2 zVwBQINH3Ry95H`%jW;<{iqZiHz7;%#=%NX^sE` zhCdc6j5N7i*6z<;^~&1yPIgY6BMSN7d2qpH*h=m6w?FDsP9KsA8-!)_hU%HiwnF(b zk`Mab&BX37l_*gq>(Vpoz$0flyFGcqPpZVZ(n^{C&WJIDH7@!%3I4uDNPNSOF8Ykx zIck#S3u?Paf}@`ePh15OoY$_Y-#O1EesDE`)6yRJ-|)1aglL|C-D zh6wxbi%z*rvT0;6U#DMLdb#>3D}_7pWS|k3u`Upl^-6hoKxeq}V8WH1+vwF5x~Je? zx+jDzJP|53rJ?`V%~3)enz;sSNS0myl0WRKi(u`I;fr%;8R%(mdcIhy`AAA9r1QOh z0+d~TN82PazL6skdYUu%x6x~-7ON(g{DFz2{UT#LU>a73W{*MWko!AbN|8p7tx(Jn z*7ox*P%^E*qW6nQ5`G0=E7XRoqn=~g2by(dK8WA`msf(b0~3ZuCV+$eo~?Z6icmMxeBd_D}f6{)g!PKjy#;(1h5x2UnSX^Fa#N z`Mh>)gdKl8-kl0&jsEFA<4T*uXKACYW|!5n&(U`^>4_8jdj3Tg( zal2t9zXe)U3CX??Gw3&jm~h6D|4myHx6ZCzz_gP(b2&yR$qKK zw4^|nI8dEH85O9Sz?i3#;LEkMjlj0uA&oxCN#Ds&{5_WYGhWzRD(M>A?2=lg=h6Ot zOF8~$`KXCYC)kb&*Y7J=tY8*3JIaC>pcM)X!Y5^cDh4~e`!=rt9!nL* zY?c~1FzBKkCJg~|-A^oqi2v2Qe!$#6h+yiCf^eNVuJPC$%T`YtQ{c|~`Q#1DvgeM? z`Dp&!HUhLt$p!X#8_w0?{{NoSsauT*@FU?2R3FGxNyM>u{JgOHtBkM?$=XB_O^TIFl>uV%760_O&CLWH`CW8;c zDl9?d78G=f5Q9zB!B>2Ycm}(v#7gYWL zxqW4isieA7*}&}n-5={yrEf$HL|p7z$6W8P9x@y3-sa`9ZM()qVtgQN;RImO(93So50V{L4AsfCmW&D&}d3g`8&)sUN znMjSX39r>C{TPzW7C>{QUe*1J_|QpV=SHuhRl?A*KC!C{)!u zXe)BFLs${H_c^@6WdZ8;?Oto8@5$k0&%I~9g9dI@fZDsX@?@wcO)^N@Q}!FDgH zg$gTbuB*X;U;#&nRbBoqSbVwJt;uFKnw^t-JITF~Tw~O}zV;*&(-Na=76rG73h^n+ zKgeZNV7R3DSZD27K}UOxly|4gy6)hBmbRIRCa)&u*#@fFV`m>ErRNx||1ukM4b>V$ z-Cn^m+C3kH-PM&$dyz%Q!|XQZm=~n(>OmuQRH%&MtBIBHauukDT>{zVg2f`BfzbT= zg5|3r+Xu-fN4I%Hi}5(d)w)AFx9*4J!?}-*ZY#&b%Kal(kXb5;7i*6d#yw|J3tC4PMD|u-jePNl z$-v1ZuayNO?m671mD@Y!Z*R=_kua0CA)Ek zl&SYo0$tVH1A1wJIeF#r)oYDZeJoqPGu%dwT817OtVU?Sx%6NdN572t5udRsepXqdWmJeY5gHqN!hSs`T781Usg;1AsZmAOfQyX3T4~c8*osuBe&b=UYqxu&tKI&1CFjP?$DpbU6tZ zRn`_d<)fX$W*SNXH*xozxyy}d>@(Z(F}}}5g_(J_PC9ncRk=+0X8@x@`d^ne1R`pR{qh>usLhRa{1^iLK0i)B%U4*@G}Wm4Ujm`R$)rpi*x+Y zl+*#;=IFeH1wj^;z{P-v?>f5ZI)@&Nh&Frk&epxID)25)-&G!8LzV3s&Uk&hIdJm* zCC%7w-LAxmn$F=#U-I$K$=i;(dynvmt|OdtP4lT0Wj^?W;dHO>{^VoovdlA+yEtsS z+jiE4b7!avJTL(*p?4J-zo1HHUy^CZiXF&3|-(*;hXtsUQT%$45N zSpn+@8XBdJ#vGLB&F@EhdXb=@ubqEyh~Yr_)!8O}J|lo2%U&tUREql{d2-~;s)l-< zoBAmD$zWD+XF<(Ih{mqYqo8xIW(Rv8do`h;Ogi768cgbZ|F}7yjZU@!8^)olzj13s z#qe4$V)Mrb_+N%&e8k>BYg?-8`y9bSQk&oSfjE?(opsN4q}0Net9bo;36fzNKF;ZL zwBD-XduCgB<-Mw;?$-$WlAOybr14bd8x9>!drB^#cfSTTe57KI%u-CZTBUS2hYYT2Kp+>wj12K6efGEb56GgFkwL2p+G-)m@Aavf$sEnS-ld!O_k5rima;Ay!M)^X&-t^UwpH_slK6C2!C zSx_jAj*T+9LKEHejhxlHo7v;%+XuPBPEj_|<(ZPJ)2(mZ7}FNV-6jzQ^~abK;U)ki z;yNG9UCmOMcCwq<8Rj+5QJO115;n@r`1V_9@sDRvt28xvD}5_LbLit*bF`jzy4ozk zr=DciVK#dX4%DfPw$)_E@0C?75KlDtpTW;g`Pk|jh%J1IeerRxVs+H(TX}ik%ks6U zVwMq{ziUzTq1B=|7osXqZbuh2_{l@7+z%6=!58&dqxl`?nFjlNK?E{dmFf&9KKxY$ zweT3VduYG*caw6g)K_K*lnxgWTtKmX642SeEW! z=C6At%vxK-6N`FU-^Y(&MEf0Ht8%VX4a)OP$V<<|BDD z$^FQmfPai}+y^$>f?eJIb79Y|`OtR$jw1lC95Lu_+57YT>?pM>_c(A1L*rd&`pKT_ z`5RXuT}uqKsyc5+a1DwzdHhli`^lbiPqNG?QDBFs0*W*tEJEQij zyrv&@AT)AdJ`|@7;QcF(axfn?G`gPwO<~0+Jx%;xwCUszcKB|4Q)iXMP2~JQMGOoL z4Si&~7N{l^9>TJLlWkS_L2PtqF5lkMcTQ~bs~LRYA(8)8(j>+~0DRtMwZ9##98wp; zGL+NE&vMrC+bslp)}75Q2Sb{Cu$_*E%oiHpy+>Y5wOp}Ua#&3@VMP^lX zxYuXU(4JNIKnW)Mnxca8HiQ@Q%C$)$>w#QNVfAdsA(LR4>DA%M2u)!wV-pejnB`dj zW?1**(H)97!J0bh6QPj@xH>~${PqfeZScD zmpdu63dFF>;%0iI13fu`n)OFBck#IT18hODt!CNgHe{xDcVct0cTjj|dvi9y{@_kh zZm)cSrfqt#*}=j{S&pwew#wQ31f3!~S!rlryX`%nwyUdJu%7HtmkFj{c$mAB!P?Px zZ=3Ji0QS)hWXb5nUER_z!%f-}wC8iqxs;S9?PNTk%hyWSp}Kg?inx(&R}An2cTdzr zV}ea7lP^}w68T@Jod0wzG-@-=y2gs!>RI{uUUkQ7U@{GWOj4pDERi{jHDW83H$b1^ z9dA(*M{7;TZu5in$A}X#Qy}+QTO9-taOkH9{Q3Sm5M??Y#K3(ja0Q?v6vcQY#s*&= zbO?=p>l$3eB}sT?=3Ua<3V^4U_qMY7E*T=X;aQWudUjPA9pQ#R(rM(YPo>~GVtG3Y zbj2E^Sa*V79UXma1i$7Cc&AKe(=x24v-RwQa;=f`8{VeA6{T^FU)}p20Ks{)j_Iqt z&t^hh^~j#+j-juxedW~I-Gs@bmggC@ZAYvpd%-8_W_`fBY(*Pa&f~@G96e<8_Ap!n z`(FAsp6S!bdMm5_d)qo7*$lhV#BpVD4}{!$Fd(7}WHLrMhkf-IIz<tmHBN*dZ5S+p| zEkmO>W`unNy|#ZS$$p7;mcy4~7JnCaweZ z7*QDH;l9+#zG}tpgjY3I%Sd8bUlMRH*&~mql{g~~03-1hR2>OH38Vk=-0G1<3Wh|N z2`0TKA;$+O*DG%M8BB!+(whVZp{1vleW&TpGJDRa>iCj}F$s4v`|M5ilm#4iuPPhF z92@E*u<(@^RCm^B4w`Ux0?7w{5Qj8|udSkdgwZ1$=gRs!MyKCMt@zgGT$icm_T5Tj zl}UwVE*4LN54S7o@p)c9-^%v7=6D_K)}M?syL3Oa+j*ZEcv*-G+7}otLXZc zF%IAxlB)rt0lDkt`6XljAgGAsgS*C;bn?ds>cZl84U_F^4n~x#DjSi6Sx@ju}TVlguXVn+xyfqFknhY1FLVYo9nGpp^v?vNX+k2RCi!u1w7D zhWO+z3X(CHm|5&K7$oFuV9`ojUTaxWc+1zdxOPfVm9F)HSBittr{IxSx0B{^i;GA| z>Mm$1?<+*vS8J=#2n#pSh;RZ(vIBIaLmfJ+5d_`W)>zXD8R+hg5d5xI7AF0bj|V8} zG@8a{g=eh%><1@GS`Tbh97SrVte+fHRtNMuQK{J&)N;W@r$hO@W10^Rw??cBssKO2 z4w@>p4^%ZzMwZjuD+?RYU`DkaxE2 z!T9A`fcW|-gBFc!o{2GH>FAr|)3}TPcv~@i_C+O=t)U`QB*c97z#eX(i>rTlxLSWz zvbrY_K1Avd3moTDVfY7;l?^ABkfW%c)1ZJ`xTh~~UplKH`sCiu_=R10n!hCXzMsh= z_W<~gnNaB~5;dW|FAc`qIQM$osd0?*G%hf*j5~H)?-#B?W%V`kTl<@8u00Cc!%q80 zv)o)5T?OTa^lb6^N|1gKu!NS1Avbkt3YLhYdAlEt?s!v;mCpeMbWs(^L$#Vf6aQ$G zUa5nG#1P;fyV4X`@hd&62Wt&y(#9A4D_NX!;y1&DC)Tj=p2q$F zpNi4dnZkBSL>i+Ib9?;p+0O}FD*S=!o(=_$fi6jR>yL`1%n>t;Lh$FDV`@}6iT&*8 z@Ep6bWBjUbaKht09K8xyh%t>zk7^fHZ!RxPW&sk&MWGRW%EDnQi9X5PKSg$iOl_m) z;l5gCnGUkMuxqpD#g=G!yWFmyOV8ESN#@Q`2w7E?MBneNH)^EU?6R!ul*g)?q?Br) zy&mH&l)k8^J&{Q^3e?oIuSK-n{iuzIU{*9D`N|&AD>|s}9CoDW<93S-=a2?_|`AHv~6p#cpe?$ z%-L$YO{+DiWzl!$_Wp9+;Y!`8G~zY~|FV%(T;@zp;Txcu9d$beB3&=r6ujP-rK(y@ z3pdj3Yg%>gk{Hyo7O%1qzlrzJgqS@H&#QiJ<9%IX?|e>T={S&DF23YA95-!mB41pG zaOH{@t*9=?dX|wd390(~AG-z^6yodybhRBn?oJ5?5Qq5GbH!3MaV)z5%+v<<)`+Gu z&EB?}lwL`Lay#50-bJG<9y42#>6PVtzW#9v7bo)Tl-T5cc;0OF6_{T(*E2yp9_s_3 zfyS=bQnde6`7mR!Fo+N(A!V;HtskR zAOV+Le8=%49d{1nMj`kD3Vh!A!UO)e8i3casU;m>IP$lx+cFS)Q9Z6z=r zPn&HJ9I_bO8$$w5(rv=J>u(^cC`G?ZH@qnI;& zPWa=2)kOE0jP;aT$JM;&Zqr2uV0A=WOy5Xkj>{_hKfZj*hdod6zuNku!Z>rH%A1430#9g!T=Y#_77-m+C)Mu0e?^8n9V?_*yn-C}vyK zk_;otM&B2U&P(Y^@uAz`j0K3h2wnv*_+@k6bJVs;hHy^i2`OX{ydy(R*r$_FziCqD zS7CK%AXaC~0B;?#^%XrzUSkOmkkc556UJ|_!JW_ zjdFJrDL0g@!%c;-9DG0w=p5@U9PAXZ80%cl2sZWX-l(3$cj6aReK%+{f`A;nI!+@3 zs%R3q@}5PSGuK*maj+n{?+mI>MXX~P;Lz&y9yMm~$Q3J>QSp2z{Q}hUCY9^W!uwp< zEBG!$=UIhvZc@&1HuP#MnnVTo%<1-N`>fMzezu5{&e7Kuj{rTQvYo83pDZ1(0XX2E zLq6l)Z;xms1Z+Q~STy&422g+zYvWK&p|sp)BPfn9^kAT(ZiYLDE|l|G&#J1ms85aJ z1HH^azO`QW;PuSH#|A@|-|pbYxzjlX)U+A~=!kSU(%Ww4Tr8PEN2_m#VdANz&Y*=f zXF?HF=8JQOjIL$bUW_b{qRapi85Rf*3SaLs-}Cq~V;?1{sw#zdalss1RLrEG)N;S7 zcM@<(#i4$$`R-Vk&klc<-PLL36QL^-o~5fQ+Np1kt0@ymq{B2a(RTy0)-(CO#B)_i z-$*FXLhj$Pan0YcYX33#Y_n<5y&`Hr(4jIsQ*nPdjJ3LV;?G;}LgBqMmnwO!C(ZKH zsj$6nt~xu;N$$UTU*15lZjMeK4*>O>la28Mj=y>5U}_s!BBdhQvc8zhhK;yiY_yvxs+0q1(6TfZA=gYWcs*8PyAeo z1OGXEX&9WZrJ`aN{Y7*()^sVa&$ka4+G$Jj1ILWHu(%z&3ZS$f2GbvY@t3rPcQzH6 zpIouMtgw(OD;o^;+)FD`pnGhYVn_WXKbYS$8%1sjLJ0%~W3G-^q=gaa=bj(L?+oSoB^PO3F|7Z z%n$=!K|6g@_w2`+O%SWLNKVVvzuvT`-)U)F`>HvxYGOTn>|;~2-clY?AQv)GdjG-+ zQpmI=l-{}ks0Q`a6pAWhnb+G`;a3-S$AIe9@_F$!$fRKMadZI92{zH7Mz7vsiCrxA zMO7j6@g_nEVPwI}k}~978#IcR=C%P{YD9A+YFy)G$!=~X>z(W-B*=@`U>P?Hp2Dx1 zqa!rd(pQM5d6#_e(og>DSpZfl(_Ddr7taR25#RW$`bP-2cSYEgB%LnjmU?%FT5mi$ zrg7!Ormi-~|M^kn{2-&xem9H0>E0`p}3yD+B*uf_n%V|}^^)aZwB3!L3b8+XCh zq~t&oy&zK017MVP#>p41y=I|9DTE9_*+$Q#J1CQ><2>bQo(m2HvedP(ss$~^9By@| z){e#YzdX#8f8CCUJ*Wp=P^(M_Wh>=LZhMU0+csSudmSzD*s$CO=upVtEo3PdDHz0j zGT79uzs#M3?$_N*(b;Ttw17Rlq^UL}RaBTGn3o1=tDcz`mtyl{EQb81!J1LY)Zs+h3~KI6ZW zORbsmaHU6BTf?DuvOw=O-!A`iB7R#2>_md#5&KWvzT-piR3!pZ4Z}Pk!+YQ1M!xv^uz`IwY_X z&DGgYr-EK=D%6vp%7jH_-JKZPGq`^YctYHhFh(#!-sTWn@3Ev0&|spI%IUlSV=6q3 zxdxNVT3r&q#A zo}K4kTf!`REwXMZ&rfMib6(`aaQJ)6I!s^BBQzlK!IN)E5~Jkmj6Zz<&$VUhJFhop zQXw6;qp`TDOO?Lmr*funF*R}}Pnr{H2(+sf>a*!x&S1lBsh=4yWD70M1I~qbk?9;P zxTzTp+R81Qxh0mRZ`5R!!uO$f2d%B{hVs<*Y2(Z)okw?tr$*>acjBj6Pu5E4L^$Or zM!QoV;m^2rw8Kyr@|je)^$Q^ia~ptoMD>?^-{21?Pj)tc7=&`tszv4H-(Y!>zI@ZY z1Yo8}SM^^;b-K~7gh!l8@|Kz_1Pv#D}Iq)KDCrR;`$sW<62P!?eDLV+#K-hNeSiYB!arJruZld6#G$$D!E=ZN88eAS= z3lwy=uD$27Xw2_xMA+#1`?HMYd=y-2?267QZXt;&Je1T+T88_+;hFC}w7x zzXbAwExbjEJw(W77oFWUHZ&UPeskP1ufKJuHfRgx6%F32LM4cjU1!jD@RyQ7v<~Rf zrV-7Q_Nh4E{^0P!$Y}gr(D?O(IMj*y_tfr{ueXy+b$DNpAmp=%GP*PGUe8vnDa@&| zIfRuZaM{8*Gp&T-?Vq{ZU><$(56h9G-tf05M!D3)#`tncJ=~o&fOFXX^sJ^@3U0UX zp-jpkdFP6~lV%FP_ug7*!>u~QJ;OnZlg!>DrNdgl5$PTAVyotz&90%#v2&_Ywq=FA z-2+%(RZz(GY8oMZOV-fFrk^~Y*=%AxE^L)5CW^vA8jg(e!Qt%P~vS9R;T!B5d4A7^f*6y1OgXWGx_EIdSG zW_y|>!)Q`NvaM|9^hL0V$v7cpgA74s!==`}1*zhK#lKd%&^;CO`SK4Vk*-Ugqo9vA z5?;&+IHCJ23YI!`sgW&d#X{onu*3MQhVZwkZI|$M9M3+acx=w@0~u*Ag{mje zQ6@RoJ%kt@pd75QM4CLra-qle*5CNrl%#1fYfT^J)*m1A)ac))6P;JQW(7hOhxwZs ze3{)4mQMoZZ|yDv&a_MWTE=wk@xh{xKAUo|GR@H&kslPS_e(Xm8SVN8O6w{&2<5Q)D!~H$#1&@yehz^`< zF_twz!jqUo!UVr&Sr5{28umSM_be!i%00NFpHHhiu9aV=IeQ`mHe8q#04X@zg3^vR zV(Ry}5IWqCHD3GP+IBM}E?P;9p4pttL015+aA0kmXF=_5QDm|Kje6?IBQ|)daCqN8 zTME(Y5K~zcNS>x~gx@u?j@<`nEIfQzGc6f-7V62CWuIXk`q$0YcfA1pZ@n9Doxe_l z+3L0c$d>K#7K^o22l0qK!Hv2MKNb;ydwO}ic~cB&>KV-Jx(25)+Hcd+wNqnv2AXGo zK5=a@BjhoHLOg5NGMSG{-;*DpZ+j(oSzlHFdSm4RVThND`X_IZP7}(qq|gV+`r!OM zfPd{s^q7r1ZlfEsKGy^lBI9M8nF%xVUo4UlPej3CU8M1f-)PS)RI@2T(lPKkmd`@tzYId{KnyR{!Ug@-X^HmJZ? zXNE+7l<0+@&r7^yuv3=VbpIE6^&N;K6x!})yf*M6`>BB+Tt+A6Me~S#B`Y$`VA|ip z&ko51bk4Sb9_;^okJNe+pq{E}*XLO|RAR0=Dfqt5AkR_ghiT)_^4kN_DMF+nSDwf# zO)GMCgIXi4fZTx28{X&+Epr^~E4aTzq7o&YVW$}fYnw(hiRg}!IEB1X#qMaf52|hV z5Li#&e#ZoK#g=_CQ0m=lOSgf)m^zZiJkj^L^;HS+*<3h|Ti&jK6#8|z*!-7OxPr9z z-nY*Z$^g@B0n_9^iw(R6)YUQ}3Rvj=2h;pk=%6w>w6|d%q`sB6_V$MCLHJ?${m)VHJu7HK$Lg1W4!vNzP?K$Y;p}JWc;0NbeJm;?s_D8ZntazA_JxwFPcsB2A zu6LV|;174-gyVr8ATeZ+<-cpdm7}!qTXF9*Ucl|BjAznS&x}SH=eG{Vo+EU{RRgUN zA23boW|3#@2iaB#Ez~J65}d+ao968BVmxY5GVGceuMbZ`m-Pn8!C^7b9(PdB5A?TG z>eaY=(C!cD*~=JhbWgG&GM+q2Nh5XE9l(A=HY+(V>q|* zdMFmnZ9Pr_QxPow=>qHQ^ zNME^1LO<_+wFDoLh3GJwX)zVN>lK0Sm*0C>;S2O!9&{2qQcpVWy}q67-J9spNGTZ! zMf3w@ZR6MaNF-9AgOr4sw{Sq$S|)1^;GUlJDZ!$NOy%GM_#Za^NaKh}Nr)9 z1X|O7Kga_s@q;t#fT5{_XsfhB>dSm=-lHp)Io~~j;><`b&8V8G+x%2=-9nCIn$OWR zE1{LHi++>PsOS<4fljFp*%}2K=u@+5oAd*;{)F`RC@n*Isx$%mX~1nQ(=XmT;~##J zf_sP1F3kGTwZFDP9qQ`^xa^8a zpV44-9J=7@qY_OHZ9=R4PhMJAG7)iqJK{~rRK{`RweO#fsV)F@K1VA5mo$2@B?r}s6QVv{v9(?yT@3&0rY=%0j+uy%Ro(3#b6Y@I0SH{ zIZp&Dmz)?x2$?-l7hdeI-CU_&HC)MR-#4S3e`mr{@=Hq%ZMM5V&~l2J(k#06?oYKw`?oh~G!)HJE4e@GLs#MZMQE4HF!bVK9Jk$Fw6f z3mZbX^XCB!+X<>8QAq^-{yaK3C!g=}(L4r&;O-P*k1=qKo6Lec%aDI@miF%ljAbFG z9H@))XMs*nBX#$X0LR^9w&-ptS#m9`i(DqaYDg<^0uGTYCHu<(4DDNMKv}I<6@R>2 ze>X(_qJL!K5YQ1~{@$bhjsd1$OBU$Vu~vyvhsn+;e>n?0doBV399hICF|`P4-D7u* zb-o*qb31g(%8*}5LY~^hzj*{sHal?`dCCred>Cemo9?9x(7N6m381`mWCub;9Bj1T z9VZmMvy1`ve~?P(9htdqJS~^#X5#Ds`?jg@G`U$^YA29zRwE)8IC@cBx5WI9Y8Hfz zhM3AT35fCHb{yy@3mLXG(&*bLk`K8T z6dA^9HWxb>NXBjCvp;h@zBbh~4mdCt9uvY7W#-+95u@hB-9U<=R;tF88{T`I=hd*I zzdjW76GLaywO^==pvZm^2NX@d0!m_EY@hgLTW`=7w%veV2kWwL1Lc2N*%hGsPGEe3 zl4%sDReu2Ya+_+zaRjR4$_a%RipT^%xcov5okE}q)T>J{HyqubCW@|YH5DUu0-v78 zc;mBmqXO`=Rl<|r{DU3Zz@v0VfKxu$^8$aLgZrPxF*%h6t_U6?tJSy9JL18%m_|?_ zesc0DT$a$~*IDN;VKc-x2+%w8SK|m5waGxRinWWtW17L~3&tC7qK_x2Ijw z#tneH9Ii4Db*oFw8_9z&wLm*e9hcR7cdFfM-g4lfVRKIk?*QwEK~mMlBXNSpPYpjs zHgdqV1$mfWE119lOsL?=eD$2aH)%^ab3zwx*>E?}ZNl;7XijQ8l2Zfl)k2R`E)pJE zvYzkQ`Om@`n2JNZmI8Wbc1>23iDX=w!EtQ&pb z*XKFs`|*vz@X5`-W34OZHRoIffYUeS@_i_GcJ)&bwDlNJ7Bk}_3vC$C^Y9*8*l69& zW$#XEWoe}!{#u=K0|;ZH*SKR;)LQ_}2jk{MbLPo|-yBZfOAnytq2Q(l|Mn1zxvJF9OWG zid2444IaXkw+$Zvw7WQF?CX6Du#XegLwr_)wR)G0mFwAHIMQhkc=k68TLJfc2$BBK zDT|ofCT{p)EEfvg5JGQXGPU`KvbDHJPYk5JS04vY|`xd18X z&3Xaa#r zv#UQ;7Ld_i%8|$whKp?El@2&%M_dLH7(ERqU^{ZG0eY#9s0a6qaB(M`tk~RqYGM9mP7EThg6P>w z7GyqD$Sshs__N&KmkeDua1!F|7hGb>-P~FYmC1WDCw7?I^LY9%pgC(-gk0`|N)&Pe6K@=4Q00KN9fivQ(0GKtGbJ3Z8-X{+CywBAYdz=wkax} z0gdH%NW4UFfzk4#gLszEGGzfzqp(}H)CR!nPEc{Zv$w-!d>V%7-F!B?gRh}`2-I{x zf6xn`e#6aIu?0kNS1PFptLK{xfy(Q5RLpgs*h-UnI&w8FGcPv9x>}a{cwY!#~irI@m}htRwQxoQEy(*Gn7+vm`HyNv&MAv-)yBMezL? z#D?{7UT#xV%Oubp%>X))HyNe%WidCp2NBqzMWxN_%@>EI4eCmK?XzG^YMuPO>7vQ$ za8x_Lw}1IHW7)w@8nHudX8Sr()Vi|oPxX$Hz?Snpc`^w@pqF4lg!ik!WW+P@K`H>`Z-eg+ zG#FmHv0FrRN-1j%W5YV~sRC|F`di>tIRT8f_lfG zR5;!MI9_3q;2G~L#w4Cu=i#a_2A9?RmhrkOR>K5gBjehqnU{+Sf6PqzZH%_)gDAWr>Dr#h~FVmRuC23q|4!l}B5|>q>Rk^@eJArHH8s zC^4#JklleY>!}Oj7aSEtwgv!c~&o(Qg zromY1d>U#Z3S+Fcs>c!SC7#FIgUJ|yynrz?a3D%7qL96QkrVI$l z3(ZSfRn5`NVA7-?^EcrUF`*@cQ-II%^*DPQX*nlvo%x2^R1`Of@;C`kpetNTogjb< zg zJA^qYay$N7+*IaGY)3mCP{P|89BDuBUl8}K<5(h^RWBs#h-M`89K_6VIE;kgubhcK z$Ch8Nmd2E)fF!ZzI_si+3<{~A?P!mIL}a3UW~5Mk7Npnpi<3^_sqG;n?x2{XOdCd9 zV!JYb#hi&;saSd7WZkdDI0o>uFb)Bv#WC&@rI6aBf-C6;z#-%BvOYmqNC*+oBuFC4YcPMyL zE+iA}kjDCXXP;YB@L^XK(LU%Z{6F2$nA`_tkwj!>v31Km0@zC zuy3bq9(?L&{Hn&z(x`#LT$I{#vL#sr(SEh>)_mJ5E1eu)p}|&x2U9| zn-Me%FFW`{Q4eO`Cnq0Hm_d{44gOm6c(qX3$|W;&X2acF_4ssnwpBb!2Kjf0!#^c? zzQWt$G=DaCoYG3-BIyQND2X1;(JFwDO7GY&0;sM=)(y;SB!tFWg?c5fzbv`0Zi zQ&=nXaef|Jrh=xJo)09fEir7Ef)5rk20Y+)`DHnKwpG!?3P>bj%VTK90Mwc@AnW1xbw4C=I0jf3|JHn;sQD9XL3r8cO|bsaEG386 zyB&+G$HIWWEBub{T4wMJ02#JHi@g5yO~XPE@p(TqPrwKI?UVO~XSF*%#|=k3U{b;ff{{@Th+g@!^+fHuf; zmGgxiG`OgEoWPpScUdOT-<*fr*d4453LU8P1`+i?umjyrWl?FV8yW)WatK+?i2SqA znWKEdn{8b~{#%Rv4*VipXax;I#KB@q(uZpw6CYKdS+6BBjn}H$pyjrX6*V{5)^l?w zXuvmwxI1FeToPI7`~ze0`84;v>&i}t1s|*M@>+MdKI50{qfz!RQ?mv*_ETu00pfRm zir76D!x}JGuZPlwec8vGCF=Q`qK_NVg-Hv4rEAS= zroN=QAt!y`DFCxzWZB5pbR>e1Q;!_Vd@L*0QaUt;V4(llK$^1+t&l-IFQ+&^tl-B| zr}REoNqqHPEg9cN(w)eGv7Q6?y0xw8XuRNiAoTGT%210Ru4z7GIR>F`;<8}+1ERG6 z=*@oqs3!$7_JT8$Ep5Z=+d$s14FbVapyt^bX3WEjetn;B$O0}>`F!!NzZlBUWjMoQ zWxMcyt&(fVMk4bLV{J6V!Dc|hX)pBfy&SQxzSb3+9lHsr2m6QGvn zf5~v&-%d0xbt=*6HP2_;Kr}lLJqF(~k*gg>y^KX9b(oJll(z}TXOYwY1z5yRrXjej z8i5tn9OfMpkHg2=qhA0WoHuuT-v+2$j+U4vZ^Gx$8XCiIsjZjrjz7`V(i7fnRqli( zixu!aUkRvupw!b|k2Zz|QDMmh<+Zm9fFSs&idS7jM3uPJ;IHK&C& zY971IZL4li-~&XS^`P?Pt!LJhEi7sZ#if(_822(CaP<3UJyuhHZL61oI%V@bfzLDq0?_xj}$*O zEMYWE%jaDNqV{&yU2?435kW5pynE!0e`ya@8>`Ef>`*o2RiP%@`2II?+7Qit~d%W3Bs!BOLk0+sPatvP!}> z>v!0MgyIY{#F%WerCvXS`%HnDuNNSOh%~zE!!R<$AsulKn^i7ZOJ^S_NoapQ2u;`HYV$PW?NF1H(&aaf;-T%4ncPQ_0~2>3wxcwo5W6^MBZG=dXtI!HHiyw2spDQ zw4NzeSS?_;2aRh9h_)9~n!OfFO4dt2R@QTQLQ=gaY-%;pZNn@^?uf*gvqx)w{oipx zk6_?B#fBOvQF*fX;V!HR2`VBKvB(8w4MLhyh1N!1LQC@TyDK9DnR-PbYOvgN$k_Im z^wAIVv{5WYMT2#GeY*u{v_hZ-n+&u>&&G>3@Z73KB$1iREhz<>t4X4p<6a2?Aw{oK zCkcUm#kqZd+M`*Zl7HI|%hPaip0=;41GH0~#0IZD;I82cu^cSpj3g>4llddU|ajS$|-DS=wIR&u{$qn>+xh~u0- zEr<|v_Fy!lM3De5ig5rJQX`xX0>Ka&R7E-4zAu*=3buE-To+O{O{w1u=Y7qa`$0ho zj*s4%g*IO8l*tvTHE|=eINLFXpeb2X*ix)dIun<%Yd(zqF^V{3`f3G=xZ+ORLo|95wXfM^f$G2(BUHc4#2-`hpr#TJ zNy9*UcI|t9O43WpU-Z<&_cSY?^ge^H4e99Bx!z({cDY_aHkx11(*nl!bTKm4=?`f3@&&1|ru3f?vG?;?SSEE#(G#u}gli$5=3N2A0TM>i8n< zZzoIaJiLSvDdX#SWnX18Eru&N(iY>De=yZx!JCjqb|LDR(`yHV^ny2%rElAQJ9|&ecVS)eUNIGDXNj@ z{ou+VJ!21qcgCGGr;dNccMIl>j~&HgHegiLS0|AApqW~!xC6MM4%t3*jk=+-sPZtE z?1mB=(}-Q2%g(B0$A$FA0KM6YjBb4Wr@6-g+IcZwYz3``RNuE2dek%IU!QLJg215A zP>OgDO?gJ*z`ZHXI?Bd z2cX5CXtsG;ry~{1ms!R9MpzQB`Nzrb{@W7-xB~;p+EYcvo+9)SL*p8US+p6ZWUA;E z5kp2U6?Gk~o9L%Ig&yH$HgeB3_;T|;hGM9Ww~_xSUMAViF*9DU#QKtBnOL$ zl=*uSXQ2G4j@4|LMMZg;?1lweOzSn9j^fI;S;_QLU{}ZKkc@cXc(qe(^gutI=O|1N z_0Lj5MZq6x|9eyTIHxRHe16>%W*jofN;HAo5}vFHAk#Q}e7WMJStYM2bGSKKDdTC$3)6TjqW@7!zh}B=P!9|L=EHYK=?v%H zRcXCFG@Iov(=T4{FkLhkZ%ylw-Za7&vQEy~7S~^Gdj`cL|8-n2-fs5nzWcBJ4e&}J ziQ36iOkRm9VGsL}&A%q~R0l;5+ls}++FqBbfw?-O59Oy`-)C zznzW?Fd&D#+I|C?e|AP9s3@Q+-v-FruQwxx^L|K|{c|3EpH&TP8}LL4Wo1zOkth8= z%kactAy0pcV0l1$3-{doS;te<)`_?GF=*vYp7n|1G*jL(SJ0piBga#{IA(Fu<8FFX z$yr%h$vHglkv7>ks_EG8dOV`NnzC*2utc2 z0RoKBZhZi^=~r2){{Q=Qv_N<3q3YU$9_FQ~8b{-bJXPh@kxMhrAbsS|;~&y=d$2U;6v+X{gv;)-{6P;sxEyed z?bt+JTlm?0zSmP=3dZ0u!5R|coZ#E5WlZ`4Jeqa)X<3gNll6JI9R+!u@6VJx?sb^X zFF}7^^_=bXJKc&0f8L15DXL5~qww=o)xVxkMi?r&*sq<-i&BEDXWZ9S!=eZ~6J^>D z2wF}8?T0i_BrSSd9|MoKn56Aq`>H<-W3!s532x`>sVeH}B8Fx&DpPd*Nw>*zN{3B8Qh zROF&?r*r43zm}Z{25uLP1cev{{qn~51Bf0L`+y_t#>%NoB7A}EO;Q38GE`7_ec^ZA zEV!?BQ+)Ud)d0dxpN;fuFD0&j2Tn-1YKV$u%FHmvg=_?oCY_Xh<#27pFuB~1@AYn$ zH5%zWgws{OS822mGRyGz<7%D1o~j7SGhC67DL^EeEi-L<(7H__2DO}Tm>{~hN4RPV zhBCd-{qSq;owrrt%IQLV5={D~g|HBjL##wWj|vD!$U-Zv8j_2QQhpKU9Q|AJ2AvqMzzI|4w&!knWz`((*hh`}VuvPrn|uOPq0b+_?DfwFZMd-Vb3M zA!ek1;KW0k*WZ};jUzrpo=Gx;5vBlmJHWPs8*U3Mw+vSMQnM|3p%5cdjUyscvFiy_ z${+QRwBVfXICvS};(Ez>zG0=^Z8h9bEtpG5nS>2qvi|{i*VauR}{dex06m3+LM8b1s0vM41UH^y)fuoucPeU&0cIz ze0VoZjZ^xPUX<~UTo1GuqA8Jaa@3s5M>YP6B>q29V6m2B(e^CW(sA+0_Hv|fc7pg( zX;Ssc5bGwG20oJ5A$CQnW|3BjMNfiJqF71{6hTF*am&~}1HNS|mEk4p*%CVq%fh?^6}{Y?JN zbDQMUj#J9K@xX23GG$9%*P+D}jrN41rQcx;9wpMVUQ}Vj{EyJn{%3)0gsS=i+R-fV zn|L4?A0{ln6e*FI1O~_1N>A{IFr!$Z3$ef^S0E$fT#kBxlLQ{J2B*K7_#sX8qyz)v zH7Vz?eU*%1jBHAk($!i^_1nIv_6Zyn`uj15Tr<04q#7ddBt$4gIMXf0-getMhCLAM z1R?K~fWnn#e*P91Glj@S0U_s;+B+otwZI~s%TznePE|Y3&!MX?mxlJyL%%hbPDNq6z^?jX zHfug!(A$v|R2;=hr=b_n-4$pecT;E!f`g59Pj-;@ zOeN3WR^gOi3xvdn$Uyb#Ht%y*d>l>bY}^@1VhFNsIFo|()!MoNw3CPN75pnSf#lrJ z5273jh3w^Y8@qphN5NC%y=eF@dd~XasT@z5T;IJ}?lZ2YK@mQ>auvlFwp@0f_Ms6c zN);mHV!i6U8`XA`Lkkq6C58eXGVwQy<-o?Xx`~Wm61Dpx_aHKi*JArq-49K@-$A#) z32YXdbL1iqXzTomR6NL&t^1lbX$5Ab$_REJv;}QuRT4Vnc5HIrkuUE!y%U8{7zA@+ zljFtuHH&B?s?g^KB2cihzX;fLDW9gv8K#yQ1a2{L8^+T?zcgM^!mD2a%21=_I6Sd;im3=Xnn^UWPkvm!&e*{>3uJb#ueFa2CJuiQLHE1Ixy%Cw7v+sG~y1&S@A?xzs_pLmE($(a^ z#;5d# z<&L6?*52qz>?i^B@TBcZ5_gx!_?VbP@Zbf@&u-y2FILl>1? zf}$)@8cz(*NYxtFt4)8RlJbg>jM>20>fMwviUfJ=2Y@N80xS-9yvft82Gky&PLLDe zI{|B1=;q)7*GT0RQ=i+7tkH;RC#UbFXrhv8szd)tBI6CpH;PdiKYoL&fBe0bGXznX zmE&6X?{*OX2#s2P8&jiUdR|on;3BX&?uv+B~-7a2?26}N=WldE7rMp zWkG;I9bKb=z3ChRqsC6;d!Ba&#+{9@KQvZ&7Ys+=`0P6^oXqq6S4PN^fOFJ1cT`kGlW_;-@kAciVy*LDr#zt*f1 z@?-#g((8M<=hCk)kpn2s=@sYHUR%O(hGb0uvCLtr%6g6pf(4E;&l2=(7{;@uqnwdd zi@>9_c`33+qq$z@ox2xF^q@>!u}q5!x<_(*sD$1MD{A`Rl2yq>d4{TrHmLnGJb_y7 z@cXc|>A75#9?cj#(w;OR7;8g|RLB=kktR6qs)B*1f?r;JsI(+EbbmjRQX{=FTkfSs z;8mS5imkKR0Bnx50wpu7Q3oir3LtGQ;{g`0>*mDEDC;#pv-LN{3jCL7)lhPML|rH- zcI)$>$M0t#<^G-eU1L^ELEICPEWCcpiD`iV!&@F%@j>Nsh4#oSdR0$Z^byX&7bn^M zWFQ#4Wi_;E7_>$mPZ&Z*kJ;B6LowoC3IbJv^0 zIRmN$xTyR)gprcU_xN4*vFN4Vf7VprIPa=ft}VWYoj>qAU{!wKi(}>)i8+ZuU1Cs{ zrol{wy%RRmwA+Jbc(`nc$g6NzJVBUh zh(U=R z_E)K_bRXHHnVYvZN-$Dog8~%Xrm+8`;~CDq?sChl*)v_K+k@T>UgMtyvszb+d#8a7 zgOnu>RKtPB?Rft3VjS8$4#0rd4Vu5|KY3>X&)$PuFv6;~iX$&I4AyAvs58%2=J>|b zO_S>fyz|pXI31M>RbA3F@0;hTcuR??L#tPY%^%<8NAcaS_1NR{DD-FL4yp97>D7Wk zvBJfH#Q%^+iJRl`T{bA%@xcPp9&UDczZ1neBBd!!xxd?8#^B?<8CTB={W2Y~uV4b| z3Q*;@`X0;ihL;yWjSdKAJJz0nE)tUpYLQ+Elk_+9b(hIpo!%Y`HL4g>M-ChHTf+=` zV-c`#;!snyeI}4^vB_W1rw$U|oSM7RYobTPj=tqe0j;oo0zLEgf;!yQx}9wugJAUy ziE>UNY$5jdZ&!W?q;7c;QZ>n4#K435HY9YDVdaC(?p8TX*JnO8*u7`8_ZA;V46W<; z{hT@Hp5`9PXM!uUbM zvyyh=FlB?piVbSV!e>SpsYNlUI$^T>m!TsDP;HuCW}I)`k=CrQ3;ozCgk>Fql9 z@?9|rxY>|Ba1%Lf5cA^!g+X`hBJQ=p3qo`pyLI2#_9w?3-K+L?jKmgY*4S7zDej?gv`RH|o z@@l=3&+=RNLeKm+MXCrgQgt1J6iP1Yy8PYujAK~Yi_2JI$xqnv&pYhtWm%GTBU&nKo~~ zNQa-uuQ7FH6k$_IQ3vf(9TW5`V0?i`+7{|1BtR6-83@~hB6$sPl&pu~SC+~z55U{@ zNQ+CTlCwnvo%%1GxAJiJ@(hOcoCihW*UVy3l}K*YFnKKhh-k%9URBb;VSM^k*`r9# zN9^uZ%}l9+51wj;No3SY_3Vg~88z2k|0!}4j|Z(v>UvKoHR%mTrNT$ip!>3VfqZ+N zV=%N5dk$l1jJ_|2; ze`n0cbbcQk&ElGbDRjyr88)yA6lS`NVh|dXNF~HOpI$uCS{zkP?N)>whVNk807_p8 zXkbc!uBF6wykxl~sE-a=7j@usde4ed%3Iq?M5u@^Y=-ZBLxZ^vFR#fmVsN##m!Bpr zS;enq&hy7g>N6S*_@&gvhzWvyDDOc7=D4 zBTu3}P2U9F#W$qDwNH<@y=p4l7%{d#_m%4*9(z97BN;G>w5bb&TbiSOrH~{o)GF3H zjrO*mMz}D&FB3BlAN@h8f`e>M*L#?H9-`)+$n0z5M)ose*YW{&0SdAgFLt*5=zahH zcmZH#JtI7`t11Ybzv0*tbkwhzfcV{$a0!Fb;v;w2n+Aj&mHX>}JW5a?JD2@viQ-r` zr#c%J7iEE>hJP!J>)TyP>%B0>2y5=mUe4GJn|{TjO2G*5k!mFq$c!Bj!si6wfX`u+ z2!N|%ZXOOH5XPR^+ugAe>sPQKsT&NK`%Vzf25 zjlZ{Ur1j{_8Fd8ik6_0uerP>>r#%9=l^c2pJAj7uA4n8#Y8ALzQ~F7xrKu&TcQ@ul zh|(*3w(Tyk(vw5R2vv0#v#;E*?IP~(YdZaKXFK%PSskZR0=b0(*VQVrp#|5*$STEA zuYJ{xVZm?_Q6vI9;u$y;SH%Gp(tcR0IbPH=Pe-)t5AVb%krCc<7;HST|ALBbxNjW~Yv<4SDDD0PJjtGK1Zeu$Sh+Hn!|fq<&oP zr6Oy!8_-HELNhoB@;J!y7|4bAIqzd9S!qyfpb$ifMQ4&~?Ti=-4WjeoH(H9T0x*+G z5ejV3GV{cW>ZM?-#=`<>O7CK$UqjVBX}PAMIC)}3Nu@l<>Uj!IAYnEnbC?H#mAZdr z`>kc*)Gkw7ia080ru1VETxL=tELX`fGwl4hWfz;rKxLBV{G9HK8$jBQi_-<19E|XG zgIaO_G~p^|DEOH6g}+?KY$=4p#FuWBU$qV~u&;d9M%P?RGdL5J^gLu~u!o_VM2`Y1 z3`W~M{VSSBE~PY{Y4%{Nj2ruN>EXEXykZj9XHVI~_0giq^&H%A5p^694R%usAJTxEx1$(2 zQJ9#uOEPLLw)2SU6UYOso<0UjVu3A$=6g38!z_*EH%|x#WaR#9f#)CY= zwkDWuo^VH(e*h1`M$7^s;RRC2nX6%;sJ}5-_ny(e_ylC|kf0$<*p-p5Iu z;82#wi9_R)mOW-E*Ga%cw)N#Hx7!XlZN8V6`JuhrecBjJ$;u-;j*QVSOBdQ0SV+p` zE+7Q5W==+ZeVTgJ3z3IjYsF2if~h`nm%nP*Gm=n0i60E&H}rpIf+v;Q9gd_PI|=P2 zz|(9Kj*=EB4govc1G5lPI)RzwWWOVDy|R|hJ_;8ZL}@h9bgNQ~ntrb?r3=%d)oOZR zT`qm6hc>aS5|!2|lR7(NLd?q;g$UPrf;(Boy#1>v-q^=6Iz`KfwD_ue-ErMBdp!jU z1uvq9Wv__Ujz1FZB{HoD@=N{3taxBKO3@W}@L9>I*scO2IAEIiv*DNW-~R(+pa}gn zozzW3SyDzRiQ#A1B5@V|WGf8qA26H4_oP_IH3vNQXm-Enr`I2CO*eoY{d3p*%klSN zy7{*1bg(S_QLx|&@59K52#O{g7BvMUNTwBP^#ofcIKw@dS-*UGOR#hI+}^A-kJ{-= z>)MTKnG}G^$qPO-rCI&e-Ys8e?mIc?7{3;GO}6Ma@Uvy|u^COIbaCL4%Jcmz%|!gdpp!fXYg}% zGbSEJ25h*%a-|RQ^N_y#{QVVD>c2vb<+>ews8Ui89kjv}>bLA!jpd_2qo6r-SI*dQ zHwbbXzQ4qOcDS+H@tSyb6-Y8th3?su!!-E(4bn8ZSY*r0ysN{toEAcPWWD~gi!<({ zsL{v_qEg{P#c~Hu29-kQPa=8MPEgu{Q*IGh-Xo$$Y0Hf`{MsBSSh**O8DaF24uv4} zo*cHv6`D(A=pe(?pg99UCO9DEJ5V<~5>Bu9K-IoD4CVrwfF$tV>UP#$e1gJV1R6{8 zqnWV@&^ot1+kMvBYLAmDCYJPj2x!HMT!@n+))j$rRh|5v(TNn}cHw6!8_`z#EZdfP zXV%0mCuGGVnve9}<7+bdYa4ryY8L~x_BL=PJLJp3%E!vZdJnZJ-ziKeqZGsK>rnM4 zBOUEdC%Z?%YJnmEHkk4zEDM?Dru4_qXv=wPV1oNzr^q;yJ~Ut&)j%0ix)v2<+7@JC zVxj`ZN$a2bKV$_mQhYd0*eS(eZPO2%i3|%+p&=-{2@iF{qc~$1J5qf2puL|;>38%1 zk3*?W=zpUziHspwj#f0h3O(~DR9F)rZNQ33^okJRDy;#gJ_4PGv-TbnqqG3@&ks>I zDB|;y>1w`!%5a-jA#uQ&PS*6Zm+vX*zBkWO1FLGcmDN-n3XC9>U_{Thc)9uFr!B9~ zxjO(!G1(%z1>g*yi|qc~DRjU7df+^D(J(Shos*?%eAcWfXkOBYGEzR>c<$l~{<{J) zeiqvo7rN2{1*v5{{-0~vvmioj^UQngcV%Z8tSYs6>?WYiZ!ELK2Nlho+n)nfls64x z%7%#`XxsK8sqiM0lK=aoatf^{&5LCfjNeET6bUlik!2}MRY1d0 z2WYj}<&=rpFJGux6@h=#hH~@vy@6e&DRC&+bW4ykNTfGrb)yLPSOY%bQUnXy^jrVz zZ|JgfY5~RHe8p*zbE$WMXGZM}bo^RKsSlkCLC^#siiABpM)|*xdQcT7h`>=88p6;@ zEdb5{)=+{U>NF1v;y#$LfO~&U6OEVU2&`MX?~dw}y%rQ=l>r@j@f(`;U7){}`!QQ% z$@yv!IA!L|sA2dIA2ovhPO_^RpkY^a6(p+(SBISaZD;FU=9aCb|9XJ{cw|Cq;?6(p zULD+47CQY@+_=|W0(u4?M=4~SDPmncX|kgWu1P#=9xdB@t^IAizSmw-Y#SI>t63K z`}(!4@EPuLgQY-P+;`B?E71HCQ=$9SQ(zU}0FF(YD+W@XI`r|ekPpzwIFRFW&BvR> z?myV3GTwoaDuoDBa~3=Uyjw+_uZBTNt_=hf5~I&7J-M6TPE?n(WuP!nsirZ^sU@N! z3^0Yx(O22|!Op{mu+Z$|O|daC}kG|@z&azB>#FG(R= zNTZ0wkawJx_24WhzMWUbK>3EM<}vgFQt!QyCjEZg(Y6WxirpYMsX65E<4&g>Jb4{Z zJ5-WU#QHJybT&vfM}HcPteg;{a=@GB zySV|_-vdD1+EooilsEN@X#XCvf|dZ4zC`5vJ1y_f9<~-QQF{@ED?1dxG+qU|HiLn3 z>p4QC?%NAHiRO_`fFe?%3C+tvXbBOyiF9*(6P&y7rt=mU{r40eMjqODvAyFkm+GiC z&jeN}hhx@~9 z1#)guHH_>5QYjNa^gT=y0IVjsKJcWU1?Nw#&Uvho9JivgO|Z_r3L`Tk+9IZ4aL)rt zh_I7A2FgvY+6GRN9WPikXpx?&c3p)90~9e#%s$->BUOrVaG|Hb?4p(SCs1ATg0Ve zAp_p~Ya$XNx-M8M1SJdY`0u zsRYwM(PyikR{>_&95K*SMLs*8_iZ6_;(Pk_!y@{~o8PgOTavujNEO4dOU{?=qb;Qa zL|gTWPe=~+3#_a1Qsb#U?A2gNMYN(~aUhGr0d+pI>*jg7BhdEzG}KraR~&`%`+t54 zn2`&ya~RP*4@W_B)Evs#w31{iwZEK3LDvQiA|TaIXV)i4v#lCh^u9uFPLrbJ3|!XE zjSPp``qOA)53(5)fs0K`;pbkDN5!v21rF#Wj9bV^>_gFzf*c@a-!;2EWf4h3)+9uy zJb>H2^;C0Fkdy#Aw(bmY;_qMA28Yyk3}j{h?=7ot+xHPwM7aT zVOTg1K-#okUM_JNeleX-Z90M=5kLns?kx^E`kv{#4N4Ml)8B?n(6+(hE3bBPLet+K zg0ly+Yyq}uoh z9p{^QlAp^--$L$(&eg!m_)^kzr04btTiTr?%;k9Ic@(Ch<@A!eU{VP2G|+^p zZu$bkdEG)=a3|(r&v2gN^KWSxuh>|n%CD2i0%Nn(Y3uyd8C}cN&z;;=N_yx>CEGqxR!8m_F zb_7)wVW=u}j7*b?N$RpiftY4$R^o9CB*r}4nV)}iYahsX_qI}juod#}1|#o#fvHX% zgbu)6%|~dOY>ve}Xo#?>9bdoJ_}992BogT@Quhq#{djk}xHB6mO>P3|ybFMrJhSe# zbJz*5WxSdJNi9tWrQh`v{cY7{P+KU>M8yZdNP%g01K45bp7-vHIk9c{b2s~6atfMY zWwAe8mJ4U<8t zOWK>yV9qFQmXEoJ)vBY@4{l+)Gg5kl5I8b)@O-**4}s26b9+GfiH6W@4ESSxZS(n5 z^EQlPd8ke*5>Acp>p|x)5XbXA6qpAf0R)=*+u zBW#^Dz7Z1Zd*uIYzX*<6JN|Gr^%LaK7M~nq> z-iDz{Qcibxb^sR#wVl~kgk>_qdYIM;ckvB|km;?)N*GC31Jd7Pua1{ z#l)>B1=)_ENjl`640%Bu_MB4~`~V{5#d+b1QI;t_3(gR=M!KXQm!Zd$m!%cXIOGOo zb+*31OX1ARA!J$tR|klL;^$dXjZv%hP}C&-MC%TaXfW&)@4~nNT@C6H9;2vH$pn6~ zLwQ!Kv_FX&ok(1MgNphF3EAmAr_o-Ry?B0DeA^3@sJmJ^A6=awqcpDm+h<^(*AZ@j z!a)cUi=UGDI%f=snxwKTyal#tR+qcyh!_=r=L2#%m+hHJsC0G!A?J|d#*fH~XyTMJ z(6+lp(DP6!+zK)Y1lI70m>6%+9J9-=e@0YZfKz>gxb=GHYfJVWoG0H}zy12SXX*pW9m45K zkAfPT(Qb6x5QQg=ouH#|5U>59Vn8Lqks0^h;4KaA*GC+;7djnpT^NHt?Mua;FV*RTW7; zy&318=&~)0{fT4qY2L9-g@w-m+4CdKEs2c2+v`F&w6;xUe#>*0cQQZ=PRhU zaB9j@%C)3Za(%)Qx!GTY=w(| z3KWZL%tE`=!aFlEBVkN=V<7iLkbp4JUqrH5u2SWbDG8&Xp2HTLj(lJrU!mWY{W@|`9uH=kp-n}8J8d$RQg`MymS zDDlux11RG>jPtQxCH|ZId{Y8-5-Y8P-z(@gA&z}mI7L7w zJ5>(p!g0)P|3WZ|JmdkSAbRszV5=5eo>El?bn-uUi4m`{qh+Nm&D74iT&~Pcg4y{A zGkcg>Wdcd0!AYBPP0>EGs~PXT2$fR#tBv|lrQ|IeBEcLEq`hX$CCi>aDH2clLt_@R zAQ=sVwuX~nxHsF!CHl`@;})5okk68IgBM`S3<`1vv~Y`mTGR`=ZaUHNd|GUNs##&l z=GEVyhuY4pVU}`HLO@wLh-V(Yj zy+bko1;9$yl^9B~{60Xbd#eVVAtv34W9AsIsTGN=)vEubt#Ge6I>p67kSGO=D@6dR7Sp6WY2slSAF261AANtI+<$GRo zP=3e?;T900UvTPO^g8N{d7-8I?%PuXaKfA&M;ord z+;<#S-0in+u><#DucX!JsV7};rUvvMfP%M|=hPoiKC;|d3)embygTs>*t)XGY}7!! zw>CwAEd)Oe4Y3QNr*UM+R!vs_V znz!~p)ftgI$X6QXqPeFhDjA_}&EFPrB2AtM`*hK~CZZjAt(zFjfcIjpt8N`$wC;g5 z=_EVME{r>kZSahcL=qfK`Lqgs6>|gM9`*#X*{B)S(!XJg-yc)D09gG;lzNH#6r>4Q zgDkhhr3J)oncK?jx@j``GB3rbc=1hNx`IIav^thN-d`LYYED$xk#4pOnA*QGqxbXY zaJC!%$ifsUQocj;1Ir5}KJTXc;kV;Bb}Sl`+6=->gB7AkAFXSWvV8}pM$~%B&F^*% zgBlf|n4XBV`e4gN3C2P$3H|A^iDrTv5OC8NNX_5(+4tS!0Nxc6<_m|os?sqFIxNhE z&a}OR?!Ka_nh4TIQ{k$hBg(=Y56*{F`(AhSvIMd6)M=5eY5g94^h*Y6+_AvxJWC4q zm_M4t?D7%KOZUXf`_0Tg`GJH$7+n#(AZ#!I5GTc~a_x+zem;1~`=#R&$c#tg)**nYV?Gc0J1ULdh5i5p|SG`8hz;6CJ6tCmx9CVQqN0XLJr7 z>jRO>sJO^Tj2A+-p~L8Gby~`p|8Pr2=u8$)*K$xX{E;~1wC?+NG`@vqxra*KAKp-- z?`sBKw`ldoPP2>tTxC5xgG!h3wFD$gD}n&}XA-(0W<2aCZLO$OUhf`3_Q-|-=3T0< zg1N~gddhISj-r%3N+AbUl2?z#!aY53??kT(r9vzbOt(=lBoued|3lVWKvl8z|HFca zhdy+7cM5nwy1NxY6p)k<1nH9QknS!)I#dv)OG=~#0qF+mdiUtP_wo0C*K)Z~U}nyq z9pCuGGNsO})-8!Qt&i3$R}lCP#=yq#xCISxiZ*Vs8N6*ZFX^Sz&_b3O|B(^ALUM_e z!(5pWa!roa@0g#|#f)wQm!{N;8+}B}{gJ*Oi+e zGiHuF7Re(DYU6%)sWx6I0n9ETivbEjH&X)I3=YMhVzZ-u{=pM9%0xG-rKLOlRlZ$t z`j{7_ln9ssK!Vloq@y)Q0Z}3z@rNmfxB-(-emcxxgjSR0sECyq%25^NYrJwgft$`o z#8A1iO}YseEtNi@mV1;6C&HN@e-PyQgf?mmkhX_U?=t6oTs$p%V2|qZPfj?G(#_s> zz$0)FhCXg3=7U_|${mV>3JMb1hAilTb;j8dOP+N0qaY*S7ODWT`mmj-bdONSkNA@h zcssP1BlD*pp-wDFs=iQno#VSP^~kn8i5Y@t!`aFAWr`>v^_v}Gis(isU;=3#p|WxT zRau*bW}yVLVfvjb^AsrbT+kg%l1yBsnlppuci{Dop#5|;bvb8(+7Do(bzl?*aP==C z$<_I@Zu`D#nikFe*VQA_IcK4uR|@m7;BSmY$_7<3w@y~jeLS@!-CGBRm=NzBd2#{m zVH7Dz>!Zy{!bkUY|KRPj5wQGL=`dpI2y9xsArs>hiWVs!Q6#xjDxZtJS$4MB&v6*I z;=QfUT{bi3nlTL0%Hs|QG(Ox- zH-7>Gy!6F65nw!hXO1+gi`WEzhj`xOmI(oJ=8}q;1bWxvDFtL2$^B8O?-rhfYwN+V z>kHxI8%v37@TY-WY9H_B^6nZ0T+^Z^%`bhvJ_qIgZ`sccjlXC*%VhV!nm}$$YIYD2 z?lIJUyC&#vI6wB7R0kgHR*7roc362W+n=mVV;CUab%&FRE)tX-Yof=Ijq^YjFP2X} ztIE;!g3tkzKG5 z?X+cB5LTfLwU94aFWP?9M7948FJ~ z(V$TKt4Ku~&)#=6N~-7`+1onrGQ#4QnCN9_;Jxc}Jkk_`5X7h-axee)nTB9=l#ILSG(Nob6=q z3yusT^w&+^6ut~X&s71*NQnMK1w`i`%3e$O)MIL>g;W;PUPexc3b$_JAT zs?Ku1Xr|tql?)d2muNk{k$D|}BQRJJ)a}6GMoro;rN)N(J()HvAQMq<^mQLs;XU>9 zmrcbCYDqf?fn}ADA3*=4j5uu1y#pimTD|qjzK7qlvS*Vdt%!O(7<@9INzgirh^hA{VIbZ`5Cq>mZBMj_L_l?%9v|H&VDQM80O0;$(wS`i8fjQBJ-I>{Jp zZWSXHSR$5WY|Rt6x)(7Fn{rihMx`c^nj@1_A>^K>9W3h!1U zMH|v54RTtfyveJSkx+1{C(z}g_}1cOg;Dc*Agf(*$7EB7+tp%@%83}lmBU5%0aZ>U z+GW=<`VLp0Q0Oty?4YaOBOs-SmnUWUMPTrWkeGup`p#XnZ(p1=(SLA+uhBE(zRrwx-4!H#7cEsOp^hOSqLD8G&1QLEuBIGdxR9oh3sf=^2OQ%gKFNaDPO~dH*enq6M^SXnsrb+5-a{89&9nBWvn+)- zkdkcDP`M4@iR>$>B}`w`e}H~VIGf}HACyP@v!^g!HCE12SUCkYPKenD6OtL^ z9BxeGAXaI1zQ2AT(W*2MdVdIX{~p-0ZwOQ8l2M5i)I#)3$aC}!K-FW5Upyy9_f%Y= z7P3o+Pwt^dgL>!r41g@Q>HbrzDW(OYU~n4ycp7V`;| zqRGC6T}X+aPYzNTX_>>wVc6yY1sGV|88$%{1jk((#Wc|iP*oSIddmjyogaeAXK12r zP_>(d2Z*F?6L4Z1pc=uogOvTp-ByfYP{@7saYt5sJ&d4^7H$Ob!pXCj{k&a`04CyU z%!Z&b4)WN5O0~8**q}L)ew;|BQ%YK!qOKq437iFCIH01f+ zWJ%Ff0rp^@@yT)hu8~yHn{DP(JdQ1I(0Vwa?15S?o&#WXfROgwO7ST$xmgp*SW_u{ zopGTF$`F1XD7!(lO$DTMT*lu(sw?mE`4;iSdIW3BJ>2Ii{U@t=(M|W&JJrNF(}7^h zEdOvl)HT?Sk;dL@F~H4#@3gjH2Q1ZFQVew+AQ7fZ5%S|HpkUS(d~BMLk#-3xYUbR# zG~V-O1a=-kDX^KOQ$lqUNapUJ2Oy`+#Vp}0;NGRe)rtbHAQ67I1IR>y9LkKE$kxw6 zH158d1l9bu4OkAdzTf7HK%Wi{T@YO|!?d2}W_{}2>tO^oZzc*V+O^A+!o3}D>Fei~8c-ddo~a%Ei!7UGU^3P`s7 zc)Be32W=joaU%@RKSsliA}*p`w)I@`0k_st3mGPu<7(1Ey+_C3X*dH{hp{#c%aO~b z!Smxbfz; zW*q}zwx_PoCdDQTuR*!zf~j>W*k5Yz;fFi==jweB*zE#}WFCX+6ASl(Tu6c}gC+5xxN{ppfi68XXj3jNJfvu zRn5V*roECPw;T6lLTbhB{>t|{G*N~oRDzeb1YX!}+jX7dM#zxxeRKotK(uGSNf|>V z$S>6~?iG;aRGcfD7B9PSWaK;L(#bW0{>4XalHt&FRsbewYU4*A#Bt2#c^-~!B1#GU zrZSA?XHY$`casSRt^^N*U~Q-vOT<^)w4#jqqBfMY`>>+W;zOWP>o&-875xS1ms!_@ zYB#wJ00LgSE!5GMTmAT)S!ycHw&n8ZswoH7RS)58!-ZZmsfLNZzgqFRV#Zsd@;)(U zZ&k*Yx{5Ap!!T!`)UEL9T7hZ$QVozE^!O1U<6SJ4c?-809fw##s`} zNQ(|Ebn*uL%A+zKU^&9r=oA$Fx`0d;8v=W6IevQ{J|FI$2J&qWY`;NF-i`d%YWuU+ z>HfRHGihH);6bj;f7;%~_n;kqC$&@-fQLU-cptJ0S&y<%C*7qeA>8i9fi>?2$_N1J z-cULU+*9>e9rldlc$a9Na0Mc~PxbK%zTXtsOa$-?8pmS~+Q-K2`qGNbTvgxzg#4rl z1E`)T@09zI$l8(5`k>XwT5!7+K*GV?TJu%x;QkM!ZsY{b#hb$#`Z#JLI1!?W1LD`7 z+%lDTysc871ErLb=X$W2lC&>e*j{x2yo~9Grz*H2t$h4~SGB$qxWsW20ZGKv{GtK7 zub>XnWhsbD<^HZ#dv#`ou6cn}R`roP*q8W!sTn!U~ku(Q*JivjdcTF;RjI zH2scM&siK$q<_#`sH>vyG`8?a@;0Y&*(!$_CFRc+prz?Gy|(KO(%2>og8B3A$5LFZ zh%qR5R)+1iyf|Ac;?~y`BF3Jn!g8gctNbDR0$BRU8c7!3nI4}Ov$3erlmmOvq3 zqrxpkCjQonMFA3))r+{cqe<+@tFOGdWC)5Fq$Ufub+2 z>do8Ozqs9+R^xmVam}k+VY>^e7V6q0IUZ&Hy}b>@UIz87v|k+c>lc-V=My%XrT_$v z>C-iNmLp7iEajPP_mR)j={vvqE+(+&;^E>u;jGNlyb+Zvulp2;T1K^$Ph(V^3`b%` zf$L%m*IHx#xp6_WHjriLf>1-j!*n*lx98r{3#s`W0BlS&uv>$m3%4Ze=`rzwdRu>W zG{fVBEi(XC{l+^VsbLuJ2Y}rSB{|{5Z-ByfBe)RnO&ms4*;EKoY5pDPM6M5r9;mng z(b4nkL7?GzX8WwXE|J(gPBbjnAKx6N+5(pTIbYkCpU!j^xVfTRywcsdnUkVHHZ6u< zfqjh3X~Glv^LNKTA$V`dOGXzxOH8gO*=wL1VkqyjUL=lMsdXb!5A%5T-{6WyJD`jqBDZ9&a!Q;DRN?-k2OQ{4&`e(C8M_8dQG zzGf=~0h1@mxs6-g{{}sWp@+RE%GUBdumU&3Gzu7JO0!vL<{5uripA|qJMw(6_4Vs> z0qSk(5ZRn0rejB$x=6vPZj3Eg$G}WYiW*QIkTKG%^?UA?j$@xU1Pc$*fMhaH_w^oo zIdeA!9%hTV4yE2VFofrc1~5CSbws9*GAU@vZ#Tqu(D)81M_N$Qp2A5_f1}MLP)20I zYkB{+;UPO1+j*4LY&7|HeP`U#xH4Kvw%IB>v!4jA>3W+DR{kt(HjxI^a_!zkTe6*| z>O++pO6}gL@?Fo=3Pa=^9$8N16c5I6muAP%Vsr4mzo})99#gfB;J%6)kOjap09T(i z2yxqI+w+#Rpl=;5)32EU$S^M!&ZY>sBWx!?2sv*PO{9a@iDUXbc!hOTN0pTGq9RPq z+`Bf>;^*Y40DQ4krVxtf3Lji(qY}(XwiFyquND3O~ zao8tL9_Je;R@4Nt$>4?8O?Yzvjg#*jmM?z zi60#mZm4BO7@jhY$jfpD`QEd4+inj`)52W`yQC*%Lnjk>$yd7tE38^E2PdZ<{p$NxY1R1Pt7DG`JKu0Vy`fC{Y7*87u>CVnsID zG3n8$!CRv})kx^NK}NlDuK=E~MH+SzLCk!NiQ;f1O(SW!>@$tR)%)ZU^n#ep`DN_e z1spp%upHV^AQDiQ5OHn4*DZB!sTJCfOV27h8y{0UcGKPsq?c)em?;gfO<`T~+l&z! z1tAAKCtewiU0#;}@Nhi6e|l5?gOKA6P9%j%V_8{cTTi)DMv^a{s z4}!*KZgTS(LGvXYyoLte(m%fRUS{xZ=`XvCQvU`M`js*bN%(uH2b{0^YdMNxsKFiGXQvb^V1QSRE4UB{0Nby_WGsS z+_s0(`WoMXt_SXQa?Lb13YzkZ$88wFZv6F{s5vJKbm3dlMhv3yuqqc&^q*MIEtdCG2Z%Qf_rot1 zwv5S!NL^5FMf1djz_!jqRXc1BraeQvN0eIwcObCO z1rq2oIXlG|8L=>7C!1=V14z9@9MyrDnqD*$O~a{p#JIQE4WZwI@BgI3z3;N!X(V59 z8(8WcY@oH6QMc~z8DD!wywEP<4uzLK+u?@Oz0f{3myiUW#8e}cXF8H zHf;x%547EBmK@G3MmS*MU0B;Yb1=v1P7b)MQjU1^sD-v7=8nNjl(5mQCAis zJ10yM9mj)p{ca0VgeC-;ny&W0K#MuG1r&=sH9Ip(?r|pz9=XN7$pq0LU1h)xw`l;{ zx|YdVZA=OXf;)j6_k7puVh+!p%Pydlev|3Cf!ww>wk0X>oaR+{??2SV zxZm=Q-}iUp+qL{n5rZPyd-*mpCfZR_Wuf-+_;+NiF=_tQ0(eCo1=$du(O0miw4b_<82T8Wm~H&% zcUBp>%VwM5Jj+yc*U4ASU%37zS^^IyiH*LGeBGcz`fX4YnSfe`O@^b; zb0WyHL)6W9AtO%laD_yy$v-wN@<0uU7-ogL$9gabWMU*^vHWnMG4H=@RrgUk#6;^G zg@~biNK)hTRd<2sN<@?TUi039^asNzC&qrd2wBo6*5!`Ws@xp z_w6Ee`Ty+3op>?9I;LprCrxNPCUGRgP^qh_&&`#Fo2Jp0{~?>5Q|UtrE4d*EC=jxf zYX<#Ru@&;9oSlWiW6TI^+~zn0Zh5m!%~s^vg8=pg{8MJ>`S9wVn+(qo$fAp!$4IB< z-MX4zB*#M3qHIIMi>bD7898J`W}8VnQ1#VW$NAy7`w&)MHM ztR__(P!CD~!Qr}h)dz0XxMpG+R9GQk50b?mK079yrEkny^Nu?giK&1m=!8|7S-;Bu z-lghfVo-nC>J`^J4mj!mQUIVYfHnq4D81H%!;}FE9vb}o3PN^@zXdh~BQm^U4Ea!G z3ZWIfl0QPms8B=84!{xBIrt3~ss2?samLGDtmEz6b0vfeR+MzGykX9WPv;jvu&fy- zCM13W#ddPS?nA&=vaK~N=YRCxDYR5=)kGhG_7g9U0JYY04sCnoLQfJdu;?AdToJfq z>gCyz@3rX)m2m%Q?fw1&9TKJEF`5#=Kw4vl&S`fYF6|{#5Vt_I{i1;-B}XzVG>tn&jtx_A@NkuGnrnmYXsz& z?N2vxoWK};FaUOz8@7XWCXk9=Z#*j~(>RcR00v#=qZCl(##SGb|oZ;W}jzcCOoR)cul*JQIeSCa!C@GA2Ogq}ReNO11ugm+_Z1F>9 zuKao5G4OsZ@3)!(4n)*bRcRW7T)U>ksJqL_uiA?NF=i&3=@A&`??7GnJAlp(HnD#P zU=sIpyk&Ja@^$0k-&#B@7)S_X3|@p?A7x@i%-OJ;qI^E-4v`b0OMYznrRo9wE@i3T_4$OrYU3qv zeUTX{Yx4O1BYy@A7+}9XgK;7sA)INYr7aapG z`|dcV#Kr)TqbiZTcKplLF(~8aO*Qyu=SsVg;bM*nSHvq;3CNYMOh3&jGYV8&%t)fh z8uin+d}lSA#nWUC=0qCCy>J_^oUcI9=Pqi2C$O(~YYBcTBG1~}d&JfL0mo>0+NLIw zI;QPOy5SDPe0{hs8%;IP%M%==+P~zwWW_Q3RRz(PHvENfRW9JKTS~d zeTI)7vNxk8+6G2SEF=oAfbHO6sotMz#hZ=jdk7pWEfD^wB z3}i}HoCAr8#91mprW9cU0j8mDiZDuX&@-> ztQR&q293Xf^N7%NCrKceB7@B=XxC#hY37g4DlRU^?u6 zI_?U?`T={38hQw1CxN(@HPAvTxz8fi4;p1_@|}-Cu}@!m)p={`4!VOBtrJy;h*6Zt zw(xk1&oycXOO*Y0_t0eofVN#P@)^KBxys{a+-U)>Vt5@ay@mKI+5Xs$dy5l9pa}dG za(e#RhQoVC8vPceH-j!A^Q;n4iK*9*A{t`{E zqb#^HT)r(}eS`BOSlcwzog}*CzX|EcO*P=Z-3jOZxJ$BIKl{WVD@>~OAW@Rp3EtJg zg8uW-&-WW|Ux;kgzJ`2nW`I|4O=eNN*`%Tr4(n8(2+CZRMuf@ZSr5S}OL~!h@f|l8 ziAFkRi~3`16x9)&c6jS!kz{;;UVWNEev7Ksk0nz4epc{fZA!0m8)rdg}yXsW@o9EM3ZJ{ z-+$zszRdl%H*YmN46_&pgNb!hQA%*;wOcLlgn8@(F#xnkK8ET z0XeJ@v=7^Y--J_Z^j5c4uY5^c>0kih&C&fb>`03zYA*yHrLuEf--wE8yDHpO9C8dkXxUJ4AbrSo5MgWlu3^lbIfIsZ% zx13@bc@!dwTD3Ub?P$%3`IjhnraS>g?E`3Ii{ov=tWLo2kFzfW3$8;t;7m(N^X#G&0&h20!Pm!Df?WlM8dH^x29az+n-ED1zMN( z?EETY2~d+YymBVbl^UZYIhSMG+-J=eg2svc1EuK8_;%Vz&XXJDI zAE-0OMtVO@!G^UirF6W=t<2PLbJ`BbFm;l^HHoIKaoYdUGg@UT&(7qa<1&#W6HL~( zlKw)?Nkv%t>J5-fWHMY@$pV~O_l;VqKVYmi7>$a4G4nv&Xy~MqMaA1V=^d7)BCvG6 zy&~TQ$Hth*GE{rPl*!>IMY4djUQLF;e{bIl=wBp53X^0xveWFXUe_(PyG8fF64S{I z6gu&5Z%=s7i{KeO`MDKAPWzgZnqV4m14!pOhF?xSlaoK=?V?zmlXsQ>Eb^=kOQ`ls zlxTVw2YkLe*$*T0AP3fsy&u;5NrxX}Mt(Ch(@A{A2W{~@`#L4{-KKTOZ~Waq1YLYw zz+yKOr#f|@WvBLo9G*Q$adXXqWNfq=@Z|+zN_-KPX zD0gch-nsOr>O(=c?QuDmVD1#u3f2$^Gi; zxJ8~FmUBO1-k5-Fwx=l5y)?WTr8`Yd#|;EXdXhXeHPpdb|ISG$l--CvF1N-RG(Vjdnz3f zme!QJAY=XPE`?%w7U~@H@yW4Rq4l)PxOsSk`P_LP`SRT<=*N_-QZuR|DSv zc!gmz+`ydqhJC6yB^Rw&``vHVWi45n&32hz+4O1HwkaPaz6j zyuyUTA_zyY%_uK*!>UUm^ z0;6zIT5==<3NFe78Q&QUh*&x($>V)-lK*JE{Mp>cW)Z;V~X>7wXejqXyOdn_~ zF0~u$o5tg2Y@y{jx(90L&w?4$+NcwOu31fA{WUs!*$)_hEg&I`vol{rVF85E%1GEw zz&PHlnghM)6G4o&>kj$nSJ)GQ<09 zZJtd$>z5kjW-WKl%fsSX5h6JHr3@eh=S>zo?ssCdKKOA7uj2CT%A3U0QIhsKgWUqry8Hck8A3y_{FJ_mIa>$!&fRE9(>8QUe`MwZ)>Q`vEUOiA`i*0qW1quSc8$(!G42@c`$# zdoOE|%yjD|Y)8amAl;JONKMc+Fu|ti>x^hyF6D-KtKYq{=plMLZ5p((k(7PseB^7y{1S`q(%a2xG1 zoX-MKSByp_%iaTPP)`yPgi-g0-#&&3G+AvR_@(0Pve;GNOB1k)`*Ox=l1!Q@og=Sn zh#BVMJ?S8bHR#_cZGm%_uzt?lBkNbX1}E$uOl;DoAVDuokTwSQEGR|@rs?Xi1uDS9 zFV`XFrF>t1fFYmYMeU8paW!3$2M>bDrdx3b-XY#EELrORdysS+Sffq5b{jOsLF+bn z<&GPX+w5XT0P{C6i#O)zcTwp)Ij6sn*x&_apM%onW~S%9;xvUSZm=-X&N5F6-D$r2 z;rh}sJ6PA<2jaERX$h4w=0URatl}MXwo^K^ z@G+hYYg0LH{N^LS_fY>ur71Riq$$fAxJJ6-*d9Dr>Xi&-7Rn{N`D`;i-5*JO0yIu9 zMy~?Sm)zB9E5?DCgL!a)xl+i--3&Zh^c+bl&5~1efeDR+pz8AmwAOj|kMd^&0wG!= ze~!yPf9FR?Sc~R!cuMuYwbxHQ2eR!`z{g$Hh=cW400dlxdZ>>8C(V({`W@v&sODfq zfDt5W0%~-`f!R5EfGC|aj7IoS)Jd%|(rw*;)ifxHVC6|bePHG1XJJvC`}Z=Qx1wS5 zS^-nG2&&Q+O-X&P-^M$ibe?sS@63^2yNwpUoU;58P~H6z?Av54yqCSOryl= zwT-rQ-}Hvr)*+w}I7xvrhm+1oC}~tNv%2a-se8P>^q$K^?#}Rv{@1XbQqcVn%7bMV zxl;4~Xp^(n%cb^)& z_xM-|n)j2WIp+w8?9lv-c?fDXd0UG7P@Z<|7>%>7IMt9i)*=0C@47@32<8MTj}v`x zp;roYv$So-5@7jGsvGxLlmseK;!u>or_AWbrk3x}7ef8hA@^(Wkb_eQygoV#Pc(b= zf~Mg!@W|dFvGo9q(GQ@V-yb}%2FeTm%v~7lwa;a$%9*)uuApPh2gP=qhseK+z@}-c zE3P25ZM_*<4ZvpWfjm>;B!MilRHz&(Q3H2Lzf&x~sN9bd-B%DN6T>W#FESKno1K!s z{o0w2(sTE-SU-G8V-R?(&3Q=74-*El)*e(w%@918z%`q`!%699Lm5^e;;#e=vY8>z zT=&T1zceY7j9)laXMw0IImLZ_HB+;+yHA%dZ`xDK6z<-&3#`GeNCF(B(JUc@;vZUx zfyJiI^QY8g{tmlO_R;l+7s+#@5{3eq0=SEFe11! zPn(gz3(w`sijfX~-g_VIZZ|E=-jzExiB90E8B#{*?va>fmb<$`W)*S#4HN>h=6O3; zt)-9`IKOGEe%hT8yao}uI}$sEt?je`XQqU!`tW7GYZT-hLi73torG{{=D>hw)}s_g z-Eq7@wtbSleug>CH+tn1uh2~tfl$E(|8r2y#8zJcA|<`*kOmA3!=t7;%=YZ2cQ01l zaV&Zz19E1h%B#%wsH8ps%1=gm5z|kaK_bWJZ>u`V?jC=si*-R0Eq7btsj>$3P+a!0 z7gYR?GV_X9^z{Xb0eqCjr|KU$pM|V^jxxNjAfAjk{BSIirE|&x$5S zNiqyKnrKA?i};=1l!67}-K1I9MXWhR*3krshZJBPdg@Zd|W7*%qFx^n;t#$x~bubtG&T& zZ*8skeGygRC)8Kta6d-5CAvae2jnA4bRW~HAFrN2x)A34yA%{6L55bVW}3G)8mmi< zB-Kv=B6qVl!pQhJFHdOHVttS`%Ybr|m_Rm4arGng3k9b_Ef?(VsCR z3SG56-^jJkMp{6SRB>FW1TD&f zTQ2(DdZObcpAb(ICOWvDs);?P(b|4(=v+z@&c2t_YP${PL4B0lAGGVA%*q#*p8@JH zCA(4^w=Dn}<-fQjK?yMIT3-b%2(`w`lqfZuBUX-=H4(c;t z(@km_jHe;RvqAGn z$r!)%M_LKnf1&+Tcu4TBDLrL|t7o?<+syBC{~qtuObCZq*j368t~!c9igYhOWG_1kl~}@;h^PCN?Ckn5 zk~1O;ArTo;2qyv2)dpx0pe#bfndN6@olUv@(j8L;aA36YwU7b+k$nAXIzC^w{@-T= zPYOi8!RQdW%SKX))TdUYtpByRGjC9412wzdS!!CmSr3>llYL1{@3qB_%8$d;w^uDM zzDJPZhO$VhC`R1L$1A5RWaGtARnld**9|XCJbPwE`0caK;m7#W`?`06Mo>%YxeH41 zq!Y;4a09aGf&wu4uCnGn>UVh$-23{X%i@G9%VV&1V9LYi_}%!I(|gahI%jt#Yl0^>*XSu$M^beY$Iu}ko;UpTc*gcngeg3(A+?RN!&n&`$}}- zTarkYE)L`3f`9$_hR$ujG8@+36WuNfU1@Fr`WFy#|7FQc`vOE~6+v8eA8_y<9YzF8 z#lO@U#-t>vzrl+t>3k%gs(nFzQA64o{ti6 z17%8)hSNaFShrP$d;9eJH?{6N-sd1AodHrxo67e;#c3S zm6X332sXP|w|;_&`rosHTiG%U@x0O@KYdOPJ{B-sJW|}$*i>`+$pHP63-o=p_RJ3# zGWon6*eJ&7kV4<=?R%01o5`lYG{Q~u2J2KmW!@9Jp+9|~utWRru-rgFL`$^6w2q>0 zN?UQi-b&_#E>;gG;Wt+?uv?^p1}{TN!soa@_0F4bQ(D$KtpsKl2tx7CBB8$3xxY5! z2)`);bw;p7QR`Kk^67}6a7+F-=r>TT)1j-UGM{((9u=vk6=I=+_mQoNvcTOoMG%k% z{iG8gQePtaHEK1}p+^Ltx17K29sJ6DPcS0n`kD@O6fQzY=aqQWzMI=A*w7JmNws;Ee2%0Pdb^^QRppLd32Mlz+W#D)xG z!phfcAAR=HJQn;SdbEkTAkQ1s^d)#0{~PFRBbY!LO#xcIja0K}XT(k9N6HeUx6Jn( zvtMTZ9vy4??BBCyx@;&*0Qtf;or{2W-UX~ZP;BU=kUrg6%g`G%46a3GV9;NpS>Wvu@H#)>Y zs>%0Row`#GeJc(YFsbRz8YBsRvRD>be^Tr0yn8IEw4-WxWr{IIFH@bKj=l8dN*%6; z-Su)lojm=3m}$`emQObw80&b6_Qx~_(E(*${?EXIB0{RZUA_7o3taIQ^4YyYXV(QE z>XIG2f?urH8X*;dyu)2n^f9`b>UqZH#Zzh8?wJ29#eXgfI2LVaT6=iZR!i1#PaLq3 z;g&YY#-Z%zKcAW68~*}#xCbAalfx^FlVOd-F)pPte~0MG#tC9&XWQCwxNOm599-~e zxxZq$N{Avx1Aj|yHpu<$uGo(@Dlj|#UG6qZZ^&W$DZv7` z`!{8!8IKn+p!t}J_B3HnyX82eD6E@EV8uJ95ay%ghxZ-X~`(~rYg z$%kLFvjTjK^UiFXS8JrzEiGFe&1TW7$z{dcY19w@8gJ+dLn$a^KwfV$KqG!c+;@>X zO#~lNMSH_V`EQWlYghB}`C)278b3Wxt*KyhpDMv|_u z8#q|2bHifVi-=(DeZMAmG}&H^wr&o~|20Dg7=~hA#rv)^hF`Wxbj|mfQ<_54MutW) zf=(pN;wE@E1juJzMb%bTQcYLhBRh&l)q=hF!}Sj9q*eQ#Vc9D^0W2b8FU-03rpM4- zB(vYcCR*;CUOsqqAZ7pG+qi-9K1retU90u2&&iZHfm4}m!tX^ON5dtzCTbEtYz*Fp z03Ooiwl--UgRxR>d^_spKsyZ_ud=Lo*Z=X^^mJ5RMH$Q09GR9W_5T@uUx`aROG`B_xu$nVMdFl=i6>pu(zqntkP7Y?MGp}5GdrdI&>EN8AhJ!>wSBDMRM?W$H;szb1O=C0hW0gDT*5i#)!*cnZ!aJLOX8c7C-(>vCo1GQkVQ4Rc=J`KswVQq@ePzgUd`tUd{X}*B z*+PL+-aWL422B`4IiZ>9_!&`?C_mZn8vq<@OHO~^RpLn1k#48DKC*-OfrWtw$}d0c zFSaaVfq5ECAy+}m*9EkMM}M7k(47RhkofZPmW{ZTe$gQWE=|)SN@8H=@Zv|viMRz> zYrBCZkqX>HK;N(fL*QoX?_s=!^2;Uu_*nl zZI-xp@b+jl(j~^`v@R%h(@pDKmoHo5OOe0vY5&S2gRQoy^P$z^J!c$KD*k;+{5wO3 zic#SC=>1JzaYC%Gr5fnf2z^5^P~Q9b8m0F9s!2Os(`lv{8m#h)t0RA}^couK^-PI4 zo{pWlsnnzVyS2DHNpi0^!Pc35N9jCMx&t1v*XqpAIRsY zbp4}vnRw=?diE@%;fs3?b<6z?nc8tStBPjZZwr_1A4@;iQYqjnuiIN(c&@%1tH}e} zYNT3O@Z7L|DlzB@5K$6w#W`~wPnDf17Ner#lL$Lf!zoqMT@R&hNR}Fz$2t7_>A<^k zB0xeZ5saxNp0y&?-ivWxv*q4;ZPeR!>70H@*BmnO+Sy0TBi&!34i58?+gxGVG;n>i zqUu?FIVAMa{+Ldxqvttv?%C?!w1(*KGOEe(OP zIwjAsEPoLqn^nq*773r5&6ZD5hLtR@+?FVEjwq)(@s`G739)(gKzBD=FO~T3%|d|* zPAm$)8+X_FaI&n7wYLqBiuuUVd<2tFqM>O)jr5`<^T9FZnX zL_(cC_m3LS_v;Tm0#YiYw07l`&M?wmr+3xFuCV_k@(vfu;_LZ=J$D9Z}8{>G4t+vOgghejcuaBy0QLETbgxvd-Xv1~6 zn(vzcv!`zW^DX2A{L0PClR?1IZv@B7|2Bd{NialW%dwjO%>=*qv@r}Sv`WsizrwbF z+;1P*P`HzsY;2BRP~c5iEpU|3Nvm|g|K9^jP(nlBrf8<{-*vO8U<#O;K~eqH1tVh& zWx;zd4@ug!7|+T?bEkx8l{RK#2)}*fm-6`U7yfKc>uD0v35;)cX!#KTI~n{QGQ;26 zGQ@`UXB>SguFjRBRT_T4TXKNF|JwdX`M|De?RK;(^51h7fxQY5%^CaGc8rMil_>!-?7Y^Jt?|{k46RQFYDmPro8%?k z#4=(38}2`|c-0Op&+qqc)Y{=Xsy zf&v2)0wOT<&?zm_-7=&C1Hup@-J*yNA>Gm~CEegfKvF_FMCnGl;ddVId++P@{r>)N zEtf1GW}dUpKKtzbi8$kqKX~$D>_=TkBteKb`;c083*-H|oq`pf;pNICy0u>a=e;e* zet-z>JJSN-YxwK0piU619nCo*UaO|-KxHx?H>`X9H5>MF)H84IpY_?_AAy$zMB~4o zb?cdg29Ef3eG?V7=*`kCPe^b&Sro?>@R6UwzH$DuRR1~TpMll_*qZaBsdj%q6fEW% zNTHdL>Sl48<+MtEZTMwRCH+8^`&RYLqHgl+{(IHX^X-IV-q zEyBxip&nbutyXO|?2k+iqIXJfVKgWmrs#caNKd_x>OyooBj2>ygak@1 z92@K3R_K1`w-y%Z0XKA@-%pTXvfRn_sS68T16bwfd(4aWl=CU=sEi;b@(xk4wkeI_>wO}DmV!>$4E>wlP zx;1>?fDMIqq_;Tz@)q40Kw9Z3?Hu|09YTp(`2%-q6CU2BsFaL+-0^+kyoM#|YVK^a z&6xSC>UnpHeI;H(9@d@DxWnXWKJraEhMeNHmXfa}A$+Tw*cobGJ&zn)UEPBT|0)09 zUj;(w0(e&}vDh_#??I^Cq`+<>?UeSQY~aKdan(J!|pwJ7YJ=SUKBBR{J0T`R{Mge>rnV%RKn1U&)E za`&?H{FuQpHcx5y-{*jCgIGl7a{C1Sx_u^om5f$cX9sOl0vc1FjCjX!;tFole`xco)f2O!Dv4J znr2f>ukt!6k){2z+5Yp_G1U~-7N(T~DP|mFZj8qAeOdCjNGj=Y!CyXVxeEBjLl&WD zzhrjkzk|I{VIJxKWi(FR-ppRTV<;f-*#6&*kFwxDo@iwnm;c-e}C5ACtxGpvh5(}{P#0_m+4Zt-Jl#C;cB`f zHY7Ta(6g_a)G>?U7X4x!r(&PX&md z>!ghb(%E(ph%lsWV)Bmj3Llaef zsRT%CO7w6x0`)>8w*i8pG1Dyrm2GDtMioSD;fs~R+QPx=3?dz|UWIE6hgwey)2Fri z8UDQnBw!5&xruiERZaiwMsRLsGd+)DeE^AB7Id8gO&w>wZ#X~*ckris!lhxG z$_0XVK`N$Or!&;&WNQm+v1vU@^rs=85~V`KR2e!dz{b*E@R!Q& z4_`fo-Xo4(z1!Ele!XG*Ty1}vAN3nv937thdK&fay9!jtz^tO9ut2vsZB=mX19wJ$ zp~LNe1tOD7N_8xS(XS!H}9zk@f2 z3ItU-&x0M}f4xue@>t+MAUx0c{`r~ls38!c;}BjCxQKeQi1Ko=U~p{FwM^f`VLFx~ zxu)dq{BdTCnx1Zv2}_>b#vvw}d&J#4^v~#M(fy$Smx&K+uc`>^`Bl1Ic?i!(_mf>1 zDc$N4HsyBB5}b(dPeU0BG#|ks%>5}wT!Im;sQtUr-szfRcz%E>c6F?Lz;iZg-jzW-f+1JAr+3kVVkKU-;4Um~Vu^2ez zLto=6ZPspk(w8h+sMCH46F*NAcKcK`7l2^!&n>ukyg_bPvO3}!_?mHO_)P%Pvb85( zk!1Fb?29k40Rif_zKnIFNzC0B<|0HJfxcE|kg2O$nU`tur&0}-jv$t;)t6r8e|3Z3 zMM!8mIR_g3whN(frF(DEa?J&>+bdz|Vu$UjHO42?q=haUU!yJ|+*Vqu5nWb47{@l* zx(F;~$!uRr#1L@yWx;$IdxnS7-Djvvad8+~&KWfQm7?SB&?a+6$cca2E4AM5e1J4q zMn}twgx%4Qnr&wl&lbZ2ZI8*P=LPCZAM-1mluri^K5k}Anwoa7Mp?YwYPDVSxR)al zCw4g39cB`sNtpCVJTOQ+7d0NOnIrO<5X{fV0o4Eh+jSn!CWXRq~?;`_SOglehuW4{scmB(WtEF>w=T`jg5 z^e|%(BZ7N6tEo6gAF@0&c-*%U`(r=okc2#@nFyKGmcB`f$=2OJIpx62;W@GKu2>k2 z*L?DvF~y&0Io*=F9T>;R{nA^SdvPG7jtkk9rZh7%OZGE0yo&%;jnik``saTYGk+Hr zYWvvMUXdB3-=(iKdLBN_4oU}FgH4klFQF>E_DYkTNDUps(6+W7EC{ykH zA1zgHxUOo^T_C~^FfY-qP$vvC{T?7)egDRIPba}OuTpHhQ{a}}-F_6-2Ja+tqiuZa zs;95O;LE03RqPF-I>4d}Mno zkbsh6ES|$1ngjh+I{wUkDTP;Kdo`B=TOlpEM8IFOT^%X51qJasP{BpaaD(E-Wij=R4)=fVz54E}Oep}l;x_ZPX0 zceP2bk!{MDb9fELbbL9aonfIZi6hg7C*e%Ih`-WWtK)3_cB}c$q*;So)1x-#z%Y{z zfufU#BhE<#<_)bpid8H^w_BS3`j~rH%T7&A$^~?OpRWCZr7a_YI))!Hp&x_7?WKFR zQVx)X>1NY?e`nAKXxthbPQ!C2U(Y`|R6Q z*GRe#mWBreN_}d`p0dzVjvAnFOc*$sy`z&0JA*uQj$3nb$mWJqS-QQ#VwdlSd3WHy z_<6XG(MtGh*>scsYj_WSqV@nAu`Nfd41|1eIqEnygA}lK07|w7lHQOv7z30OPfR0_ z5g26Ci!>7IQdmiR+QBSO!zxBvcwgHw|E56Gb%Mn^STJOOS&3XI6YcHTHsWhA`RRP_ zH%et&Xjzn*Z2K^G`j!;U$8g4!*k0Om#J$faEZ+wzb3>fy?9ol?M$~KOL^DA#i6M5~ zMOW#96-hp8iX;Ws%ZUdb#V3!k^!K5rvBd2kb^ z@@s@a!6xwl_8t#T^lWQPg(!@F)QqSS5XNdD$SfpbVGAe_s@HdW@8jRaQ3kbHvVO6N zG;8DMwl`^J#;jd$b2PAEw?KkKHod&wBb6m$jDf*Y&BW3f?gw#C(W1WPQ(x3MR?F%& z8I|AaF?l#GA01-i=j~=1;}iZkdt2T-6Kh*4wU+~}GU&f6p!k|c#%;|XKnB>d*4tqR z;nl{9Zfya{@-awOS1Hg1#08G5h18|HcwI3MHHT-zV;TpzLr+VBYeqQIX zQc&5U!s?NmBdFtSdg)3cRpax*Z`UrUei#zyzZdOlGJP+p?V&FH>sgdg`va5L)=zqT zje}G_Y)nnl}pC8Y>r5(CXSQkF8$q~Nm-2#EFo4V|1I>QT{X0T|ydz&V4xcDR{#H#fawH;V7W1#GKW z37`X<_}w2a%g8Bc;aCNOG=PWC@Zh^ zFfQc?CU&0!ybvq;d5ITFaCtuERMl}Z<>8&*N|`&ohS=~E-K+sW9HOVS#lTjl8rVKn z1C#Kq0Co9$kUTsjCI+b-^&|pfk2hOxuWIJeYNowxp!)Z~CulnB1TV<(PodNm{&f~Z z%J9BT^%xl^JI_kfVIeygxf2o6u+_zlnT;7TMhoZp)J&a}B}dI=u5fRwqAOeS=4w4{ zM3!aakxGU=L~dd=U*3py`HK#B@9@b;A6dqkBO9Uu)VGq=-IF-4@?!adE6`Di6Xg5} zU^gpcBD2HBK9f|yX-CC=$M>yOWz`)%pR*II4enF)MQ|+eOU#h`>reoTyy%JrTAN$< z=!ZsbsZdQRxJ#8=j;e!0!nDRzxG;7xi-6_&>w77;aNwc|F!&aCxHW>|^(+oI?gRFK zj_sw}$oOY4^GUhLc9?L!S7%y@H@(dqt2i>g&D7=ri%e!zL#el}__kpCl;+Bph1Utu5?0k8|&2 zQusW(S8XP-kxXj@YBHw)!PU`B7Zn)sBx=xNS6l2Whx9BL7d$UA`N(X|slK53nDpP2~N?Yz7P=J!g;Vv`V5 zd>uGKv5N!&FWG}#cHF7Z>6~)DyvgatR|FaFyjsVG#IbWL>}S~jXZ5hLV9#sxA~i4) zok7d1PU*x`)&R^=(Mf`(t@uI8pO6`RBYPr_0JHkV25)7r$!|#)(-FCxI#qT}80;Vf zm%@JR6Z${tI){&}kXj&Zv~&ib(lBEj+Gi@=8o9T(_vO29+N|3Iu>Y=nQ!w8V_$BF6ftF z2ksmJ#s%wi^h&Ag=Cncbx}z&Vb$7;Zj91!?a$Wh~G1*@QOb!L;O+w>^#BjFE5;Iu( zDp7SJn@kI1+gNb&0vXnzWD47Jsjz%f3gf9PF{3AKFcBreK2bL;|K=Ml;i#w}T%I>2 zB{@o0I|EVA)d{&2ygEDD`geQt@r7dey%QJ@55m3gPNyE`HHv!hu2krd9L-|xNqc=P zHsdP)+-BTlLdZ3+{s`CCiv9`E9+ooH9Qq>9m^ce6RPA6h;++djJfQ&Z>PT_gX`Q$giwW z^Fh>7U<{U)@$|wNpP;S3#1%p&7Vez8M6PAY4DpNcO7`8L^nJ^mqDwuS%=obFfk_ZX z)RKM%ksxo8;Hovy?=e7Iq@3V>u%r1H%ez<~v7hcFps2%AQ8t z#8`NbkuWbYiq8~@k@=J;NMFv=#(Hu2I?GB;iLVL}>2n!H#Y82l3b_LL%L$-t^sS6x z0CaK|;cCESrW1IoekuU8dM&u9Jthb>^q+x z?K(h=sd4iJn~sy(t#66>INQ};{?p37!EpELrk|lzxi(H+FLc|~RA8LOY6Z}j z=`4{aF&sCmp0v<5)~7~h-W0$=eRn!`3qCI2J$P__s+e807wa8@Fs$zCHa;sv+giER z3#2gDpCpZip9=}+uN*}y(0{fz8s>=y2jGUkx40Es{YzHT>Icyc^DPB5wfRYc0z(_$ z(`762PqkseAUGu-6}0it`w$hDOzVk*!57I@IZt;$5TBTBk4Rq^JNfakufaK*O@Ds= zbK=VCLO;8a_u-t1v86NMvekeoiDus%5PdilgtQeMRfM4?oA4WI5r8VS1;-R7c|Ck4 z|!0e0_}>j@s!@1bes_sl5TKTcz4?TH~neFonbMKcc?DxcH|D#^%Q zRzE@f!tC6CUwlGzDR-TCy1I4LW zbX8?D+wRAuIrr%1_hM%kFImovZ?Im;3@~_v(e}2>BAoRsCvu$!b7bHQINy>DDYLJE zb6RXh9<2zd*IqWuXGK1z=zR=uL+*~{`Uo`@#3J7HO8{ld`yeb72m zWk1tl+X(8-Nm;S2uo4H#0WoOE>vMzr!oTnMzW#Rr-qB0GH!MyNHU8tR%WJPMoqdHY z-K6N6RkhQr$4a9O+#-~Y7cYpuNbP1rQ3mUKP5scyUPXI_g7_%Ww(fM*Gc7;EpedQ} zYcPmMEtYC{ee|&DXNguh0iw{(5dQgiHZ}R_F=b82KKYnKy-q`EOvADf3+jrEZbzF! zjjceNoe}dm`s>#I#E_Rv72i|t2*i4(@j)@ZJ*XjTp(|xmluh>ouTc73x+Kf!fmc;m z#XUyR^!C7)ntx?D9T@#U7PQ7SikbHpiadLgdpgU)HATFMIY;ET<%Ns%Ef8*BsZIxb zEX)mFQj1)TH?2&-IRR<|_8kX(J@MASV6!38fFA1=w;v?IZv4g;_(LJI{0kFa8QO0{ zYV%(nJO-U?n@3yd;a*;>-?h`b1qY$#zX1`K^&k-p#Sal7Si3csox!{ zHi?|K0NGz9^Stc#AmErS!{04^-HY$I;-6*DaC%O&^Qe&@KyU`<>-x~Yoo3e+KXe>e z=8-Oa?^N!54Wsn1EFcvaEK|{}g3muS*aJ++DVN5cY|zEeWM&z7)gg zS|u(i?~9WJ$t~NMw;E2%uUbUScPA$^3#CJE0VC7t4S1CJ+tP==$GQ%R6?*;d20FBx zrrpAaCC-*_o2GbZ@(v7^T;_qQ$*RAPV+MuG^gWFAvHqheXK{}qmp z@hf{}g9+cQ5lLQu6L`edbZ{|CxN7|2cx=-r0ipu5agzod^bHg z!M)sVdmeSe26(gY#;HOM?V9}PF?V&?cF8Iv%>g{v-W#(jpjEX^z%Ho-B;M_b?H-@^ zH&P~jpj$jJ7W=>_u%LK7-mc&!y}hDopPp_By<)VkV57EiU^8ni=0*IS(DTTFX3j+j zZT+ua0Wa-^ddKU=bwk%KV$AkThAPVYU5=3F*p~&0qVCG!!1QJ;*5>(mS|~1s6_v!w z?0jt&egBPGg;pjWIl-IyaXo^`6m2lduN2`a0La%L~eG{^4X79;u(8OipEFbz^L!kxDv3L%C^F)>>C|xL z#T$YjaOL|jk%>i~bF6(w@g1CF;MhC4;?iX}cLKN4wgPR5L`c{~!>$&%$tGeK+Ig+$ z=ZbAF^tgy0^@U6TNkQZJ^aV*2t~gLyi%FcaNRGB1Dv6*Rxa6BvLw4!K&rbipR&0R4 z_sJxyldutYfxo}x1B#$$m@6>5@a9B!p|;9>NNi|SQG&j_xQDp&~`i=Wqg>%f-xa zY;hg=f_8WxysIRPDZ?oW8W6q_J9?bmMH$ze18BbE zl|Dd;#C3C_PnJ6Pm7@?;kW_;kj%bZD(Ef3|C)*TzblWI`48->er)7O<^69EYpV#*{ zB{06d_fm-p2b17>tJ3tQxzHGx88;X^8z+l-SqnJiDP@YEZHAmFN+u0OUn_Y0_D2$= z&!iO8e*(C_?PB^0sPis3FS)6(#W-k(9c?zZFc}V4HJ+~79b2@Yl$Rs+e^4I(|LWCe zU`F`b0>=-nFxbnvX!U~dYRg5G$Q&}W2`WC*g8XoaUiJIY5XlfboNc*yJ@D9L;vj}o zcPeziRIDdpFU%}$jsi37=8dIz`$ZXeqB z%AIuUMQb=iK2kR@cZN0}+*LYlQRCg%a1qblJY%+F&X+?0H9+t)JwFTBkZ~{1p^Ksd z|2!h_?`UjU1FtK;7R=!$fZW+yC5)wsFDKH=5lEdx;oMse6q`-OyRC1!1*H3(ASno9 zBXV7cZD>a5q1rx2ukj@Q< zA}M6ByYRLd5iA-+PmQ_?8AI=`PmWKO4Czsuy|eNvOshDYPZ0Lvo8Az zedZJnhiozWMD>ak&|6->RCJ$qDj7H$F2=h8V=4im3My3J>9Egfj4?&m=bNLzG_LeD zg%}?+wYtn0%>M839_UN05vh%e@kDjeDt0`6Um$~(>e?ID6xPCa>x)z>LY;wF2E4rS zUG#FoYb~UF3F3S6Q8t1uIV_5Y@60KvD6*2t5s>{OYw9rpbue^zGER-e`V) zQ!|l|9y7<%pUkk#!k#`S`C?(086>>=itORw{ck|ZU=JxcM2G4)0f{M@gO!mc14x#O zrMH)4z7bkxa@f^pRVGcUg9!8FqA1)YF`E#e(PXT#1*X0jgQ!EI=n&1;h*2PirrOBb z$85{0lm7%?bSKn~&RdyWXLB_vQ!&O}pa7j2#E#-<8LsLV)vIim%Hu5jtf-5tY~RKE z27#-H$FSK^BT#4Vkm7(#%8th|w{GmY*%B7w#};)HQq@U!=$O66W@*TRN`JYcVrj^U z<8}_DK@#T`GjVPT*y>qa%Top8s@=OqLDzmS8Q;B?sI#UoXQe9Q(XQr&_gK3~-@!Q4g;&4M z+pc~=a>c{B;Bk{U*9g7`t&i0l6o+Ca6Uz9e5Hw0Q@9Ghcx=miab!2BPEDZELz+XSS z*ZTUi>7=a0+>&JNRtKwLY+pKBDEe8tb*7!?3`J@!(4>Amx{2@o<6rmN;TsA*jsj9&Cy5Lz(el60K`LU3tG%`A_6%ZU@sgb!rKhU$)A{2=i zC4+aIzYXhesvbcX)z`hxcFI56TU)t&gK}|rLt|cdVjYN49x%UI@%wN^)j1LG^`qar zs(^Lm``2qO=R^mO%67t8!guL5CDQy5%4S>vxDOC4TJP9`az`xb7m0`c2-mW41R0)~ z1v3&uFIdx~mASy@k&cO znRjH;QewfMhDZV{JMH%g4?a?#5%3kRrhp2aU+9L5mNT2>WZIJg#={LzsDHP?9P7m} z^A?m73q;L-8B>?viz@QWR1(w}($@XF4xkZoHQmV(BjIqN+>9w;0lvQ|R-n!2ygITd z2uz39nDtq^VgqdT%1eRJt8{kwPeRok-OD_a;!h6|Yui*O}+!qe2cy#x} zB3#%Bg5a`eh28u8%>vW2*Nu1#itk7)iW`m%q`q-tG2D(741<@%7FDoh#w?`A!lH>a zo1})uw71GD?q2|sPF(7xL!UtB&ycQZ@M65PRt0C3rJYF2Z1a9=#+oS}C6Mk$oxl+xJor_09(hQI`(!@~~DY za@W$DqNZ7Sx`r|oI@~ox$_wlhIHUP(TO!C=*Rq4VkKzWh)^WR!Lg!eid3xRrOd)s* zrFBPLxBNcl?^O-CVM`WEt>7^`WWfS0F26+M(YG89H=fNY&W!RE+~;-u@laT4A_-K6 z-fZs?n{0?O2(wWRf0u_hLzedMN z#qi?3%mDq;^eFvUFJj|77RMVetE(g!#0ySQ=BL^W+Dx#Y`XE5ZtwHOj2iM=tmBUre zf^%N8HUeDritCEk845Tu@k#-=b#x8elx#$~(OBXup+V#ZSO-N|c`)wH{!8j`b{ah~EE)3=iXp<+86ByP<$gGv5 z{US11V@7R^J}7kTl@!Ze)xh~cYabl-JR3!Y6#r5vOU1etYo~GE)pL|P!%5Ly_^GnB z{B@hf8Y#I)9o3s!{|`W!Uw0_fFaF~V;VwwCvtcKJ+Xg~a?Ss53h~m$MPe7)z^F7HM z8^?>X;(E@F+)u)>X(o%%KwQBG9%>DyK@ExTGYpaM`~L(Mec#1)b5&bZJ!llX_{M=w zhxKqwW8e*#GkmpqxG6m0$N|VcO06&kQ{LsS(Wl>yW;eV=g4_tyIb53Y!J1#% z6fhF$WLNCPZHWgGZ!w1W_>zxSKwI?&$6-*;`<<^031zQ4^j27i{ek)N?8Z1q_me>O zb$LOm=L1lgFec$xk!Xs*F>=450y6nNIIySs*sL~yThrF&+AP=zGo?AJu{K@=2Pv=3 zyng9qI zI&-+fQrldu6KQZ@ZfQX{bXLRMPo`=6o1y|eOS_f3hT~!)*}!mnk&Wzng(# zaZQJVvZXX>+LW30HFJuTVB8U5NbdF{@rc4s*+v&L23z?mh3#_JE%k9X+8B&_K(YV> z+PGoUZQw3a_j&Og-c)V;vmphIRwX7{%Top~RNT-54UAyaqGp`nTBjDw4wA4Ly$^Nj z^vfQHk(M?gs1D_cYk<^u$B=jbVO6509yNKsng9Pt>K+kcnbeEi}%|BspX{k}b-`)|ih0@>2^A(sfblYp2Xfo~Or9|2v4(j9Qn z9+$WkHJ;UIn)d-sFY@XQAjj0_?a4@-5!udf_f;g@nf?aA+F6FxDr{lG;^q}-(^i3H z-;Sx;XuXIgQiAo*93_69g6wk0$fY{4=o3Acf=SlMDnQJwKWIFPYEOe$$8cBiZ6hlXorzy;>C$@DPV;trO78Dqa(lskyD9G zogri|!jt|K-%Gq`1n@;Ixigjc<5 zI?E9M<4ehhbw2Z?t)*@X+k>L~Bx`}___mA{Zld8WsX`LF1{cx`QJu$NoUsAd(=q`L zdzQ0>C3dL~Q*mMT>y78$9M&x@IWKMqwM_!8N0sNQk2jTaf$S3977kA}kmGr$gC>*z zLp@F~ZVdM&j7Zy2d8Ulpr=x*8f|Ho~MIjY%2biC3saQ-YBMQ(&qORPxm)l1pI^F@2 zTv;vfflCVob~I13BuER(GdC^Q6j1(NTM@}Sv@w^C<^>upYrXRhag=?sAJ;PyjX5ee zrTUdX+H_u*xteNT&;kk}e1G$>$C%lf;LpXRG?`^|dA5Tlsn`kB7d4yC6NQQGR)pc=O z!VS6%YZO)$bU(_G>m;mIxpgWtPx0Nr&>M}gV9?0S<)(_#4kCZ$IHpo-X~p)2gE3^Q zx@lhnA|%3d7QMnnf2co^@4bAf5^2P**ESKTc((@=+_GQZ%*Qc!JqgYpPAE@8+qU&klL6E{RuqM^2Rx6a!A`3@EQYI@RL*vAsTa444rY?^Wbi=DqvFTb>cmP4Nz)yVs4EFmx&W6P*+ zsQqF5$KdzSJZCv~H_L5;hWABdH(c(2k}RfEetgxdq?vxD0UPJAy`&_vN$EC5-LfHo z@lW#=F}i$=j{ZtI_v-+&6W#G^k^JlL35iexvM8ArHe$DuR#;=tQ6d9B(qQsIM)Fo? z0^JPVcb1+9r(a^}*OwaVf*2F>XyXic6sDtFUp>x5t{6qQ-cgq}-uv?!kyCtpe40~K zzdQ4v(+yOcF(ZhWmb1l43wZ8?dCMzjibcQZM>+Hp-b@rtX8j?(hEaGu+1^7n($5HSkVQbg@G8`cms1!OizG0I!= zu=3TfU^JecPy7wiZtq-||Dfp{IWLP;XNif0bF#c4I|W6YBS;`g%;szF3kUEzMbF_Nwz$+~j&F^R zgsjb<-Cg6Na@Ra&7%Gzv(g@A4W)I9;kv?J;M`}cCcSiN!(xkUnH3>u5iGh1u6GNj% zWg~~8NJ+H<(QrO~h4y7*9g%I*aZ%vto)dmsP6^aZ!7SC8UBlc~9V#RuYRM5tRM)gF z-Iun7t@Izk!MK6-6A*j`#?v>T5?dWT{gZtC}~pf!Qh@SN2@Tm8qMmCb6h-C8u`amZC4s z?VWF9b5lT-0O$DI1~clKa^Ghbo<6x^xJ+)NbTb`MA2CA#HA8%+VWrsPJn#89&wzlJ zTjh7lsypI+W-`RgP){u<9AX-xhP4atr>tKM0xguKy!tE+UmgGV{klMn{Ygi7f6P#@ z#*#!7tpRX2d&@lvZ+Jw&qySJI0pU%*&|m*Tb=>K_gpHt(O^fuykJ~Fx`Us3+`!9*6 z;LzRiH6O}HdS8!tVgy$j_2eCUD`(aQOIP9yBTK`}%)q-nlUIO41`~mIm>uTX9LAAF z!K-3Fq@wL&XE%aN;gYsw_qN%cuXU-HKum_gZ@sbb$C=z?iwU#Vb z1JFz}lb_iqeQwv6X@Lx!_?xW zqsGR8>e}Q7-;0FlH6%WUWitL+hDvn*(+mYT!4$qgty{9F8MtRGL7JCW`!cg|cKm`e zmOqY)Ng;>ujRm^5+3racI2TLGlt?;{!m1vQ%F5FQqD-!8`wmnT+smT>RyJd*)!hf*HxeowoC+>!oh{i!19`{-u zNrOc@hfD|grnEzU?-w40bLJzLerjXfQH#KKkL+L388Yix;{Cn5FOl*FNA7_0f&#}6 z)Lx0fC)m3t(lw``w9Z!~CHNEq_PtWs2=Q&4jiMp1D8GSHlw?>d`9`CkZ$2J5Q97Kx zje}KKCU8O=IKA4Qb_+Nfi*FNim|5$>g#GsnINV$B>%jbOdmX4V+RI{vV}~~=dr8JL zA9qj+%F(Ah)i>OASL=zDs&#VK9p0DiX-}E=JsT~(F}6{z78I`Q))Z#ExBYgFrqqC4l`y^wr!peWZaQl0M4wxn@aeCztK2gzQe zUNfcH%^VH}pRm&`Oso#?tXX{eeA{c9%XR{D)-PcDxejYMj8mRYRSgFF4qvxR!% zF1~m6mwSqSe46c5FIhv1-puFf%^L`{1 zqK(Ac?JZwU=R)==Zr0GRi|?M5+Ja%C#;Mb3u*E_ZYs2}}*M+ndj!p7Mm_MIKuyEvP zxRw+g2R%VYmNe6vuo+ah)oTO{R>CQ7*x0tKg^&LnE~@+h zGMg^Hi$=oe*t?#=BXB{>kHPp8q z3Xd1cK220)I{AD1eA$?0KO5;N&(#{A$Yetc(ia%sJjyH(&wTovUsjl^F>kzRDWz*S zIRE6Ex$SnBQ$haB^2aenlB#k=#_*v(`*&5VXx7)h(%`j$+N%t&4Q$a}RBKwe#x->ubyPnba zhDnO4F@q(~I{bN7YSzOX2jtqg=zoIT86czwKO6LE{x%OKc>E9x76*4;Y^jmDam>B> z5gVVTAJ-h`%2s8}@b#cV#aFi5X)-NS74l7yhDEJ+xgULyN`Z%IikTx<6yrj$2@A;0 zq9fNb&@huE!>#9}E&;FA-RLK8>;Aw;_t$T=A@++6OL&sH#5oF|`_n4MlhKRguvQ{Y zvz>AEM?$JNP1g!1>#{;f8*ke+H#6;Wb+1?X1PHqy_Z)Y+ZRfg1*AuU2w^S_14}Up& zD7F_g8_Zn5_W5V$U1AsZ_}$mSKAzvd5jbciv2>bLX|)VqU+a-S36 zxa8WT>AEIQFU4bMzn4FmY7Sp(wcb$0U#m89tsZPRLAlw_nR@klgD>|6rZb!O)Jd8x zkD6gbK1icgGwDfsMZGb`QiCK8Rga9TtE+x6#T&!VKCViUF!rD;Uo1AnguJxO_3cl_ z4lwZrNZz&2RHFYw;JHjs(4&J>nhvx{Mk*vTOvmFxrS~FbryG@lMzT<-E3q*vW6g-X zu$AWXN1jH)xz!Wz_4JwLu}dhFMMAG;B9h=-t)Askjd9KXWnD~%Oi9fte~Mm_DPA)g zkG~Vacq`YC=+Ee?OlOZ}oim1bVnR3g>-O+gyP#(ECBgYLXyL%9q15o)owTF4o0^+S&-xn>xglqgO9|y0pETl!;l_uH)oK%Udu;6g z>{dXIxofv`>-RSQ9RhIda|7$arbvvM4v9(?+;hZje7VjOp_pa_IH+wI_WqI)x>^yI zV(+9w_40Bv@eE+pL@(fav96JkPNw_*CNHF};PA$bwTujv#3==ZzYZrJA5(@2ATluY zLeXtpDY=b?3KQ{9!s#bQf?Fj%$$ORQnwT|=QI~zWGIxzl#}i%WD8{_fMP{iaZ}Vz1 zpBibZan8%ZZQ~{Wq9kC`Rfxo8#UlENrD+TqidKhA^>|3uBdKI8WUBKop@)InppQ0| z!1KSba)QUw5+?pQnl+D~Yckd84n<92YFVw<%BLod##s zjXd$0?}8&xfcCaQ6g{^+^8ZR4{AX}HKy{(Bq)(^G-{@O>o7wn}%tDN!5a~K@(BbIN zLnNiJ%9vh$271#~UM%>D3RY_O1HrFV4`D_tpS_4u2VRv0k**|TuEWS9yCdk0+J>i5 zI4WBa^cadw=fNZ#eI)IxUCn(9)UmE7j$jG=UYisdkxu?;{+XV^imaZcu&LP@lIx=` z(v*A1E+}d3V^)-gmIx8?%=3g@#)iRC6M%H#d?We*XMe$c-Y2V=~iE#NT(CxV_S*yS5}dR-gJXt1Iw_t``SwO=iny;baD>xyj+g zVOByZCF{_t@Ja?XCTHEtYT=xZXceR1qt;kvTAQOYbX)sQoUF-}!Js2Awgk?lu)s#4 zU_)YQyx*2jJ&kHI=!}ZqXq6k@Y8DO56pm0hLry#qEA1fnpSU^R8cV-L&R8DR%ooJt zV%a&Sj@-@=T>cak{vPuaQ7&Nvf!)>))AxVxhWz2%gsl{N&F(#dB@y@KyWVV)z5Dm% zPs_Pqd_PQ!lTD=08cZXgkA_|BLTI+WnHg8J1_7pAr(0%XAL(X_B@eztnp}7lVTD~B zzdWt1Mzuhvv&KZ3n4TvFp`*Z&8>k zP`;Hul;nhCjR>ivN)Q=W%J07%j+mZJ#Fb9(9hRaOCf#Rc;T%1kEZ1(Nml;&>xv#=- zS?(L)F34G5~~KUCx<5U(*}Ei zVs_#Qy}Lz1sUz+(W|m&>1-F>DMB00>^W!ot2wq6I5a|}1yhK}BEw9`5ieXzt1NKk1 z3KiAwxgMzauwNijvmUQ57&P9|BT2L7hhJ_Bczp#%0nd+hjbKQ!=C|@AMGff|G?L1d zm?V&e!LgCl%GfMps)zd2rPC}%OyA&#%kE4vsNj|(wEoJgH!~}Um(N2_`mt{ZqMCG) zl2NzfdQtmmv}D{E|9bq#Cqbz`RUFj8hUOwoD`O9+m3ft{#Mg^VTlk7?6~Zqr-06LZ zZCmyIGH^~>fUxQ{p#G@C)z4zSI)Hy+cH$^bQ3mUu7WF@cv_up^6@a-+f?Cyzo3(>L z0WVaZ;59A-kR>3Z*z&uLp<*yV=vN~&7d5&JeCY`!;ZHMmzm%+KF{pXF+4Hdni+2tm z$mR5V)0+%X?WR!86k8tqmc+L5@a5Vu1yRFNQvO}#s+XDsi&|J zC|3nkTH4EFAq8zPKD(kh6zw@YpK}KYD zeLGvLgrgl{ZZuA!>ZJ^GZW@6wg!tLwCmR!7FKHFS@0gpk1FDkayuK8k=9TE^lBTZk zX7)~AO@^@6n47~#sjkTUpzM?Iu>m)pg!_R7T;_2{kE2@^Od^j+0}P#2`-jFpuYgJr zC{8^fgwRw%<+&|^q_wI(M_(XT*v^{gx%@8R@ zhJ($k`vyqe-~Yt=dF^Vm7uy4F9!gKGpf`Dw6S| z@Uok-wT!HV>w}cB^xZKneSB-AmQx3`GUUjLL4|M7{Ar-D9vFp!UcNu1UN7LbHLGG1 z@r_{d8dkx1&(zI={>%S!IfjM-7qNE1!l=k5wV9aQ*U9A%v@r2Qo#szD7}jc4l*hVB zzHa%_EiLtiGcY^%ddm!~p+>H)Zdq89fB$@j4f~pc7`{iwc z34?7QTg7YRX)-Ln`#>uvK8{7`+W)ct%L#IJ^K%eyJ~~LyANjMiP(cfOpXv08uQvoNS`m*?5ADr z>nG#Xp1I9+t_H6pWC@@Y>;VWvla1%6cKgZaCydt)0b8tEJzb_6GR#W^SoC#|spF)d z#$cA&iSrF7a~!n*?RaO)rt>r890*><+U4uFw47^?d?A7b=pS^J%1-@0mWG%wPqB96 zZ)y`~;Ly5_%EN@=GIEcLt!Z)SG-0<&y3%>SOI60Y(yF&jGx!Clv>T^kFmle1fHix; z&{24il9w{KVdbWRVDTw@>fr8573a^jv z{-B6187_g6H-Q#)cu7uBl%&GrPW@LM$Cn(Uh&9C=V(K%4D|xAFKa*gs(cht2{v>`@ zTnlhPW5ib7eEQr(9}}YF%K^!!1{i$2%GTTmsm89dlfAYIL_oJF)QwpyA9e;~FOPRV zvWBhzmNfwYcMId5z@A_dpra=B&;@glw$9l~<5^yMJdMf=pebh6i?JB_pHc)!tjI`^ z-vYd#cdJ=K+x+*W;-sYga4|GBpRkEB_uQ+Q6U2QQzHDl4h_8U=mW$o~+ZlGZAN!bQ zl0t)7Bj$pOyG`jmwJ6qT0O^WhaIbi4P-arBUu6=u@}NU*d3+$dYa5caK<5|BII1+9 zzunVyoVY0*W8|KD+^*`~zn+CaGvS$;4p;N(tZdjWZf^g0T6R8PQYo5Dq2PJ8JJ5J? zi1vO`UOVSw8{pZL*ew(M8iejso`mGb^z1D2w3QT3A1Lk`I-T`E@UV2FgzoQw%W#!2)t2HVe*^P7j6$B8?Er zUWdCyK5O0p#8*wuV_s$@i8KLOxKJ1n0Sv=OhL$g@NZfe|3Y2SPD!(J$e!?#3dfSld zu;AOCKkivH`AX5?$XPxQ^$d(1dPojrxe14FZ1&-kQx~}pELSxmos1r|$Yddc zcvo^N`lnwl+`q@t#fj@RVn>MuQ!K3C(}S4iwSb-!c^#4@*qPfB0~VbziIo>oLY4*D zfB-nc+?dE?@uqh9U0&UtGS^KLj#fX~et=jpoTnD=JyNrHYwte}+VXXr=2#jNJ#Xk` z+Tq0Y0K(sZ_YJsp|CFk5*hI<;sriWQSNBLS1gJqCQktlPL=A!rSaG>IjRh- zLB-Z#6KTV@OYxAv%PGLq@6T<)WvE=g`L08@cPAkL)wd7uC4-QIQt~lyP+l*Mf*fUh z5XH-xF-BXG;VO%FArGt5a)m}VgQ{LAX`@WE$nKjK4ge$smkaO*%l?{WPYI|Fwg5C@ z3k*I2Pa>V1ySP3@7QYXK{5B}=KFzS`FESQSwj7)Ydi&6Gq$2naRki<1g9lU_PAVzt zC24Rnv#1c^6F=43@(SZxt)oJ_JigE-*$*AdxjXSlCf=N&NF@%5D9o2iAMr{c;$M>_ z+Hb5XUlz({uV~0I{Quhf&akGoEm}n^L9w7%KopfCAR-_{dQ+6%Lazd$N+&IDD zkndawu`E0My#N_h^*#Ob&IAkntSvh-IH<$o=zTFMT4u7*2RX<-+7rERgg|8T{(ok_>%T7pJ zR)z^`21pZ(>2qP2n6k4`ZblDoJfMLc0ZypW>cDJOsc|*cgK`CiFaH@KTsdyjkVX&T@3A(!*mv~+X|Z3<_Q z90TSJ%hbsNO+laaeRh_D%6)FN{~RbIJsbNtuf)kAVk8;^yAI&C@0gRDd@mA4=K>7g zqH-7at+UIzl2~)KBQN*4HRf)-AM7H_AO1`)p150|fedE3WD&0a+J-gaZb!G1nKs)} zw|Y^7%%W%JIfHA~6a>+C5+3H!yJh!suVHt>r3D#8FUMwVRrG$i+vM`Tc4l5eFjuuy z0#81V_iy+GHY;{Fmw@Gf&!30k1k}|A;*swcwSS}^29?0e$lIng>42_$pTcB%za9v$ z6C{s}Xh^k_k$K)YadubuMS1D>Y4+Y=`gDNb!uH= zPivmCzw5P*?&K}G_UbS44`5(Td%o^vPS6cUSvLCX&jQ0FLfbfy#m~;LOI-}Pw1&a3 zB8!OJ8|<4Zn&z5sA72O)Tk7cdqw^9Q%DFVgg88}TL%n@&nWrbDo`Riu@o9pW4>ff) zM}1K<&W}kR9|vxqjF+y`&Q5mfvUwZWVH62!=$?-&21@w4lgV7Z6i!gMC{0n2}N=aVI<@Q<7i zKiepcvA(ye>|YPs3uZ2st+=na)~%=b;J`8BWidOQkr)hDs;JW30Pqoz$|x-pjc7#y zp{2{aMZjD=)`&Rex94yn#Zb5UyA9`GY-r~*g-96kE?r2Y6OZ{LltJf@Q2q9jvkFnM zCSi<4cYXbDJ2wWo!M9ALj2i4hWQU%!aN0aIP`UZ$LkX?(UT8zQ(}BjmoTrqELrfuC zZyn6hd5lu(nl6JhC1u)&+iij9xo0G*Iy_IC&MU$EOk=gG^5&oUUY%usxCloce?6>l zLUdS|_+}Kea?HYx;fAB1dsm%GseL!A$;VSPbk^}B-}5-Li;%X{h8k|EmFf{q9(C*#De$Ew(+ z-oP5O=JA)n@CMp2k=x=))(ZYtF)^vu`h?4`Yh(gHRrbKw3-|ahwCJEj1tJX!md3tEL`S}8#+t=J^-cDqhtY~`Ow}lEO_%(o*&uQ;66Osm< zdxro9G;Q=TVWGsM50Vo)jM=nsJZGvh9gw}+l4m-O%`F>gKfC7urM$snVT`<$X2u{? z43uSBGh67zAg?`b*DDIl0Xz5Y)dAUx{m5nn9iQapYs~%)U{s&Hf!y2&>ZGuMa)|J3 z+2{k4Mecnt1$vUCE@&g?2Az!`zx%ixWCegJ*Nl89@{B%E+J4LlQ(|6$1GbkVa6?nR zG^Af(@d7MzK>`PX|6!!|&lNB7!c`p#qaxB+`7;~}Z|@0P>*+t%DML5)=RR>mr`}PZ zV;W^J5u^}9C|a5=)YEZ(!9E*)_uT_1G84*$N0|lF}=3GV3Ev&f!6S zw%hWb87HRsB3H}VXZ@cyw_RO$WV%~rIcxy!3I?NBNgFv(?Y6x{1)##o=r$xPj$JkR zP@G8j@j9Rc=2}v7=U@^t?q`lAKBmvjy;LqA+FJ@Dn<$h^8YMP$%@|r-W);f(vFNZXXgT(A2m22XgvRcfhy}%G0)fcl96(%6FL$bFFK3G_A+*5s3?+Vo=*hAdP{XTqm7;@{AVJBua|vt0|CX?wC&*&G4z@YEYU zmBtlBw`DT92mrYsu>#Rrw*H_sRL!Vigv?l!rscv7mv6ULIDjDJnF$qG$LFXd{b(Fw zUT@H^y!F=0(5?4CiVg#Bt@)X38FW5pgh?^LJBCvye=d!dFFvwx&*sSJ_4>fWi3Ew9 zO7eE6u~y8d^m{naHgCk(Irahlmj$3wv4c;#74rd%1QJM0OKm|a^2>v)IyH|*ppNCW z4^U7~SoA43NQn+9J|)jLcbvy#)9n-FV2pe#n3ByU(m_%g`hQ>4$F#9^PaR}=HbN>nxKDyE{!(q@gQ=i$60Rk2DU zha=UC!&wyHrvEOAb>2ga(b=?(6z4Ye+r&8`*G%*g6Q6^sOW&34!yxk=X!*7QmJ9JW zbA0aR9;CCOvowng{XzEhc)IC|t~~@*P=w}H&QWb<2<^2+r%A1r+C9B)<6|X-A(Y%u z3TNtA0JF~#lH#Ow{mZ8H;m$>Z zcRi?7J5F)lx^K03S!s;%g-50+A2Rm&hMaJIE}9Y}o&FZP5e&J_v58aGw(KIN0}z13e~WNf1*)d+=G^30AEclI zNYerD`J*vsX;qB1sJb$97u>8r^IO*7LKQ%s6D1EQOM$#{+4f9ibyN%BaZG60Cs|5Vohg*tiw!Dh85 z_|N|>MB|*3B0};R%df3rOtoJglJ>&ni`Wv$RsXC-{VS_|DJ*s|v^C`(RqSKmoL(~q zov+(WQMrYP#~>1EN{cb8H!+e2UQ0Hz>aSw9vHsjt0kQ1phG-C3*cd&TC@{Ja$^$Ndn`*_H!vR zwtoN{?!`=_Cd0tK*l_IJx-LZqU7V@KU8FfV)x*0AFW+qHB2SbqN7WeyTxMOhxulsh zL(p?(tP@EQ1^YfF$@|~zG@#3SNanSW&7raSAwu%WCw5_^kTLoU@H;y8c zVupzd6z8SSw*or|cl4Wk&h&ei`bYz7%4E*4+-|-r_H=fBRX=O*tv-T2fo|-Q>11;x z{rAiejt+h1#)lNawjYG4mBM1?b`D0XFItZ_6VFWr+{V!j*^g`x_6p{1c&^hWE(_!>O8IR z%OlX6zg~FJ-<2ovSrGNL<$;|eu4lI-;HK%_cT{>KALMk$RL%){wZ3S~$ zG`SYN+3(m{_f8>st^21PerMlqSts?&y;nVfxC`KS8;b7ewbvo%i)}H#9mxOm^&j5H zyVQqNC~`gCYd(z5bBZu}zSi=NM`{BxEH4@r`zk_Rd3U!w&Ir4(X-i&A{+>vB+x>ke zW>$dy%;Lr74cJz*vmEE^FjNETr-U)LaWq@^A1wj-b%yOL6Mrmo|2hE*bgqkfcRO-T=QQ8x|VeWzg2VAlv^?35#2A@Wkj z+|SHmTLeX%TwNDg=m$FfcqwdlZ97Ga;m^zX#|yn)lGq}Ptc6Li5>teE!)0Rno=(mK;9E_%DAR$g7R)R5}B}-2S(F|Id#!)Lc3L ztZOI#$dSMQh~aTk#fS|q-hY1QUysW_-qD*leJXu!PxZ)efBEy#(klXYzKuwg__rT< zumTRVw$F0p&+q*9;(_{6uYN=KpkFiLFE2V~6CB3*fbz({Kh*zu-Z42~1{KwbGyd&I zI$gnGY7WSb{QW(aRsr`GvRss$Uoos{Z2E0i4>QU{|TCdzg_FE zcmDsFCLKu)ua2LN1Lh}nJfJiYT^r!o1ny3ZA4$jF{0R5C$?|gP->$D9^7PlT0oA%y zw9$aFss%T&T{n77siM}N|9O^G!YczvI=g@zrauWp$8s+Qwd}g+on7W{gJJPfC(8_t zqmNnseFjL0PsJDnn*zmOMj4q=w6=pf2O}zq{^Mn%KBu>?HtX_{g_1_5y^` zU1B$hJ##?jsj&#(D6ikMC~S!#9Nh<%A2@Ian9w?B&JUifd&SU>^E1uv%i)vu8is*u z<0|Hx;$3zpE@Yh3eg`6P{&g2cwkWO)2_9+NG083$mCgl-=m|+_$sVYF3XM z0k!Ojxn;+C*W#wjnh!x*^3)DoIT!HG80fY4&wCA?H7PA&+R_hgiJ^y|yMnLB&+>c? zH3CXY6CGl2^!@FK5a(v#Cyo5pDxvz$uqz9MWVyhq9Tn$&PSEMwm`-P+Fg!*biB@i= z+P!(O)iPB9Wwze)LmLq+c@=NBr%nnzst61L^3JZbn|t4kfS;2>yGGS~*LtfdXe` z<=1ej6YkB<;WPEbfqZTh{g+2Gz#a6#nr?vBN`OA$Zzwq!5}VT>yV4%yH#A z%M=4LLx@{7s0fPN6~~X~RCZZd_Vt@j1upXYQA0|8IPNwNsHMaMw54(wv`(0}Ua!1~ z2fmLM`Qp%If898d|V!OKWfH5@qdcLS=HUBYknuy%aNW5&`pj z7%bXGELfrcT(@Y2^?ZZ}A2FlJDi_=q3tR5nrU}4<>q6@96Gi*!8FKJQ;>Q3Bziv?nK@2T(aFMR-%fgG%5b8cumD$1cRexiiG z_Tx?Ci)Wx>w#@#-cVGC!Zs(77amGSB!(BiYEi~i01tuupsWghK<9LeGv7gw~MUhUo*g))(EGzm3L} z%Ux>#uaHkDHwKp1E?IP)6#VY>u8hBpe7u)J2g#g|P-0zBQH{E{V7Dot+n8cm{UgX~ zZ2aM;p^jI5_D9qWn1XS>JOCPwN6hbmO6v{MYkf4MZEKn@nO>^9E_2R2Z)cA%379AY|Y;?%O@CFi9t#@0&{ z$^n#9=JkAd$UP?;$315KeqtaB)^SlB)3gFDLyq5=bTf)UIqdN>vDELX#Uq5O!RE8I{*c>s0c8HxpQ;!4FOf$hHJ{gT#! zdkf4lRV2n?0qSIMx~XS9vDObMUB*{Nr*&+Z33G1;yb#zR>I?{C67yqa_3DqVTgPC} zrcbOTZ3ZU%WIy1ho02-%<&!V>@yJMY(4gLz*+|T1IRzg!j39nDP433JupGk>@nz+; z7JNav*em^DXDfIF*0-ilcJ~g?*(nV#M6Z1|4M~JGEuk=yAGs&4cU*w=Tcyy@kcE^>VhZHaB$Fk_ zQLCLmA&(?hJ<2U?rpoODy5^zUPI@A*`9b9R5EG-G`)kPz^Vi_~cpb!^W^5P2ZsU>qO17nXw1ue&z?j;4km)3*V34UsqOPc? zEr^_Rk>814APW904B-OU+Chr${?cd5>ZOrxuE}yssVQv@?7`wFlt#_bvpGwwf%Pvl zccgtjamAT=N>b|h0Klq-?=8IfVCxYA!b(pN=?qPF5AxkPwC_iS3S+-e4HR)p!_pyN zvcMcQ$ZmqGP?#CMq!bk1z%Pow@k-0nvMn{=DcG{vUs)5P;IOfeah?k(GI&|IP^YbMa zqdpp;p&fjI4cIDgns4n|;>$`%6=7?5?yTPYiR5|MG2H{=5FBFn#WM;lNO)WGB`cBS z0vZ`8;HR(V`iySBSOJUJ*`?%#CIFOi^v`Ne5(njJHwyd&a>(iZc%7lOjpsZFH;=x( zbtoG0+GhQpt-EJNC(Nue0qj}0aSz*MCKXr*812R>9)qHqj(4fP5uZn`VAqR~#O!@& zi|dh(IF+vWV;#7T)NF$}C1jsh$A*oQ&0YCn_lZrZI>-!X3vq7ZW#Ho_+VAgD@|fak zB)*45Juu)N;93%~`<~1jDtVPtmT)3U>b-H;Cgj*m4jMzGv~OCVl~t~d=+;-v(5!r} zS&*?Z(*dSc=eUiTdn{?us1!-puXk=%e)e+EwumC+=T|1Q+yOIxP839iSKc%q4(H}d ze-)C$R)mxy=yTd-CdAJ27#b!IlPbn?-&aB*vuej)*0%^Of08{}27HLl1tpSYN>>k* zj{}&^3M`*HT#mH_?NQTVBTbu6oBYKKjsS5yi;`>L7+Q)*-Zfl#!2G4_*v+j52eNw9 zkaqc=G~gFEZ1kMa z)<)OsbvPP%P%CUn7yT0y&`TkPjJysvV}iA4gjq7=z2h8 zu*md#3|WLUi>hjsVO-sUhUL^s6(P@}qG+E;fU|(D&3^ppvJ`D-Df&KW?aKgPSJkvA zW%wex>ATzNogEad>*357q!rxl!i|B{)A!^?TcB-8EI-2~EPg*$lsyE7x3me+ESZ@r z9JO6u(7;wg1f-GgQ)eA3#55PUwGR|_jO+x`f|E@(MGGOvs3l%u^=uO?zKpk3eM~aO z&Cwvy^a$tDb1Z4+CcDUxJpnAf{h8fJ>}J9qXueVso0apCf334ju~3H*cRij9eU|rI zO6rQvF#Jl$b}n+)?Od;7P;`CHP4tTE8SwPgbJv3a^n`o6vz-O*WtOIg9kRVCuoi?O z1Jx}qI&$-|1iK12$5{(nv0CNmoG39ZlEI%K2dnXhCO^q6Ie$ z$|*7@qUS5a_98R*k>9SPk{9Jn!cA)=o1`oq=8YUIZ8I>_Oz@%(nIAI1!_yhGs?a{y zx8sbZkvhVyd|U;4W#~DrwUk8T{P@4#vN$fYu=cq{u*|vfVv@YhN4!k} zA$9rAX@g;;#{*%}INfgk>IFCZHfE7tYk=6PdxX$NIChTT+FE8jer(o_Q$Pn)Z!AKUb>%i=Y9>i3W25ew8wZ&c zwr$vJiWWoF$0gm`{Ayp>TDERafWtrRcAMj@j`!1H%xRDGL1B~yv$a+L^!)jRQ)Okc za3P{Mr=(X>%wXM}eR*S@#?yv4AoB4M@cFd>d%hWPm&;^2*X_ug?D?u=m!XQ4QW=A+ zF$fVTOB~@CWdqH`%BJt59e@p`?2fC%hE_THJo{({Q7|JsbEmd-g{e(A+D9P? z`P|ov)&5Lj+V9JC>a?Q0MF18C@TLpz<1bDx0ZhDW=Ei2l7n-5E_7V?JyZNlQQ}KL3 z@8y9Y2%POs#FHu>Ba;Jh;7gEAlVtkR9ojJG{DiSgJHW0&z^NEl(1t+ba+tjO_2xa3 zzl!ex5;}ioyTohx!J=yeMM=`yBJ0iYx_!!g9+`{;xHBw(agz>1TWff!KR>Mibp1i_`c&v+MW(ZjtwmJzo z*4VU<<6by7vFNau2=#rYw{JyZG#CdIOabfDzz#V6!$DrSbaKybeB+@vYD+;5$^A&r zmodOqfb&fK=jv*&wihR)V>WV(1+pau6vkI+S|fdJ_?IhYMugc?bv%x(IJNNM^OpKO zP%)9+hS)L?fWjdn$Y@H|daie=g0iS{GMme%fD`q_*Q2xYa*jns^=v7Gxb*8WclqSF zXLpVq#cp+j^JFiX2YHDp_k`07cY-UhJmoE6V_T(`Y)Du!Bk7ZV2Ui|YBPpRc!C28g zVrBP@UbTheCDe5r^IHbUxkC#Yl%`%+;HDq*@hz!m@Hz;lGus}`pZ_HW3BmvT%En)K(chjqsj-| zO6+PU9|23m>pluRds@fY(@1cE^tZq$-zN=XR%=C(ZkrlQIhe}a77!15o(ZGt=fN&$qFn1 z)o=3x)^~lGQKl<+haQoWr6MwxB`csnBGZ^poZAPHrn;lgUrn(lxcR>#rY>Z>D* zt776$uo+wTuucB^HU!%8@TO^PTrtm?z=L`s%=h z+;thGw)?(Ig5`1TpD0fu$kTR|8AGZB8>ILa8I`gHLI^46O0^K}K2m24#4oT=i)PNO z!q_se)GlkTbgv-@kWHF**LZ*egjI&*^>jwAY~Y17wTIPb5|L$o+)EJ?$)!tC6*)+*TO^=lg38XOSat(h{(V1k zfw}7&F=BiNJ8#e4#>a1?CVgaaFC{)JW4ELxi&T_}3??0MkH(`)ldG1hC-*EPnB#i! z!g#<4b}t6`bd52n61qCYOBzR6dhZ6whB4C~M ziaZ}GTU#>Oc3E0kH4D*R)@z)Xmk|h84M9yge0p+a;|8~gtSL@dDzi`f&f>3t+d6jt z^yPb3iFNj3bvd}GjoKf=bWHFv`Xi7lfZZZN4-3?TuoIgk|s8Z09xNT@(oOYK8uMA zO4}Huds?sU%r4+-v$2h~m${qC3c*|I8UdAe6nJwSSp*S9$A%krulp!3_ z8wtQGa#kMI2KcrQzQwf%E6ht?jmW}TW{U+bW@k)vRPl^MKp^>edj6&(i`BHc1~y1) zL!ulqr@DSkwEvyjW=#C#?wq&G_=8(~9IU#pnh>WTvn%hqh^nM_u(@v%+A@*25o~rl zQ%2{*Hz?Y0(+dPWmc9G6>%9(%8Me~S(vJ{WpN_I+o7lWUYH_pyXDw2+z88NaH_5VK zhX=`9*Gpjn`I^tp_87cML>LAUm(QRQngS5S0ahh*2Tjp##0dwnv0a^{AA!aXL28v; z<#)zCWCg@X_lrEo$ghA_2va>iNGfP>nE*VHo>zDXE%qW{gqf-FJtiPYx}lMb_2{d7 zmq3Pfv4|Rz&$H4pX3eS^fTyoONQHX-0-zsSaXit(-ssT9KG+$^_uxW!AwoX6>!@1M z(|(inihhtudJ?8BSv~dYbbB8IJippY6l=(W*7XiPaJL0;3s*f@-$zu(r_!iJ=^~|G zs+RLUYmKzo;EJ~Ct9R`aQhs#uy0!4HY~Y6FcYcB zG5s5nKj!g;n(my;35!YDd2ie5EMIF2RwGV1?4WJ%X7#%&imYlHG;_ z$dy?N^yO?rQ*wv?~x25yL8HYpQQz zsh?qZ({sw#AQ&n!E;q@DmGT)#d3N+N7qj2{OZ%&7FsGgc?o$p~?{hfUp)fg-n6oH{ z90G~l{%R4qtSrxLn;I?-WW>O4H? zp5vXeD20;CnqDo3QHN&&fH z>on@|OnwVIDzu0lEr&rS7P(n&%UBe5ad!pitiLTIV^W$xWn914a_{2|=mjfiza?F_ zeHNpipGO)n3wUvSr~cra0_=FftMH>gLF7MHR>k6_*qH6(xXsTYV?;6}Ef!-+$%!(A z#hpB+wL@3mva1=DQeJ}KSuW|5l^Lw+u=)ji??kt^0dTs)e(f2!d40KFH01 z|9o)vaHM#u^PE=>GSV!xY8LzbMfPcm6hj(JFWAVluoPQg8LUlX%h>jed7oN-HD)eG zt7zb^^E!3PER5;hXI0jW^Goh5_)c3;V2~%I1)<3s`peGsQ;%AU$f{Wn-M=Jl7+$`+|Bv z(LMa-S1i1Cp{I%ucEq;0Zj>T;*2vQoc6u4me&MnA@58qV8zrlClBM10c^O(QS^}Uk z>yIw5pu(>%13!PW8$S#D$aUm4z9QLN)s-5Xa74o$&56}0BM?xVh>b$(zaB1;=XEWn zfQ#)wEj%b9QO5|YD{@fR&Q(3}>2zXHio5@`HGofbH#S`1LvsxChs5PaI{*TbKb;wx#l?R(l{(c)8yeq5 z|44GkhnMsClQds9kkRm$R3?$#AE1VzB&DV{fGucM?)i4|m0sYvI$4UQ!_i4@v;Y8L z)bLHw@+G5% z_9&mTv$X;sUse11eOcxQLL3u^^C78}(A8k(ZNKyiH|+wl23?0BH|fyG(0k@HU-vSK zGD;AdsP05}6jDxy@w(JiTTl-1yc^iARyv=a$2jyLI(O$lMhGNHF68w93O?(;@{}fW zqmx1$L;nZPlMzKPoAs(lN3m9DGJ;e&RwMsx7KaM9tiv8X%NJ%bj*7&avMFgJRP!;C z%zYwrj))0QkEgM#97ADTt%?`yhpHz~)_XiHk64G)R80mbefxr<+Y1pl=XfQvGsNB6 zm6ABVDihb8J2&~26JnEWKsCxT`>nY9W(F0@BR)X8I+vU-XdqA|QL+GSRWSeL=K5?g5D7r6STvcT&vs?d#H8{lxrdIb7nN2e~Fg9H1wpVwz&cz2}+S~Cmo!F!4Yl$r)o z6|!U~EF-zif-#;efpextf}%@ z!}takWs&V#BeSMmk5xWNq?(5@4AZV5H|vOKrkXF@jSro)62Qz#8(?@@`sR@BH6^^A zRo(8(eX^fa58~PToHq8I9EUWrH>2lq`mUV@H~LPfRz?Anz%sM~MlpN+*9vAsDm@Gj zjx#s}jvghk1JA<}PVFx1b9n(8y+Jie;RsYZv1II}CqZ*^#4xTJp=Gbx+`)?&e@Wkh zA-U}}#GIYeEAZkw?*KhczMZH9Y~55eVH|k=*_Wg*aD!Q5q0%x@dUupA8F_w;4yFAf ztbJ6e`ECec;CbvkoL?#nj6gY3e4sbU6w9=ntdX6Sz~0Uo3dq_)wpSq1QJRfeIUbV@&D z2H28EuNfzN%JVJ(v!GPjY&(G4S_-5_<$Y)j$(;dK&~kG-5?)_DAJ3LwtfJN! zjQDBjQh^BSSJqo=damx7yd8w0V5u)$CL*1!aE;7L99S~btn()7j@EnNp;My*Xr0vi zj47%^x82K`HIytEk??hH%U<~w?!Kh3Eokmst`vi`NYD9^eu4Huv2Y`fv!mlmjevzL8bM+Eq2vDfj@J)xW;2yMZ~p#w zQZm5e!5C@zx4&_eWCp}k&31N|{`}eBUI@z_^+L$q9sj%Y0AFanc>YHyA=M*$qInVj6kaW;kwMfo!M16fbl2Ar~TVohmYt2KyDY& z1rSjFcB%h(=Rp#Hv-sgp|I5j)UpoJ6Mt{(lqY@EkaG{(lqY_gVM973D9U Date: Thu, 11 Jun 2026 13:52:52 +0800 Subject: [PATCH 061/255] perf(clean): Avoid extra getPathInfo RPC per file during clean execution (#18963) * perf(clean): Avoid extra getPathInfo RPC per file during clean execution Cleaner plan file entries are always base/log/bootstrap file paths, never directories, so delete them directly with deleteFile instead of calling getPathInfo first to test isDirectory. This halves the RPC count of the clean execution phase on cloud storage. Directory-capable deletion is kept as deletePathAndGetResult and is only used for deleting whole partitions. * review: Rename to deleteDirAndGetResult and drop the isDirectory probe Only partition directories reach this method, so delete recursively without the getPathInfo type probe, saving that round trip per deleted partition as well. A missing directory on a retried clean surfaces as a false return from delete and is treated as already cleaned, same as the file variant. * review: Combine file and directory delete into one method with an explicit isDirectory param The two variants were identical apart from the deleteFile/deleteDirectory call and the log noun, so merge them and let the two call sites pass the path type explicitly. (cherry picked from commit 91f3d227264da8d5ca91c9943481867569135db6) --- .../action/clean/CleanActionExecutor.java | 45 +++++++++++-------- .../functional/TestCleanActionExecutor.java | 20 ++++++--- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java index 2a636221ae60f..0d27aa2a9f293 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java @@ -68,38 +68,45 @@ public CleanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config this.txnManager = new TransactionManager(config, table.getStorage()); } - private static boolean deleteFileAndGetResult(HoodieStorage storage, String deletePathStr) { + /** + * Deletes the given path and returns whether it is gone afterwards. Cleaner plan file entries + * are always base/log/bootstrap file paths and partition deletions are always directories, so + * the caller passes the path type explicitly and no getPathInfo probe is needed. + * + * @param isDirectory true for a partition directory (deleted recursively), false for a file + */ + private static boolean deleteAndGetResult(HoodieStorage storage, String deletePathStr, boolean isDirectory) { StoragePath deletePath = new StoragePath(deletePathStr); - log.debug("Working on delete path: {}", deletePath); + String pathType = isDirectory ? "directory" : "file"; + log.debug("Working on deleting {}: {}", pathType, deletePath); try { - boolean deleteResult = storage.getPathInfo(deletePath).isDirectory() - ? storage.deleteDirectory(deletePath) - : storage.deleteFile(deletePath); + boolean deleteResult = isDirectory ? storage.deleteDirectory(deletePath) : storage.deleteFile(deletePath); if (deleteResult) { - log.debug("Cleaned file at path: {}", deletePath); - } else { - if (storage.exists(deletePath)) { - throw new HoodieIOException("Failed to delete path during clean execution " + deletePath); - } else { - log.debug("Already cleaned up file at path: {}", deletePath); - } + log.debug("Cleaned {}: {}", pathType, deletePath); + return true; } - return deleteResult; + if (storage.exists(deletePath)) { + throw new HoodieIOException("Failed to delete " + pathType + " during clean execution " + deletePath); + } + // Hadoop file systems report a missing path by returning false from delete instead of + // throwing FileNotFoundException, so this is the regular retried-clean case below. + log.debug("Already cleaned up {}: {}", pathType, deletePath); + return true; } catch (FileNotFoundException fio) { - // With cleanPlan being used for retried cleaning operations, its possible to clean a file twice if a file to be + // With cleanPlan being used for retried cleaning operations, its possible to clean a path twice if a path to be // deleted is not found, treat it as a success. In other words, there is nothing else to be cleaned up on the // FileSystem, except for updating the MDT. By returning success, we would remove the entry from MDT. return true; } catch (IOException e) { try { if (storage.exists(deletePath)) { - log.error("Delete file failed: {} and file still exists", deletePath, e); + log.error("Delete {} failed: {} and it still exists", pathType, deletePath, e); throw new HoodieIOException(e.getMessage(), e); } - log.warn("Delete file failed: {} but file does not exist", deletePath, e); + log.warn("Delete {} failed: {} but it does not exist", pathType, deletePath, e); return false; } catch (IOException ex) { - log.error("Delete file failed: {} with exception: {} and existence check also failed", deletePath, e, ex); + log.error("Delete {} failed: {} with exception: {} and existence check also failed", pathType, deletePath, e, ex); throw new HoodieIOException(ex.getMessage(), ex); } } @@ -113,7 +120,7 @@ private static Stream> deleteFilesFunc(Iterator String partitionPath = partitionDelFileTuple.getLeft(); StoragePath deletePath = new StoragePath(partitionDelFileTuple.getRight().getFilePath()); String deletePathStr = deletePath.toString(); - boolean deletedFileResult = deleteFileAndGetResult(storage, deletePathStr); + boolean deletedFileResult = deleteAndGetResult(storage, deletePathStr, false); final PartitionCleanStat partitionCleanStat = partitionCleanStatMap.computeIfAbsent(partitionPath, k -> new PartitionCleanStat(partitionPath)); boolean isBootstrapBasePathFile = partitionDelFileTuple.getRight().isBootstrapBaseFile(); @@ -161,7 +168,7 @@ List clean(HoodieEngineContext context, HoodieCleanerPlan clean : Collections.emptyList(); partitionsToBeDeleted.forEach(entry -> { if (!isNullOrEmpty(entry)) { - deleteFileAndGetResult(table.getStorage(), table.getMetaClient().getBasePath() + "/" + entry); + deleteAndGetResult(table.getStorage(), table.getMetaClient().getBasePath() + "/" + entry, true); } }); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanActionExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanActionExecutor.java index 10b9891cb27f0..f7fee3437ab00 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanActionExecutor.java @@ -66,7 +66,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -161,23 +163,26 @@ void testPartialCleanFailure(CleanFailureType failureType) throws IOException { CleanActionExecutor cleanActionExecutor = new CleanActionExecutor(context, config, mockHoodieTable, "002"); if (failureType == CleanFailureType.TRUE_ON_DELETE) { - assertCleanExecutionSuccess(cleanActionExecutor, filePath); + assertCleanExecutionSuccess(cleanActionExecutor, filePath, true); } else if (failureType == CleanFailureType.FALSE_ON_DELETE_IS_EXISTS_FALSE) { - assertCleanExecutionSuccess(cleanActionExecutor, filePath); + // a missing file on a retried clean counts as a successful delete so its MDT entry is removed + assertCleanExecutionSuccess(cleanActionExecutor, filePath, true); } else if (failureType == CleanFailureType.FALSE_ON_DELETE_IS_EXISTS_TRUE) { assertCleanExecutionFailure(cleanActionExecutor); } else if (failureType == CleanFailureType.FILE_NOT_FOUND_EXC_ON_DELETE) { - assertCleanExecutionSuccess(cleanActionExecutor, filePath); + assertCleanExecutionSuccess(cleanActionExecutor, filePath, true); } else if (failureType == CleanFailureType.IO_EXCEPTION) { assertCleanExecutionFailure(cleanActionExecutor); } else if (failureType == CleanFailureType.IO_EXCEPTION_AND_EXISTS) { assertCleanExecutionFailure(cleanActionExecutor); } else if (failureType == CleanFailureType.IO_EXCEPTION_BUT_NOT_EXISTS) { - assertCleanExecutionSuccess(cleanActionExecutor, filePath); + assertCleanExecutionSuccess(cleanActionExecutor, filePath, false); } else { // run time exception assertCleanExecutionFailure(cleanActionExecutor); } + // file deletions must not stat the path first; deletes go straight to storage + verify(storage, never()).getPathInfo(filePath); } private void assertCleanExecutionFailure(CleanActionExecutor cleanActionExecutor) { @@ -186,11 +191,16 @@ private void assertCleanExecutionFailure(CleanActionExecutor cleanActionExecutor }); } - private void assertCleanExecutionSuccess(CleanActionExecutor cleanActionExecutor, StoragePath filePath) { + private void assertCleanExecutionSuccess(CleanActionExecutor cleanActionExecutor, StoragePath filePath, boolean expectFileDeleteSuccess) { HoodieCleanMetadata cleanMetadata = cleanActionExecutor.execute(); assertTrue(cleanMetadata.getPartitionMetadata().containsKey(PARTITION1)); HoodieCleanPartitionMetadata cleanPartitionMetadata = cleanMetadata.getPartitionMetadata().get(PARTITION1); assertTrue(cleanPartitionMetadata.getDeletePathPatterns().contains(filePath.getName())); + if (expectFileDeleteSuccess) { + assertTrue(cleanPartitionMetadata.getSuccessDeleteFiles().contains(filePath.getName())); + } else { + assertTrue(cleanPartitionMetadata.getFailedDeleteFiles().contains(filePath.getName())); + } } private static HoodieWriteConfig getCleanByCommitsConfig() { From 94ceafb511b5e11d48582cc22b5b2dc0c32c56b2 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Fri, 12 Jun 2026 05:33:54 +0530 Subject: [PATCH 062/255] fix(spark): catch HoodieSchemaNotFoundException in 3-arg DefaultSource.createRelation (#18977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(spark): catch HoodieSchemaNotFoundException in 3-arg DefaultSource.createRelation The 2-arg createRelation(sqlContext, parameters) overload catches HoodieSchemaNotFoundException and returns an EmptyRelation (HUDI-7147 / #10689), but the 3-arg createRelation(sqlContext, params, schema) overload — which Spark's DataSource.resolveRelation() invokes directly via the SchemaRelationProvider path whenever a user-supplied schema is present (e.g. spark.read.schema(s).format("hudi").load(path), or HMS- catalog resolution that already knows the schema) — has no such catch, so the exception propagates and breaks query analysis. Mirror the catch on the 3-arg overload. Preserve the caller-supplied schema in the EmptyRelation so downstream analysis (e.g. column resolution in WHERE clauses) sees the HMS-known columns even when the on-disk table is schemaless. The 2-arg overload re-enters this method with schema=null, so fall back to an empty StructType in that case to avoid an NPE in the 2-arg overload's relation.schema.isEmpty check (surfaced by TestCOWDataSource.testReadOfAnEmptyTable on spark3.3 / spark3.5). Adds TestCOWDataSource.testReadOfAnEmptyTableWithUserSuppliedSchema, a sibling of testReadOfAnEmptyTable that exercises the 3-arg path. Closes #18668 Co-Authored-By: Claude Opus 4.7 (cherry picked from commit 182428f803d0432de518ba4de49a71d1a423484a) --- .../scala/org/apache/hudi/DefaultSource.scala | 16 ++++++++- .../hudi/functional/TestCOWDataSource.scala | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala index bf8dee324f4ff..2682de70f1fab 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -134,7 +134,21 @@ class DefaultSource extends RelationProvider parameters } - val relation = DefaultSource.createRelation(sqlContext, metaClient, schema, options.toMap) + // Spark's DataSource.resolveRelation() invokes this 3-arg overload directly via the + // SchemaRelationProvider path when a user-supplied schema is present (e.g. + // spark.read.schema(...).load(path)). The 2-arg overload catches + // HoodieSchemaNotFoundException and returns an EmptyRelation, but that catch is bypassed + // on this path, so we mirror the same handling here. Preserve the caller-supplied schema + // so subsequent query analysis (e.g. column resolution in WHERE clauses) sees the + // HMS-known columns even though the on-disk table is schemaless. The 2-arg overload also + // re-enters this method with schema=null, so we must fall back to an empty StructType + // when schema is null to avoid an NPE in the 2-arg overload's relation.schema.isEmpty check. + val relation = try { + DefaultSource.createRelation(sqlContext, metaClient, schema, options.toMap) + } catch { + case _: HoodieSchemaNotFoundException => + new EmptyRelation(sqlContext, Option(schema).getOrElse(new StructType())) + } log.info(s"Created relation ${relation.getClass.getSimpleName} with ${options.size} resolved options") relation } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala index 106c1f09a7ae2..599a105e6d001 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala @@ -2215,6 +2215,40 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup assertEquals(count, 0) } + @Test + def testReadOfAnEmptyTableWithUserSuppliedSchema(): Unit = { + val (writeOpts, _) = getWriterReaderOpts(HoodieRecordType.AVRO) + + // Insert + then delete the only completed commit so the table has no resolvable schema. + val records = recordsToStrings(dataGen.generateInserts("000", 100)).asScala.toList + val inputDF = spark.read.json(spark.sparkContext.parallelize(records, 2)) + inputDF.write.format("hudi") + .options(writeOpts) + .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Overwrite) + .save(basePath) + + val fileStatuses = storage.listDirectEntries( + new StoragePath(basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME + + StoragePath.SEPARATOR + HoodieTableMetaClient.TIMELINEFOLDER_NAME), + new StoragePathFilter { + override def accept(path: StoragePath): Boolean = { + path.getName.endsWith(HoodieTimeline.COMMIT_ACTION) + } + }) + storage.deleteFile(fileStatuses.get(0).getPath) + + // spark.read.schema(...) triggers Spark's SchemaRelationProvider path which calls the + // 3-arg DefaultSource.createRelation overload directly. Without the catch on that + // overload, this would fail with HoodieSchemaNotFoundException. + val userSchema = inputDF.schema + val df = spark.read.schema(userSchema).format("hudi").load(basePath) + assertEquals(0, df.count()) + // The caller-supplied schema must be preserved on the EmptyRelation so subsequent query + // analysis (e.g. column resolution) sees the user-known columns. + assertEquals(userSchema, df.schema) + } + /** * Test incremental queries and time travel queries with event time ordering. * From 50198ac6b433582387e757c8b682f1090233a838 Mon Sep 17 00:00:00 2001 From: voonhous Date: Fri, 12 Jun 2026 09:53:08 +0800 Subject: [PATCH 063/255] refactor: Add Lombok annotations to hudi-common module (part 7) (#18944) * refactor: Add Lombok annotations to hudi-common module (part 7) * refactor: Address review comments - HoodieLogBlock: revert lombok @NonNull to javax.annotation.Nonnull; null header/footer is expected on the read path (v2/v3 blocks have no footer), so runtime null checks would NPE on every log block read - IncrementalQueryAnalyzer: make QueryContext constructor private again via @AllArgsConstructor(access = AccessLevel.PRIVATE); fix EMPTY constant to pass Option.empty() instead of null - HoodieTableConfig: use parameterized logging instead of String.format - HoodieTableVersion: document why versionCode uses @Accessors(fluent = true) (cherry picked from commit 612e327b51fbb4fc792dcaeecf47aa86f5ddab89) --- .../hudi/common/table/HoodieTableConfig.java | 24 ++- .../common/table/HoodieTableMetaClient.java | 159 ++++++------------ .../hudi/common/table/HoodieTableVersion.java | 24 ++- .../common/table/TableSchemaResolver.java | 12 +- .../common/table/cdc/HoodieCDCExtractor.java | 6 +- .../common/table/cdc/HoodieCDCFileSplit.java | 24 +-- .../common/table/cdc/HoodieCDCOperation.java | 14 +- .../common/table/checkpoint/Checkpoint.java | 21 +-- .../log/AbstractHoodieLogRecordScanner.java | 92 ++++------ .../table/log/BaseHoodieLogRecordReader.java | 103 +++++------- .../hudi/common/table/log/FullKeySpec.java | 15 +- .../common/table/log/HoodieLogFileReader.java | 26 ++- .../common/table/log/HoodieLogFormat.java | 12 +- .../table/log/HoodieLogFormatReader.java | 8 +- .../log/HoodieMergedLogRecordReader.java | 20 +-- .../log/HoodieMergedLogRecordScanner.java | 20 +-- .../hudi/common/table/log/InstantRange.java | 17 +- .../table/log/block/HoodieCommandBlock.java | 7 +- .../table/log/block/HoodieDataBlock.java | 11 +- .../table/log/block/HoodieDeleteBlock.java | 6 +- .../table/log/block/HoodieHFileDataBlock.java | 2 + .../table/log/block/HoodieLogBlock.java | 94 +++-------- .../common/table/read/BufferedRecord.java | 47 ++---- .../read/BufferedRecordMergerFactory.java | 7 +- .../hudi/common/table/read/DeleteContext.java | 22 +-- .../table/read/IncrementalQueryAnalyzer.java | 70 +++----- .../DefaultFileGroupRecordBufferLoader.java | 8 +- .../read/buffer/FileGroupRecordBuffer.java | 19 +-- .../PositionBasedFileGroupRecordBuffer.java | 15 +- .../split/HoodieSourceSplitSerializer.java | 12 +- .../source/TestIncrementalInputSplits.java | 4 +- .../TestStreamReadMonitoringFunction.java | 4 +- .../source/split/TestHoodieSourceSplit.java | 14 +- .../TestHoodieSourceSplitSerializer.java | 76 ++++----- 34 files changed, 359 insertions(+), 656 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index ca775136257c4..73703d70d85a4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -65,8 +65,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import javax.annotation.concurrent.Immutable; @@ -119,6 +118,7 @@ import static org.apache.hudi.common.util.ValidationUtils.checkArgument; @Immutable +@Slf4j @ConfigClassProperty(name = "Hudi Table Basic Configs", groupName = ConfigGroups.Names.TABLE_CONFIG, description = "Configurations of the Hudi Table like type of ingestion, storage formats, hive table name etc." @@ -126,8 +126,6 @@ + " initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.") public class HoodieTableConfig extends HoodieConfig { - private static final Logger LOG = LoggerFactory.getLogger(HoodieTableConfig.class); - public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; public static final String HOODIE_PROPERTIES_FILE_BACKUP = "hoodie.properties.backup"; public static final String HOODIE_WRITE_TABLE_NAME_KEY = "hoodie.datasource.write.table.name"; @@ -475,7 +473,7 @@ public static HoodieTableConfig loadFromHoodieProps(HoodieStorage storage, Stora public HoodieTableConfig(HoodieStorage storage, StoragePath metaPath) { super(); StoragePath propertyPath = new StoragePath(metaPath, HOODIE_PROPERTIES_FILE); - LOG.info("Loading table properties from " + propertyPath); + log.info("Loading table properties from {}", propertyPath); try { this.props = fetchConfigs(storage, metaPath, HOODIE_PROPERTIES_FILE, HOODIE_PROPERTIES_FILE_BACKUP, MAX_READ_RETRIES, READ_RETRY_DELAY_MSEC); } catch (IOException e) { @@ -509,7 +507,7 @@ private static String storeProperties(Properties props, OutputStream outputStrea checksum = propsWithChecksum.getProperty(TABLE_CHECKSUM.key()); props.setProperty(TABLE_CHECKSUM.key(), checksum); } - LOG.info("Created properties file at " + propertyPath); + log.info("Created properties file at {}", propertyPath); return checksum; } @@ -556,7 +554,7 @@ private static void modify(HoodieStorage storage, StoragePath metadataFolder, Pr propsToDelete.forEach(propToDelete -> props.remove(propToDelete)); checksum = storeProperties(props, out, cfgPath); } - LOG.warn(String.format("%s modified to: %s (at %s)", cfgPath.getName(), props, cfgPath.getParent())); + log.warn("{} modified to: {} (at {})", cfgPath.getName(), props, cfgPath.getParent()); // 5. verify and remove backup. try (InputStream in = storage.open(cfgPath)) { @@ -582,7 +580,7 @@ private static void modify(HoodieStorage storage, StoragePath metadataFolder, Pr private static void deleteFile(HoodieStorage storage, StoragePath cfgPath) throws IOException { storage.deleteFile(cfgPath); - LOG.info("Deleted properties file at " + cfgPath); + log.info("Deleted properties file at {}", cfgPath); } /** @@ -685,7 +683,7 @@ static boolean validateConfigVersion(ConfigProperty configProperty, HoodieTab boolean valid = tableVersion.greaterThan(firstVersion) || tableVersion.equals(firstVersion); valid = valid || CONFIGS_REQUIRED_FOR_OLDER_VERSIONED_TABLES.contains(configProperty.key()); if (!valid) { - LOG.warn("Table version {} is lower than or equal to config's first version {}. Config {} will be ignored.", + log.warn("Table version {} is lower than or equal to config's first version {}. Config {} will be ignored.", tableVersion, firstVersion, configProperty.key()); } return valid; @@ -1009,12 +1007,12 @@ public static Triple inferMergingConfigsForPreV // Check ordering field name based on record merge mode if (inferredRecordMergeMode == COMMIT_TIME_ORDERING) { if (nonEmpty(orderingFieldNamesAsString)) { - LOG.warn("The ordering field ({}) is specified. COMMIT_TIME_ORDERING " + log.warn("The ordering field ({}) is specified. COMMIT_TIME_ORDERING " + "merge mode does not use ordering field anymore.", orderingFieldNamesAsString); } } else if (inferredRecordMergeMode == EVENT_TIME_ORDERING) { if (isNullOrEmpty(orderingFieldNamesAsString)) { - LOG.warn("The ordering field is not specified. EVENT_TIME_ORDERING " + log.warn("The ordering field is not specified. EVENT_TIME_ORDERING " + "merge mode requires ordering field to be set for getting the " + "event time. Using commit time-based ordering now."); } @@ -1324,7 +1322,7 @@ public void setMetadataPartitionState(HoodieTableMetaClient metaClient, String p setValue(TABLE_METADATA_PARTITIONS, partitions.stream().sorted().collect(Collectors.joining(CONFIG_VALUES_DELIMITER))); setValue(TABLE_METADATA_PARTITIONS_INFLIGHT, partitionsInflight.stream().sorted().collect(Collectors.joining(CONFIG_VALUES_DELIMITER))); update(metaClient.getStorage(), metaClient.getMetaPath(), getProps()); - LOG.info("MDT {} partition {} has been {}", metaClient.getBasePath(), partitionPath, enabled ? "enabled" : "disabled"); + log.info("MDT {} partition {} has been {}", metaClient.getBasePath(), partitionPath, enabled ? "enabled" : "disabled"); } /** @@ -1342,7 +1340,7 @@ public void setMetadataPartitionsInflight(HoodieTableMetaClient metaClient, List setValue(TABLE_METADATA_PARTITIONS_INFLIGHT, partitionsInflight.stream().sorted().collect(Collectors.joining(CONFIG_VALUES_DELIMITER))); update(metaClient.getStorage(), metaClient.getMetaPath(), getProps()); - LOG.info("MDT {} partitions {} have been set to inflight", metaClient.getBasePath(), partitionPaths); + log.info("MDT {} partitions {} have been set to inflight", metaClient.getBasePath(), partitionPaths); } public void setMetadataPartitionsInflight(HoodieTableMetaClient metaClient, MetadataPartitionType... partitionTypes) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index e20d9d3a12209..f248dd979db8c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -74,8 +74,11 @@ import org.apache.hudi.storage.StoragePathFilter; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.Serializable; @@ -118,10 +121,12 @@ * @see HoodieTimeline * @since 0.3.0 */ +@NoArgsConstructor +@Getter +@Slf4j public class HoodieTableMetaClient implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieTableMetaClient.class); public static final String METADATA_STR = "metadata"; public static final String METAFOLDER_NAME = ".hoodie"; public static final String TIMELINEFOLDER_NAME = "timeline"; @@ -163,20 +168,27 @@ public class HoodieTableMetaClient implements Serializable { protected StoragePath basePath; protected StoragePath metaPath; + @Getter(AccessLevel.NONE) + @Setter private transient HoodieStorage storage; + @Getter(AccessLevel.NONE) private boolean loadActiveTimelineOnLoad; protected StorageConfiguration storageConf; private HoodieTableType tableType; private TimelineLayoutVersion timelineLayoutVersion; private TimelineLayout timelineLayout; private StoragePath timelinePath; + @Getter(AccessLevel.NONE) private StoragePath timelineHistoryPath; protected HoodieTableConfig tableConfig; + @Getter(AccessLevel.NONE) protected HoodieActiveTimeline activeTimeline; private ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build(); private FileSystemRetryConfig fileSystemRetryConfig = FileSystemRetryConfig.newBuilder().build(); + @Getter(AccessLevel.NONE) protected HoodieMetaserverConfig metaserverConfig; private HoodieTimeGeneratorConfig timeGeneratorConfig; + @Getter(AccessLevel.NONE) private Option indexMetadataOpt; private HoodieTableFormat tableFormat; @@ -187,7 +199,7 @@ public class HoodieTableMetaClient implements Serializable { protected HoodieTableMetaClient(HoodieStorage storage, String basePath, boolean loadActiveTimelineOnLoad, ConsistencyGuardConfig consistencyGuardConfig, Option layoutVersion, HoodieTimeGeneratorConfig timeGeneratorConfig, FileSystemRetryConfig fileSystemRetryConfig) { - LOG.debug("Loading HoodieTableMetaClient from " + basePath); + log.debug("Loading HoodieTableMetaClient from {}", basePath); this.timeGeneratorConfig = timeGeneratorConfig; this.consistencyGuardConfig = consistencyGuardConfig; this.fileSystemRetryConfig = fileSystemRetryConfig; @@ -213,21 +225,13 @@ protected HoodieTableMetaClient(HoodieStorage storage, String basePath, boolean this.timelinePath = timelineLayout.getTimelinePathProvider().getTimelinePath(tableConfig, this.basePath); this.timelineHistoryPath = timelineLayout.getTimelinePathProvider().getTimelineHistoryPath(tableConfig, this.basePath); this.loadActiveTimelineOnLoad = loadActiveTimelineOnLoad; - LOG.debug("Finished Loading Table of type " + tableType + "(version=" + timelineLayoutVersion + ") from " + basePath); + log.debug("Finished Loading Table of type {}(version={}) from {}", tableType, timelineLayoutVersion, basePath); if (loadActiveTimelineOnLoad) { - LOG.info("Loading Active commit timeline for " + basePath); + log.info("Loading Active commit timeline for {}", basePath); getActiveTimeline(); } } - /** - * For serializing and de-serializing. - * - * @deprecated - */ - public HoodieTableMetaClient() { - } - public String getIndexDefinitionPath() { return tableConfig.getRelativeIndexDefinitionPath() .map(definitionPath -> new StoragePath(basePath, definitionPath).toString()) @@ -250,7 +254,7 @@ public boolean buildIndexDefinition(HoodieIndexDefinition indexDefinition) { Option existingIndexOpt = indexMetadataOpt.get().getIndex(indexName); if (existingIndexOpt.isPresent()) { if (!existingIndexOpt.get().getSourceFields().equals(indexDefinition.getSourceFields())) { - LOG.info("List of columns to index is changing. Old value {}. New value {}", existingIndexOpt.get().getSourceFields(), + log.info("List of columns to index is changing. Old value {}. New value {}", existingIndexOpt.get().getSourceFields(), indexDefinition.getSourceFields()); indexMetadataOpt.get().getIndexDefinitions().put(indexName, indexDefinition); } else { @@ -386,35 +390,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws IOException { out.defaultWriteObject(); } - /** - * Returns base path of the table - */ - public StoragePath getBasePath() { - return basePath; // this invocation is cached - } - - /** - * @return Hoodie Table Type - */ - public HoodieTableType getTableType() { - return tableType; - } - - /** - * @return Meta path - */ - public StoragePath getMetaPath() { - return metaPath; - } - - public StoragePath getTimelinePath() { - return timelinePath; - } - - public HoodieTableFormat getTableFormat() { - return tableFormat; - } - /** * @return schema folder path */ @@ -492,21 +467,6 @@ public StoragePath getArchivePath() { return timelineHistoryPath; } - /** - * @return Table Config - */ - public HoodieTableConfig getTableConfig() { - return tableConfig; - } - - public TimelineLayoutVersion getTimelineLayoutVersion() { - return timelineLayoutVersion; - } - - public TimelineLayout getTimelineLayout() { - return timelineLayout; - } - public boolean isMetadataTable() { return HoodieTableMetadata.isMetadataTable(getBasePath()); } @@ -540,18 +500,10 @@ private static HoodieStorage getStorage(StoragePath path, consistencyGuard); } - public void setStorage(HoodieStorage storage) { - this.storage = storage; - } - public HoodieStorage getRawStorage() { return getStorage().getRawStorage(); } - public StorageConfiguration getStorageConf() { - return storageConf; - } - /** * Get the active instants as a timeline. * @@ -621,18 +573,6 @@ public String createNewInstantTime(boolean shouldLock) { return TimelineUtils.generateInstantTime(shouldLock, timeGenerator); } - public HoodieTimeGeneratorConfig getTimeGeneratorConfig() { - return timeGeneratorConfig; - } - - public ConsistencyGuardConfig getConsistencyGuardConfig() { - return consistencyGuardConfig; - } - - public FileSystemRetryConfig getFileSystemRetryConfig() { - return fileSystemRetryConfig; - } - /** * Get the archived commits as a timeline. This is costly operation, as all data from the archived files are read. * This should not be used, unless for historical debugging purposes. @@ -696,7 +636,7 @@ public static void createTableLayoutOnStorage(StorageConfiguration storageCon Properties props, Integer timelineLayout, boolean shouldCreateTableConfig) throws IOException { - LOG.info("Initializing {} as hoodie table", basePath); + log.info("Initializing {} as hoodie table", basePath); final HoodieStorage storage = HoodieStorageUtils.getStorage(basePath, storageConf); if (!storage.exists(basePath)) { storage.createDirectory(basePath); @@ -871,31 +811,6 @@ public List scanHoodieInstantsFromFileSystem(StoragePath timeline return instantStream.sorted().collect(Collectors.toList()); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - HoodieTableMetaClient that = (HoodieTableMetaClient) o; - return Objects.equals(basePath, that.basePath) && tableType == that.tableType; - } - - @Override - public int hashCode() { - return Objects.hash(basePath, tableType); - } - - @Override - public String toString() { - return "HoodieTableMetaClient{" + "basePath='" + basePath + '\'' - + ", metaPath='" + metaPath + '\'' - + ", tableType=" + tableType - + '}'; - } - public void initializeBootstrapDirsIfNotExists() throws IOException { initializeBootstrapDirsIfNotExists(basePath, getStorage()); } @@ -1037,6 +952,34 @@ public CommitMetadataSerDe getCommitMetadataSerDe() { return getTimelineLayout().getCommitMetadataSerDe(); } + // Not using Lombok @EqualsAndHashCode/@ToString here: this class is subclassed (e.g. HoodieTableMetaserverClient), + // and we rely on the runtime subtype - exact-class matching in equals() and the declaring-class behavior below. + // Lombok would switch equals() to instanceof, making a base instance compare equal to a subtype instance. + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HoodieTableMetaClient that = (HoodieTableMetaClient) o; + return Objects.equals(basePath, that.basePath) && tableType == that.tableType; + } + + @Override + public int hashCode() { + return Objects.hash(basePath, tableType); + } + + @Override + public String toString() { + return "HoodieTableMetaClient{" + "basePath='" + basePath + '\'' + + ", metaPath='" + metaPath + '\'' + + ", tableType=" + tableType + + '}'; + } + public static TableBuilder newTableBuilder() { return new TableBuilder(); } @@ -1044,6 +987,7 @@ public static TableBuilder newTableBuilder() { /** * Builder for {@link Properties}. */ + @NoArgsConstructor(access = AccessLevel.PACKAGE) public static class TableBuilder { private HoodieTableType tableType; @@ -1093,9 +1037,6 @@ public static class TableBuilder { */ private final Properties others = new Properties(); - TableBuilder() { - } - public TableBuilder setTableType(HoodieTableType tableType) { this.tableType = tableType; return this; @@ -1667,7 +1608,7 @@ public HoodieTableMetaClient initTable(StorageConfiguration storageConf, Stor HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(storageConf).setBasePath(basePath) .setMetaserverConfig(props) .build(); - LOG.info("Finished initializing Table of type {} from {}", metaClient.getTableConfig().getTableType(), basePath); + log.info("Finished initializing Table of type {} from {}", metaClient.getTableConfig().getTableType(), basePath); return metaClient; } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java index 945ed23666077..7ec8445809c9b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java @@ -22,6 +22,11 @@ import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.exception.HoodieException; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.experimental.Accessors; + import java.util.Arrays; import java.util.List; @@ -29,7 +34,10 @@ * Table's version that controls what version of writer/readers can actually read/write * to a given table. */ +@AllArgsConstructor +@Getter public enum HoodieTableVersion { + // < 0.6.0 versions ZERO(0, CollectionUtils.createImmutableList("0.3.0"), TimelineLayoutVersion.LAYOUT_VERSION_0), // 0.6.0 onwards @@ -51,26 +59,14 @@ public enum HoodieTableVersion { // 1.1 NINE(9, CollectionUtils.createImmutableList("1.1.0"), TimelineLayoutVersion.LAYOUT_VERSION_2); + @Accessors(fluent = true) // Required so that #versionCode() is generated instead of #getVersionCode() by Lombok private final int versionCode; + @Getter(AccessLevel.NONE) private final List releaseVersions; private final TimelineLayoutVersion timelineLayoutVersion; - HoodieTableVersion(int versionCode, List releaseVersions, TimelineLayoutVersion timelineLayoutVersion) { - this.versionCode = versionCode; - this.releaseVersions = releaseVersions; - this.timelineLayoutVersion = timelineLayoutVersion; - } - - public TimelineLayoutVersion getTimelineLayoutVersion() { - return timelineLayoutVersion; - } - - public int versionCode() { - return versionCode; - } - public static HoodieTableVersion current() { return NINE; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java index ee2cfbf0d362c..157f9d4c79ca3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java @@ -49,8 +49,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.util.Lazy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import javax.annotation.concurrent.ThreadSafe; @@ -67,11 +66,10 @@ /** * Helper class to read schema from data files and log files and to convert it between different formats. */ +@Slf4j @ThreadSafe public class TableSchemaResolver { - private static final Logger LOG = LoggerFactory.getLogger(TableSchemaResolver.class); - protected final HoodieTableMetaClient metaClient; /** @@ -254,11 +252,11 @@ private Option getTableParquetSchemaFromDataFile() { .map(writeStat -> new StoragePath(metaClient.getBasePath(), writeStat.getPath())); return Option.of(fetchSchemaFromFiles(filePaths)); } else { - LOG.debug("Could not find any data file written for commit, so could not get schema for table {}", metaClient.getBasePath()); + log.debug("Could not find any data file written for commit, so could not get schema for table {}", metaClient.getBasePath()); return Option.empty(); } default: - LOG.error("Unknown table type {}", metaClient.getTableType()); + log.error("Unknown table type {}", metaClient.getTableType()); throw new InvalidTableException(metaClient.getBasePath().toString()); } } @@ -373,7 +371,7 @@ public boolean hasOperationField() { HoodieSchema tableSchema = getTableSchemaFromDataFile(); return tableSchema.getField(HoodieRecord.OPERATION_METADATA_FIELD).isPresent(); } catch (Exception e) { - LOG.info("Failed to read operation field from schema ({})", e.getMessage()); + log.info("Failed to read operation field from schema ({})", e.getMessage()); return false; } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java index 612837c4dc4df..beb9b56bc66ad 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java @@ -219,9 +219,9 @@ private void initInstantAndCommitMetadata() { try { Set requiredActions = new HashSet<>(Arrays.asList(COMMIT_ACTION, DELTA_COMMIT_ACTION, REPLACE_COMMIT_ACTION, CLUSTERING_ACTION)); HoodieActiveTimeline activeTimeLine = metaClient.getActiveTimeline(); - if (instantRange.getStartInstant().isPresent() && !metaClient.getArchivedTimeline().empty() - && InstantComparison.compareTimestamps(metaClient.getArchivedTimeline().lastInstant().get().requestedTime(), InstantComparison.GREATER_THAN, instantRange.getStartInstant().get())) { - throw new HoodieException("Start instant time " + instantRange.getStartInstant().get() + if (instantRange.getStartInstantOpt().isPresent() && !metaClient.getArchivedTimeline().empty() + && InstantComparison.compareTimestamps(metaClient.getArchivedTimeline().lastInstant().get().requestedTime(), InstantComparison.GREATER_THAN, instantRange.getStartInstantOpt().get())) { + throw new HoodieException("Start instant time " + instantRange.getStartInstantOpt().get() + " for CDC query has to be in the active timeline. Beginning of active timeline " + activeTimeLine.firstInstant().get().requestedTime()); } this.commits = activeTimeLine.getInstantsAsStream() diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCFileSplit.java b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCFileSplit.java index c33df8414bd2e..24fb94356a0d3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCFileSplit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCFileSplit.java @@ -22,6 +22,8 @@ import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.util.Option; +import lombok.Getter; + import java.io.Serializable; import java.util.Collection; import java.util.Collections; @@ -45,7 +47,9 @@ * For `cdcInferCase` = {@link HoodieCDCInferenceCase#REPLACE_COMMIT}, `cdcFile` is null, * `beforeFileSlice` is the current version of the file slice. */ +@Getter public class HoodieCDCFileSplit implements Serializable, Comparable { + /** * The instant time at which the changes happened. */ @@ -103,26 +107,6 @@ public HoodieCDCFileSplit( this.afterFileSlice = afterFileSlice; } - public String getInstant() { - return this.instant; - } - - public HoodieCDCInferenceCase getCdcInferCase() { - return this.cdcInferCase; - } - - public List getCdcFiles() { - return this.cdcFiles; - } - - public Option getBeforeFileSlice() { - return this.beforeFileSlice; - } - - public Option getAfterFileSlice() { - return this.afterFileSlice; - } - @Override public int compareTo(HoodieCDCFileSplit o) { int cmpResult = this.instant.compareTo(o.instant); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCOperation.java b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCOperation.java index 90540bc05a69b..2cb0f19687e0a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCOperation.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCOperation.java @@ -20,9 +20,15 @@ import org.apache.hudi.exception.HoodieNotSupportedException; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * Enumeration of change log operation. */ +@AllArgsConstructor(access = AccessLevel.PACKAGE) +@Getter public enum HoodieCDCOperation { INSERT("i"), UPDATE("u"), @@ -30,14 +36,6 @@ public enum HoodieCDCOperation { private final String value; - HoodieCDCOperation(String value) { - this.value = value; - } - - public String getValue() { - return this.value; - } - public static HoodieCDCOperation parse(String value) { switch (value) { case "i": diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/Checkpoint.java b/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/Checkpoint.java index 67248ba4adcdb..eea1c32dd373a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/Checkpoint.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/checkpoint/Checkpoint.java @@ -19,6 +19,9 @@ package org.apache.hudi.common.table.checkpoint; +import lombok.AccessLevel; +import lombok.Getter; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -27,13 +30,16 @@ /** * Class for representing checkpoint */ +@Getter public abstract class Checkpoint implements Serializable { + public static final String CHECKPOINT_IGNORE_KEY = "deltastreamer.checkpoint.ignore_key"; protected String checkpointKey; protected String checkpointResetKey; protected String checkpointIgnoreKey; // These are extra props to be written to the commit metadata + @Getter(AccessLevel.NONE) protected Map extraProps = new HashMap<>(); public Checkpoint setCheckpointKey(String newKey) { @@ -41,21 +47,12 @@ public Checkpoint setCheckpointKey(String newKey) { return this; } - public String getCheckpointKey() { - return checkpointKey; - } - - public String getCheckpointResetKey() { - return checkpointResetKey; - } - - public String getCheckpointIgnoreKey() { - return checkpointIgnoreKey; - } - public abstract Map getCheckpointCommitMetadata(String overrideResetKey, String overrideIgnoreKey); + // Not using Lombok @EqualsAndHashCode/@ToString here: this class is subclassed, and we rely on + // the runtime subtype - exact-class matching in equals() and getClass().getSimpleName() in toString(). + // Lombok would bake in the declaring class (Checkpoint) and switch equals() to instanceof. @Override public int hashCode() { return Objects.hashCode(checkpointKey); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java index b1eb9e0e2c985..49c17eda75c83 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java @@ -48,8 +48,10 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayDeque; @@ -87,10 +89,9 @@ *

    * This results in two I/O passes over the log file. */ +@Slf4j public abstract class AbstractHoodieLogRecordScanner { - private static final Logger LOG = LoggerFactory.getLogger(AbstractHoodieLogRecordScanner.class); - // Reader schema for the records protected final HoodieSchema readerSchema; // Latest valid instant time @@ -98,14 +99,17 @@ public abstract class AbstractHoodieLogRecordScanner { private final String latestInstantTime; protected final HoodieTableMetaClient hoodieTableMetaClient; // Merge strategy to use when combining records from log + @Getter(AccessLevel.PROTECTED) private final String payloadClassFQN; // Record's key/partition-path fields private final String recordKeyField; private final Option partitionPathFieldOpt; // Partition name override + @Getter private final Option partitionNameOverrideOpt; // Stateless component for merging records protected final HoodieRecordMerger recordMerger; + @Getter(AccessLevel.PROTECTED) private final TypedProperties payloadProps; // Log File Paths protected final List logFilePaths; @@ -117,6 +121,7 @@ public abstract class AbstractHoodieLogRecordScanner { // optional instant range for incremental block filtering private final Option instantRange; // Read the operation metadata field from the avro record + @Getter private final boolean withOperationField; private final HoodieStorage storage; // Total log files read - for metrics @@ -132,16 +137,19 @@ public abstract class AbstractHoodieLogRecordScanner { // Total number of corrupt blocks written across all log files private AtomicLong totalCorruptBlocks = new AtomicLong(0); // Store the last instant log blocks (needed to implement rollback) + @Getter private Deque currentInstantLogBlocks = new ArrayDeque<>(); // Enables full scan of log records protected final boolean forceFullScan; // Progress + @Getter private float progress = 0.0f; // Populate meta fields for the records private final boolean populateMetaFields; // Record type read from log block protected final HoodieRecordType recordType; // Collect all the block instants after scanning all the log files. + @Getter private final List validBlockInstants = new ArrayList<>(); // table version for compatibility private final HoodieTableVersion tableVersion; @@ -301,7 +309,7 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole */ while (logFormatReaderWrapper.hasNext()) { HoodieLogFile logFile = logFormatReaderWrapper.getLogFile(); - LOG.info("Scanning log file {}", logFile); + log.info("Scanning log file {}", logFile); scannedLogFiles.add(logFile); totalLogFiles.set(scannedLogFiles.size()); // Use the HoodieLogFileReader to iterate through the blocks in the log file @@ -310,7 +318,7 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole totalLogBlocks.incrementAndGet(); // Ignore the corrupt blocks. No further handling is required for them. if (logBlock.getBlockType().equals(CORRUPT_BLOCK)) { - LOG.info("Found a corrupt block in {}", logFile.getPath()); + log.info("Found a corrupt block in {}", logFile.getPath()); totalCorruptBlocks.incrementAndGet(); continue; } @@ -347,7 +355,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA instantToBlocksMap.put(instantTime, logBlocksList); break; case COMMAND_BLOCK: - LOG.info("Reading a command block from file {}", logFile.getPath()); + log.info("Reading a command block from file {}", logFile.getPath()); // This is a command block - take appropriate action based on the command HoodieCommandBlock commandBlock = (HoodieCommandBlock) logBlock; @@ -367,8 +375,8 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA } } - if (LOG.isDebugEnabled()) { - LOG.debug("Ordered instant times seen {}", orderedInstantsList); + if (log.isDebugEnabled()) { + log.debug("Ordered instant times seen {}", orderedInstantsList); } int numBlocksRolledBack = 0; @@ -424,24 +432,24 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA validBlockInstants.add(compactedFinalInstantTime); } } - LOG.info("Number of applied rollback blocks {}", numBlocksRolledBack); + log.info("Number of applied rollback blocks {}", numBlocksRolledBack); - if (LOG.isDebugEnabled()) { - LOG.info("Final view of the Block time to compactionBlockMap {}", blockTimeToCompactionBlockTimeMap); + if (log.isDebugEnabled()) { + log.info("Final view of the Block time to compactionBlockMap {}", blockTimeToCompactionBlockTimeMap); } // merge the last read block when all the blocks are done reading if (!currentInstantLogBlocks.isEmpty() && !skipProcessingBlocks) { - LOG.info("Merging the final data blocks"); + log.info("Merging the final data blocks"); processQueuedBlocksForInstant(currentInstantLogBlocks, scannedLogFiles.size(), keySpecOpt); } // Done progress = 1.0f; } catch (IOException e) { - LOG.error("Got IOException when reading log file", e); + log.error("Got IOException when reading log file", e); throw new HoodieIOException("IOException when reading log file ", e); } catch (Exception e) { - LOG.error("Got exception when reading log file", e); + log.error("Got exception when reading log file", e); throw new HoodieException("Exception when reading log file ", e); } finally { try { @@ -450,7 +458,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA } } catch (IOException ioe) { // Eat exception as we do not want to mask the original exception that can happen - LOG.error("Unable to close log format reader", ioe); + log.error("Unable to close log format reader", ioe); } } } @@ -506,7 +514,7 @@ private void processDataBlock(HoodieDataBlock dataBlock, Option keySpec private void processQueuedBlocksForInstant(Deque logBlocks, int numLogFilesSeen, Option keySpecOpt) throws Exception { while (!logBlocks.isEmpty()) { - LOG.info("Number of remaining logblocks to merge {}", logBlocks.size()); + log.info("Number of remaining logblocks to merge {}", logBlocks.size()); // poll the element at the bottom of the stack since that's the order it was inserted HoodieLogBlock lastBlock = logBlocks.pollLast(); switch (lastBlock.getBlockType()) { @@ -519,7 +527,7 @@ private void processQueuedBlocksForInstant(Deque logBlocks, int Arrays.stream(((HoodieDeleteBlock) lastBlock).getRecordsToDelete()).forEach(this::processNextDeletedRecord); break; case CORRUPT_BLOCK: - LOG.warn("Found a corrupt block which was not rolled back"); + log.warn("Found a corrupt block which was not rolled back"); break; default: break; @@ -535,13 +543,6 @@ private boolean shouldLookupRecords() { return !forceFullScan; } - /** - * Return progress of scanning as a float between 0.0 to 1.0. - */ - public float getProgress() { - return progress; - } - public long getTotalLogFiles() { return totalLogFiles.get(); } @@ -554,14 +555,6 @@ public long getTotalLogBlocks() { return totalLogBlocks.get(); } - protected String getPayloadClassFQN() { - return payloadClassFQN; - } - - public Option getPartitionNameOverride() { - return partitionNameOverrideOpt; - } - public long getTotalRollbacks() { return totalRollbacks.get(); } @@ -570,14 +563,6 @@ public long getTotalCorruptBlocks() { return totalCorruptBlocks.get(); } - public boolean isWithOperationField() { - return withOperationField; - } - - protected TypedProperties getPayloadProps() { - return payloadProps; - } - /** * Key specification with a list of column names. */ @@ -595,16 +580,11 @@ static KeySpec prefixKeySpec(List keyPrefixes) { } } + @AllArgsConstructor + @Getter private static class FullKeySpec implements KeySpec { - private final List keys; - private FullKeySpec(List keys) { - this.keys = keys; - } - @Override - public List getKeys() { - return keys; - } + private final List keys; @Override public boolean isFullKey() { @@ -612,12 +592,10 @@ public boolean isFullKey() { } } + @AllArgsConstructor private static class PrefixKeySpec implements KeySpec { - private final List keysPrefixes; - private PrefixKeySpec(List keysPrefixes) { - this.keysPrefixes = keysPrefixes; - } + private final List keysPrefixes; @Override public List getKeys() { @@ -630,14 +608,6 @@ public boolean isFullKey() { } } - public Deque getCurrentInstantLogBlocks() { - return currentInstantLogBlocks; - } - - public List getValidBlockInstants() { - return validBlockInstants; - } - private Pair, HoodieSchema> getRecordsIterator( HoodieDataBlock dataBlock, Option keySpecOpt) throws IOException { ClosableIterator blockRecordsIterator; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java index 332a9c920c6ef..159662f73bb97 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java @@ -43,8 +43,9 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayDeque; @@ -74,9 +75,9 @@ * * @param type of engine-specific record representation. */ +@Slf4j public abstract class BaseHoodieLogRecordReader { - private static final Logger LOG = LoggerFactory.getLogger(BaseHoodieLogRecordReader.class); public static final String LOG_BLOCK_FULL_READ_DURATION_IN_MILLIS = "logBlockFullReadDurationInMillis"; public static final String BLOCK_SIZE_IN_BYTES = "blockSizeInBytes"; public static final String TOTAL_RECORDS_PRESENT_IN_LOG_BLOCK = "totalRecordsPresentInLogBlock"; @@ -89,13 +90,16 @@ public abstract class BaseHoodieLogRecordReader { protected final HoodieReaderContext readerContext; protected final HoodieTableMetaClient hoodieTableMetaClient; // Merge strategy to use when combining records from log + @Getter(AccessLevel.PROTECTED) private final String payloadClassFQN; // Record's key/partition-path fields private final String recordKeyField; // Partition name override + @Getter private final Option partitionNameOverrideOpt; // Ordering fields protected final String orderingFields; + @Getter(AccessLevel.PROTECTED) private final TypedProperties payloadProps; // Log File Paths protected final List logFiles; @@ -107,6 +111,7 @@ public abstract class BaseHoodieLogRecordReader { // optional instant range for incremental block filtering private final Option instantRange; // Read the operation metadata field from the avro record + @Getter private final boolean withOperationField; // FileSystem private final HoodieStorage storage; @@ -129,15 +134,18 @@ public abstract class BaseHoodieLogRecordReader { // Scan duration in milliseconds private AtomicLong blocksScanDuration = new AtomicLong(0); // Store the last instant log blocks (needed to implement rollback) + @Getter private Deque currentInstantLogBlocks = new ArrayDeque<>(); // Enables full scan of log records protected final boolean forceFullScan; // Progress + @Getter private float progress = 0.0f; - // Record type read from log block // Collect all the block instants after scanning all the log files. + @Getter private final List validBlockInstants = new ArrayList<>(); // Block-level scan stats for processed data blocks. + @Getter private List> blocksStats = new ArrayList<>(); protected HoodieFileGroupRecordBuffer recordBuffer; // Allows to consider inflight instants while merging log records @@ -271,7 +279,7 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole */ while (logFormatReaderWrapper.hasNext()) { HoodieLogFile logFile = logFormatReaderWrapper.getLogFile(); - LOG.debug("Scanning log file {}", logFile); + log.debug("Scanning log file {}", logFile); scannedLogFiles.add(logFile); totalLogFiles.set(scannedLogFiles.size()); // Use the HoodieLogFileReader to iterate through the blocks in the log file @@ -279,11 +287,11 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole logBlock.getBlockContentLocation() .map(contentLocation -> totalLogBlocksSize.addAndGet(contentLocation.getBlockSize())); final String instantTime = logBlock.getLogBlockHeader().get(INSTANT_TIME); - LOG.debug("Scanning log block with instant time {}", instantTime); + log.debug("Scanning log block with instant time {}", instantTime); totalLogBlocks.incrementAndGet(); // Ignore the corrupt blocks. No further handling is required for them. if (logBlock.getBlockType().equals(CORRUPT_BLOCK)) { - LOG.debug("Found a corrupt block in {}", logFile.getPath()); + log.debug("Found a corrupt block in {}", logFile.getPath()); totalCorruptBlocks.incrementAndGet(); continue; } @@ -314,7 +322,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA case AVRO_DATA_BLOCK: case PARQUET_DATA_BLOCK: case DELETE_BLOCK: - LOG.debug("Reading a {} block with instant time {}", + log.debug("Reading a {} block with instant time {}", logBlock.getBlockType() == HoodieLogBlock.HoodieLogBlockType.DELETE_BLOCK ? "delete" : "data", instantTime); List logBlocksList = instantToBlocksMap.getOrDefault(instantTime, new ArrayList<>()); @@ -326,7 +334,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA instantToBlocksMap.put(instantTime, logBlocksList); break; case COMMAND_BLOCK: - LOG.debug("Reading a command block from file {}", logFile.getPath()); + log.debug("Reading a command block from file {}", logFile.getPath()); // This is a command block - take appropriate action based on the command HoodieCommandBlock commandBlock = (HoodieCommandBlock) logBlock; @@ -341,10 +349,10 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA if (rolledBackBlocks != null) { numBlocksRolledBack += rolledBackBlocks.size(); } - LOG.debug("Reading a rollback block with instant {} and target instant {}", + log.debug("Reading a rollback block with instant {} and target instant {}", instantTime, targetInstantForCommandBlock); } else { - LOG.error("Reading a command block with instant {} whose operation is not supported", instantTime); + log.error("Reading a command block with instant {} whose operation is not supported", instantTime); throw new UnsupportedOperationException("Command type not yet supported."); } break; @@ -353,8 +361,8 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA } } - LOG.info("Ordered instant times seen {}", orderedInstantsList); - LOG.info("Targeted instants that are rolled back are {}", targetRollbackInstants); + log.info("Ordered instant times seen {}", orderedInstantsList); + log.info("Targeted instants that are rolled back are {}", targetRollbackInstants); // All the block's instants time that are added to the queue are collected in this set. Set instantTimesIncluded = new HashSet<>(); @@ -377,7 +385,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA // For compacted blocks COMPACTED_BLOCK_TIMES entry is present under its headers. if (firstBlock.getLogBlockHeader().containsKey(COMPACTED_BLOCK_TIMES)) { - LOG.debug("For instant time {}, compacted block instants are {}", + log.debug("For instant time {}, compacted block instants are {}", instantTime, firstBlock.getLogBlockHeader().get(COMPACTED_BLOCK_TIMES)); // When compacted blocks are seen update the blockTimeToCompactionBlockTimeMap. Arrays.stream(firstBlock.getLogBlockHeader().get(COMPACTED_BLOCK_TIMES).split(",")) @@ -410,20 +418,20 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA } } Collections.reverse(validBlockInstants); - LOG.debug("Number of applied rollback blocks {}", numBlocksRolledBack); - LOG.info("Total valid instants found are {}. Instants are {}", validBlockInstants.size(), validBlockInstants); + log.debug("Number of applied rollback blocks {}", numBlocksRolledBack); + log.info("Total valid instants found are {}. Instants are {}", validBlockInstants.size(), validBlockInstants); if (ignoredBlockCount > 0) { - LOG.info("Ignored {} log blocks from {} instants not in the range: {}", ignoredBlockCount, ignoredInstants.size(), ignoredInstants); + log.info("Ignored {} log blocks from {} instants not in the range: {}", ignoredBlockCount, ignoredInstants.size(), ignoredInstants); } - if (LOG.isDebugEnabled()) { - LOG.debug("Final view of the Block time to compactionBlockMap {}", blockTimeToCompactionBlockTimeMap); + if (log.isDebugEnabled()) { + log.debug("Final view of the Block time to compactionBlockMap {}", blockTimeToCompactionBlockTimeMap); } totalValidLogBlocks.set(currentInstantLogBlocks.size()); blocksScanDuration.set(scanTimer.endTimer()); // merge the last read block when all the blocks are done reading if (!currentInstantLogBlocks.isEmpty() && !skipProcessingBlocks) { - LOG.debug("Merging the final data blocks"); + log.debug("Merging the final data blocks"); processQueuedBlocksForInstant(currentInstantLogBlocks, scannedLogFiles.size(), keySpecOpt); } // Done @@ -432,10 +440,10 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA totalLogRecords.set(recordBuffer.getTotalLogRecords()); } } catch (IOException e) { - LOG.error("Got IOException when reading log file", e); + log.error("Got IOException when reading log file", e); throw new HoodieIOException("IOException when reading log file ", e); } catch (Exception e) { - LOG.error("Got exception when reading log file", e); + log.error("Got exception when reading log file", e); throw new HoodieException("Exception when reading log file ", e); } finally { try { @@ -444,18 +452,18 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA } } catch (IOException ioe) { // Eat exception as we do not want to mask the original exception that can happen - LOG.error("Unable to close log format reader", ioe); + log.error("Unable to close log format reader", ioe); } if (!logFiles.isEmpty()) { try { StoragePath path = logFiles.get(0).getPath(); - LOG.info("Finished scanning log files. FileId: {}, LogFileInstantTime: {}, " + log.info("Finished scanning log files. FileId: {}, LogFileInstantTime: {}, " + "Total log files: {}, Total log blocks: {}, Total rollbacks: {}, Total corrupt blocks: {}", FSUtils.getFileIdFromLogPath(path), FSUtils.getDeltaCommitTimeFromLogPath(path), totalLogFiles.get(), totalLogBlocks.get(), totalRollbacks.get(), totalCorruptBlocks.get()); } catch (Exception e) { - LOG.warn("Could not extract fileId from log path", e); - LOG.info("Finished scanning log files. " + log.warn("Could not extract fileId from log path", e); + log.info("Finished scanning log files. " + "Total log files: {}, Total log blocks: {}, Total rollbacks: {}, Total corrupt blocks: {}", totalLogFiles.get(), totalLogBlocks.get(), totalRollbacks.get(), totalCorruptBlocks.get()); } @@ -469,7 +477,7 @@ && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), GREATER_THA private void processQueuedBlocksForInstant(Deque logBlocks, int numLogFilesSeen, Option keySpecOpt) throws Exception { while (!logBlocks.isEmpty()) { - LOG.debug("Number of remaining logblocks to merge {}", logBlocks.size()); + log.debug("Number of remaining logblocks to merge {}", logBlocks.size()); // poll the element at the bottom of the stack since that's the order it was inserted HoodieLogBlock lastBlock = logBlocks.pollLast(); switch (lastBlock.getBlockType()) { @@ -482,7 +490,7 @@ private void processQueuedBlocksForInstant(Deque logBlocks, int recordBuffer.processDeleteBlock((HoodieDeleteBlock) lastBlock); break; case CORRUPT_BLOCK: - LOG.warn("Found a corrupt block which was not rolled back"); + log.warn("Found a corrupt block which was not rolled back"); break; default: break; @@ -494,7 +502,7 @@ private void processQueuedBlocksForInstant(Deque logBlocks, int private void processDataBlock(HoodieDataBlock dataBlock, Option keySpecOpt) throws IOException { String blockInstantTime = dataBlock.getLogBlockHeader().get(INSTANT_TIME); - LOG.debug("Processing log block with instant time {}", blockInstantTime); + log.debug("Processing log block with instant time {}", blockInstantTime); long totalLogRecordsBefore = recordBuffer != null ? recordBuffer.getTotalLogRecords() : 0L; HoodieTimer blockReadTimer = HoodieTimer.start(); recordBuffer.processDataBlock(dataBlock, keySpecOpt); @@ -507,7 +515,7 @@ private void processDataBlock(HoodieDataBlock dataBlock, Option keySpec .map(contentLocation -> blockReadMetrics.put(BLOCK_SIZE_IN_BYTES, contentLocation.getBlockSize())); blockReadMetrics.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME.toString(), blockInstantTime); blocksStats.add(blockReadMetrics); - LOG.debug("For log block, scan metrics are {}", blockReadMetrics); + log.debug("For log block, scan metrics are {}", blockReadMetrics); } private boolean shouldLookupRecords() { @@ -516,13 +524,6 @@ private boolean shouldLookupRecords() { return !forceFullScan; } - /** - * Return progress of scanning as a float between 0.0 to 1.0. - */ - public float getProgress() { - return progress; - } - public long getTotalLogFiles() { return totalLogFiles.get(); } @@ -543,22 +544,10 @@ public long getTotalValidLogBlocks() { return totalValidLogBlocks.get(); } - public List> getBlocksStats() { - return blocksStats; - } - public long getBlocksScanDuration() { return blocksScanDuration.get(); } - protected String getPayloadClassFQN() { - return payloadClassFQN; - } - - public Option getPartitionNameOverride() { - return partitionNameOverrideOpt; - } - public long getTotalRollbacks() { return totalRollbacks.get(); } @@ -567,22 +556,6 @@ public long getTotalCorruptBlocks() { return totalCorruptBlocks.get(); } - public boolean isWithOperationField() { - return withOperationField; - } - - protected TypedProperties getPayloadProps() { - return payloadProps; - } - - public Deque getCurrentInstantLogBlocks() { - return currentInstantLogBlocks; - } - - public List getValidBlockInstants() { - return validBlockInstants; - } - /** * Builder used to build {@code AbstractHoodieLogRecordScanner}. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/FullKeySpec.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/FullKeySpec.java index ede7918649b4e..a186abd73e9a3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/FullKeySpec.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/FullKeySpec.java @@ -19,6 +19,9 @@ package org.apache.hudi.common.table.log; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.util.List; /** @@ -26,17 +29,11 @@ * That is, the comparison between a record key and an element * of the set is {@link String#equals}. */ +@AllArgsConstructor +@Getter public class FullKeySpec implements KeySpec { - private final List keys; - - public FullKeySpec(List keys) { - this.keys = keys; - } - @Override - public List getKeys() { - return keys; - } + private final List keys; @Override public boolean isFullKey() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java index 2c1d91e7b7c12..03ae4a2c4c07d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java @@ -44,8 +44,8 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StorageSchemes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import javax.annotation.Nullable; @@ -63,14 +63,15 @@ * Scans a log file and provides block level iterator on the log file Loads the entire block contents in memory Can emit * either a DataBlock, CommandBlock, DeleteBlock or CorruptBlock (if one is found). */ +@Slf4j public class HoodieLogFileReader implements HoodieLogFormat.Reader { public static final int DEFAULT_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MB - private static final int BLOCK_SCAN_READ_BUFFER_SIZE = 1024 * 1024; // 1 MB - private static final Logger LOG = LoggerFactory.getLogger(HoodieLogFileReader.class); + private static final int BLOCK_SCAN_READ_BUFFER_SIZE = 1024 * 1024; private static final String REVERSE_LOG_READER_HAS_NOT_BEEN_ENABLED = "Reverse log reader has not been enabled"; private final HoodieStorage storage; + @Getter private final HoodieLogFile logFile; private final int bufferSize; private final byte[] magicBuffer = new byte[6]; @@ -118,11 +119,6 @@ public HoodieLogFileReader(HoodieStorage storage, HoodieLogFile logFile, HoodieS } } - @Override - public HoodieLogFile getLogFile() { - return logFile; - } - // TODO : convert content and block length to long by using ByteBuffer, raw byte [] allows // for max of Integer size private HoodieLogBlock readBlock() throws IOException { @@ -244,12 +240,12 @@ private HoodieLogBlockType tryReadBlockType(HoodieLogFormat.LogFormatVersion blo } private HoodieLogBlock createCorruptBlock(long blockStartPos) throws IOException { - LOG.info("Log {} has a corrupted block at {}", logFile, blockStartPos); + log.info("Log {} has a corrupted block at {}", logFile, blockStartPos); inputStream.seek(blockStartPos); long nextBlockOffset = scanForNextAvailableBlockOffset(); // Rewind to the initial start and read corrupted bytes till the nextBlockOffset inputStream.seek(blockStartPos); - LOG.info("Next available block in {} starts at {}", logFile, nextBlockOffset); + log.info("Next available block in {} starts at {}", logFile, nextBlockOffset); int corruptedBlockSize = (int) (nextBlockOffset - blockStartPos); long contentPosition = inputStream.getPos(); Option corruptedBytes = HoodieLogBlock.tryReadContent(inputStream, corruptedBlockSize, true); @@ -276,7 +272,7 @@ private boolean isBlockCorrupted(int blocksize) throws IOException { // So we have to shorten the footer block size by the size of magic hash blockSizeFromFooter = inputStream.readLong() - magicBuffer.length; } catch (EOFException e) { - LOG.info("Found corrupted block in file {} with block size({}) running past EOF", logFile, blocksize); + log.info("Found corrupted block in file {} with block size({}) running past EOF", logFile, blocksize); // this is corrupt // This seek is required because contract of seek() is different for naked DFSInputStream vs BufferedFSInputStream // release-3.1.0-RC1/DFSInputStream.java#L1455 @@ -286,7 +282,7 @@ private boolean isBlockCorrupted(int blocksize) throws IOException { } if (blocksize != blockSizeFromFooter) { - LOG.info("Found corrupted block in file {}. Header block size({}) did not match the footer block size({})", logFile, blocksize, blockSizeFromFooter); + log.info("Found corrupted block in file {}. Header block size({}) did not match the footer block size({})", logFile, blocksize, blockSizeFromFooter); inputStream.seek(currentPos); return true; } @@ -297,7 +293,7 @@ private boolean isBlockCorrupted(int blocksize) throws IOException { return false; } catch (CorruptedLogFileException e) { // This is a corrupted block - LOG.info("Found corrupted block in file {}. No magic hash found right after footer block size entry", logFile); + log.info("Found corrupted block in file {}. No magic hash found right after footer block size entry", logFile); return true; } finally { inputStream.seek(currentPos); @@ -330,7 +326,7 @@ private long scanForNextAvailableBlockOffset() throws IOException { @Override public void close() throws IOException { if (!closed) { - LOG.info("Closing Log file reader {}", logFile.getFileName()); + log.info("Closing Log file reader {}", logFile.getFileName()); if (null != this.inputStream) { this.inputStream.close(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index a675e0d9da895..3877a682240dd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -29,6 +29,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -246,18 +248,12 @@ static HoodieLogFormat.Reader newReader(HoodieStorage storage, HoodieLogFile log * A set of feature flags associated with a log format. Versions are changed when the log format changes. TODO(na) - * Implement policies around major/minor versions */ + @AllArgsConstructor(access = AccessLevel.PACKAGE) + @Getter abstract class LogFormatVersion { private final int version; - LogFormatVersion(int version) { - this.version = version; - } - - public int getVersion() { - return version; - } - public abstract boolean hasMagicHeader(); public abstract boolean hasContent(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java index 77c3e78fcc328..1c342fecf39c5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java @@ -25,8 +25,7 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.storage.HoodieStorage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; @@ -34,6 +33,7 @@ /** * Hoodie log format reader. */ +@Slf4j public class HoodieLogFormatReader implements HoodieLogFormat.Reader { private final List logFiles; @@ -45,8 +45,6 @@ public class HoodieLogFormatReader implements HoodieLogFormat.Reader { private final boolean enableInlineReading; private final int bufferSize; - private static final Logger LOG = LoggerFactory.getLogger(HoodieLogFormatReader.class); - HoodieLogFormatReader(HoodieStorage storage, List logFiles, HoodieSchema readerSchema, boolean reverseLogReader, int bufferSize, boolean enableRecordLookups, String recordKeyField, InternalSchema internalSchema) throws IOException { @@ -91,7 +89,7 @@ public boolean hasNext() { } catch (IOException io) { throw new HoodieIOException("unable to initialize read with log file ", io); } - LOG.debug("Moving to the next reader for logfile {}", currentReader.getLogFile()); + log.debug("Moving to the next reader for logfile {}", currentReader.getLogFile()); return hasNext(); } return false; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java index b3f1eeaa988ec..1ee8e7560d054 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java @@ -34,8 +34,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Closeable; import java.io.Serializable; @@ -52,9 +52,11 @@ * * @param type of engine-specific record representation. */ +@Getter +@Slf4j public class HoodieMergedLogRecordReader extends BaseHoodieLogRecordReader implements Iterable>, Closeable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieMergedLogRecordReader.class); + // A timer for calculating elapsed time in millis public final HoodieTimer timer = HoodieTimer.create(); // count of merged records in log @@ -102,8 +104,8 @@ private void performScan() { this.totalTimeTakenToReadAndMergeBlocks = timer.endTimer(); this.numMergedRecordsInLog = recordBuffer.size(); - LOG.info("Number of log files scanned => {}", logFiles.size()); - LOG.info("Number of entries in Map => {}", recordBuffer.size()); + log.info("Number of log files scanned => {}", logFiles.size()); + log.info("Number of entries in Map => {}", recordBuffer.size()); } static Option createKeySpec(Option filter) { @@ -134,10 +136,6 @@ public Map> getRecords() { return recordBuffer.getLogRecords(); } - public long getNumMergedRecordsInLog() { - return numMergedRecordsInLog; - } - /** * Returns the builder for {@code HoodieMergedLogRecordReader}. */ @@ -145,10 +143,6 @@ public static Builder newBuilder() { return new Builder<>(); } - public long getTotalTimeTakenToReadAndMergeBlocks() { - return totalTimeTakenToReadAndMergeBlocks; - } - @Override public void close() { // No op. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index 318533859746d..f7521fc207c5c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -47,8 +47,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import javax.annotation.concurrent.NotThreadSafe; @@ -81,10 +81,10 @@ * This results in two I/O passes over the log file. */ @NotThreadSafe +@Slf4j public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordScanner implements Iterable, Closeable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieMergedLogRecordScanner.class); // A timer for calculating elapsed time in millis public final HoodieTimer timer = HoodieTimer.create(); // Map of compacted/merged records @@ -92,9 +92,11 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordScanner // Set of already scanned prefixes allowing us to avoid scanning same prefixes again private final Set scannedPrefixes; // count of merged records in log + @Getter private long numMergedRecordsInLog; private final long maxMemorySizeInBytes; // Stores the total time taken to perform reading and merging of log blocks + @Getter private long totalTimeTakenToReadAndMergeBlocks; private final String[] orderingFields; private final DeleteContext deleteContext; @@ -220,8 +222,8 @@ private void performScan() { this.totalTimeTakenToReadAndMergeBlocks = timer.endTimer(); this.numMergedRecordsInLog = records.size(); - if (LOG.isInfoEnabled()) { - LOG.info("Scanned {} log files with stats: MaxMemoryInBytes => {}, MemoryBasedMap => {} entries, {} total bytes, DiskBasedMap => {} entries, {} total bytes", + if (log.isInfoEnabled()) { + log.info("Scanned {} log files with stats: MaxMemoryInBytes => {}, MemoryBasedMap => {} entries, {} total bytes, DiskBasedMap => {} entries, {} total bytes", logFilePaths.size(), maxMemorySizeInBytes, records.getInMemoryMapNumEntries(), records.getCurrentInMemoryMapSize(), records.getDiskBasedMapNumEntries(), records.getSizeOfFileOnDiskInBytes()); } @@ -240,10 +242,6 @@ public HoodieRecord.HoodieRecordType getRecordType() { return recordMerger.getRecordType(); } - public long getNumMergedRecordsInLog() { - return numMergedRecordsInLog; - } - /** * Returns the builder for {@code HoodieMergedLogRecordScanner}. */ @@ -313,10 +311,6 @@ protected void processNextDeletedRecord(DeleteRecord deleteRecord) { } } - public long getTotalTimeTakenToReadAndMergeBlocks() { - return totalTimeTakenToReadAndMergeBlocks; - } - @Override public void close() { if (records != null) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/InstantRange.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/InstantRange.java index 9dd56cc66182c..eda285d478d87 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/InstantRange.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/InstantRange.java @@ -21,6 +21,10 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; + import java.io.Serializable; import java.util.Arrays; import java.util.Collections; @@ -37,6 +41,7 @@ /** * An instant range used for incremental reader filtering. */ +@Getter public abstract class InstantRange implements Serializable { private static final long serialVersionUID = 1L; @@ -55,14 +60,6 @@ public static Builder builder() { return new Builder(); } - public Option getStartInstant() { - return startInstantOpt; - } - - public Option getEndInstant() { - return endInstantOpt; - } - public abstract boolean isInRange(String instant); public abstract RangeType getRangeType(); @@ -248,6 +245,7 @@ public RangeType getRangeType() { /** * Builder for {@link InstantRange}. */ + @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Builder { private String startInstant; private String endInstant; @@ -256,9 +254,6 @@ public static class Builder { private Set explicitInstants; private List instantRanges; - private Builder() { - } - public Builder startInstant(String startInstant) { this.startInstant = startInstant; return this; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java index 965c57309b652..ea4c25db661a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java @@ -22,6 +22,8 @@ import org.apache.hudi.io.SeekableDataInputStream; import org.apache.hudi.storage.HoodieStorage; +import lombok.Getter; + import java.io.ByteArrayOutputStream; import java.util.HashMap; import java.util.Map; @@ -30,6 +32,7 @@ /** * Command block issues a specific command to the scanner. */ +@Getter public class HoodieCommandBlock extends HoodieLogBlock { private final HoodieCommandBlockTypeEnum type; @@ -53,10 +56,6 @@ public HoodieCommandBlock(Option content, Supplier compressionCodec; // This path is used for constructing HFile reader context, which should not be diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java index 554b83b3455d7..0b703ed7ac910 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java @@ -28,9 +28,11 @@ import org.apache.hudi.io.SeekableDataInputStream; import org.apache.hudi.storage.HoodieStorage; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.roaringbitmap.longlong.Roaring64NavigableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -55,8 +57,11 @@ /** * Abstract class defining a block in HoodieLogFile. */ +@AllArgsConstructor +@Getter +@Slf4j public abstract class HoodieLogBlock { - private static final Logger LOG = LoggerFactory.getLogger(HoodieLogBlock.class); + /** * The current version of the log block. Anytime the logBlock format changes this version needs to be bumped and * corresponding changes need to be made to {@link HoodieLogBlockVersion} TODO : Change this to a class, something @@ -65,32 +70,24 @@ public abstract class HoodieLogBlock { */ public static int version = 3; // Header for each log block + @Nonnull private final Map logBlockHeader; // Footer for each log block + @Nonnull private final Map logBlockFooter; // Location of a log block on disk + @Nonnull private final Option blockContentLocation; // data for a specific block + @Nonnull private Option content; + @Getter(AccessLevel.PROTECTED) + @Nullable private final Supplier inputStreamSupplier; // Toggle flag, whether to read blocks lazily (I/O intensive) or not (Memory intensive) + @Getter(AccessLevel.NONE) protected boolean readBlockLazily; - public HoodieLogBlock( - @Nonnull Map logBlockHeader, - @Nonnull Map logBlockFooter, - @Nonnull Option blockContentLocation, - @Nonnull Option content, - @Nullable Supplier inputStreamSupplier, - boolean readBlockLazily) { - this.logBlockHeader = logBlockHeader; - this.logBlockFooter = logBlockFooter; - this.blockContentLocation = blockContentLocation; - this.content = content; - this.inputStreamSupplier = inputStreamSupplier; - this.readBlockLazily = readBlockLazily; - } - // Return the bytes representation of the data belonging to a LogBlock public ByteArrayOutputStream getContentBytes(HoodieStorage storage) throws IOException { throw new HoodieException("No implementation was provided"); @@ -110,22 +107,6 @@ public long getLogBlockLength() { throw new HoodieException("No implementation was provided"); } - public Option getBlockContentLocation() { - return this.blockContentLocation; - } - - public Map getLogBlockHeader() { - return logBlockHeader; - } - - public Map getLogBlockFooter() { - return logBlockFooter; - } - - public Option getContent() { - return content; - } - /** * Compacted blocks are created using log compaction which basically merges the consecutive blocks together and create * huge block with all the changes. @@ -161,10 +142,10 @@ protected void addRecordPositionsToHeader(Set positionSet, try { logBlockHeader.put(HeaderMetadataType.RECORD_POSITIONS, LogReaderUtils.encodePositions(positionSet)); } catch (IOException e) { - LOG.error("Cannot write record positions to the log block header.", e); + log.error("Cannot write record positions to the log block header.", e); } } else { - LOG.warn("There are duplicate keys in the records (number of unique positions: {}, " + log.warn("There are duplicate keys in the records (number of unique positions: {}, " + "number of records: {}). Skip writing record positions to the log block header.", positionSet.size(), numRecords); } @@ -176,7 +157,7 @@ protected boolean containsBaseFileInstantTimeOfPositions() { } protected void removeBaseFileInstantTimeOfPositions() { - LOG.info("There are records without valid positions. " + log.info("There are records without valid positions. " + "Skip writing record positions to the block header."); logBlockHeader.remove(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS); } @@ -252,7 +233,10 @@ public enum FooterMetadataType { * This class is used to store the Location of the Content of a Log Block. It's used when a client chooses for a IO * intensive CompactedScanner, the location helps to lazily read contents from the log file */ + @AllArgsConstructor + @Getter public static final class HoodieLogBlockContentLocation { + // Storage Config required to access the file private final HoodieStorage storage; // The logFile that contains this block @@ -263,38 +247,6 @@ public static final class HoodieLogBlockContentLocation { private final long blockSize; // The final position where the complete block ends private final long blockEndPos; - - public HoodieLogBlockContentLocation(HoodieStorage storage, - HoodieLogFile logFile, - long contentPositionInLogFile, - long blockSize, - long blockEndPos) { - this.storage = storage; - this.logFile = logFile; - this.contentPositionInLogFile = contentPositionInLogFile; - this.blockSize = blockSize; - this.blockEndPos = blockEndPos; - } - - public HoodieStorage getStorage() { - return storage; - } - - public HoodieLogFile getLogFile() { - return logFile; - } - - public long getContentPositionInLogFile() { - return contentPositionInLogFile; - } - - public long getBlockSize() { - return blockSize; - } - - public long getBlockEndPos() { - return blockEndPos; - } } /** @@ -359,10 +311,6 @@ protected Option getContentAsByteStream() throws IOExcept return Option.of(baos); } - protected Supplier getInputStreamSupplier() { - return inputStreamSupplier; - } - /** * Adds the record positions if the base file instant time of the positions exists * in the log header and the record positions are all valid. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecord.java index d68e608463df4..fb78adaa772c6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecord.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecord.java @@ -23,6 +23,10 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.OrderingValues; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + import javax.annotation.Nullable; import java.io.Serializable; @@ -34,41 +38,22 @@ * * @param The type of the engine specific row. */ +@AllArgsConstructor +@Getter public class BufferedRecord implements Serializable { + private String recordKey; - private T record; private final Comparable orderingValue; + private T record; private final Integer schemaId; - @Nullable private HoodieOperation hoodieOperation; + @Nullable + @Setter + private HoodieOperation hoodieOperation; public BufferedRecord() { this(null, null, null, null, null); } - public BufferedRecord(String recordKey, Comparable orderingValue, T record, Integer schemaId, @Nullable HoodieOperation hoodieOperation) { - this.recordKey = recordKey; - this.orderingValue = orderingValue; - this.record = record; - this.schemaId = schemaId; - this.hoodieOperation = hoodieOperation; - } - - public String getRecordKey() { - return recordKey; - } - - public Comparable getOrderingValue() { - return orderingValue; - } - - public T getRecord() { - return record; - } - - public Integer getSchemaId() { - return schemaId; - } - public boolean isDelete() { return HoodieOperation.isDelete(hoodieOperation) || HoodieOperation.isUpdateBefore(hoodieOperation); } @@ -81,14 +66,6 @@ public boolean isCommitTimeOrderingDelete() { return isDelete() && OrderingValues.isDefault(orderingValue); } - public void setHoodieOperation(HoodieOperation hoodieOperation) { - this.hoodieOperation = hoodieOperation; - } - - public HoodieOperation getHoodieOperation() { - return this.hoodieOperation; - } - public BufferedRecord toBinary(RecordContext recordContext) { if (record != null) { HoodieSchema schema = recordContext.getSchemaFromBufferRecord(this); @@ -124,6 +101,8 @@ public BufferedRecord replaceRecordKey(String recordKey) { return this; } + // Intentionally not using @EqualsAndHashCode: Lombok generates instanceof/canEqual based equality, + // while this class requires exact runtime-class equality via getClass() @Override public boolean equals(Object o) { if (this == o) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java index 81d2de0ac0a23..06e903dd1fa4f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java @@ -33,16 +33,17 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.io.IOException; /** * Factory to create a {@link BufferedRecordMerger}. */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) public class BufferedRecordMergerFactory { - private BufferedRecordMergerFactory() { - } - public static BufferedRecordMerger create(HoodieReaderContext readerContext, RecordMergeMode recordMergeMode, boolean enablePartialMerging, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/DeleteContext.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/DeleteContext.java index ce6857dc2aebb..f1bd8b0b0a8ac 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/DeleteContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/DeleteContext.java @@ -27,6 +27,9 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; +import lombok.Getter; +import lombok.experimental.Accessors; + import java.io.Serializable; import java.util.Properties; @@ -37,10 +40,13 @@ /** * Schema context for deletes. */ +@Getter public class DeleteContext implements Serializable { + private static final long serialVersionUID = 1L; private final Option> customDeleteMarkerKeyValue; + @Accessors(fluent = true) private final boolean hasBuiltInDeleteField; private int hoodieOperationPos; private HoodieSchema readerSchema; @@ -108,25 +114,9 @@ private static int getHoodieOperationPos(HoodieSchema schema) { .orElseGet(() -> -1); } - public Option> getCustomDeleteMarkerKeyValue() { - return customDeleteMarkerKeyValue; - } - - public boolean hasBuiltInDeleteField() { - return hasBuiltInDeleteField; - } - - public int getHoodieOperationPos() { - return hoodieOperationPos; - } - public DeleteContext withReaderSchema(HoodieSchema readerSchema) { this.readerSchema = readerSchema; this.hoodieOperationPos = getHoodieOperationPos(readerSchema); return this; } - - public HoodieSchema getReaderSchema() { - return readerSchema; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/IncrementalQueryAnalyzer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/IncrementalQueryAnalyzer.java index 01f3e513e3411..da0bfb567abe2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/IncrementalQueryAnalyzer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/IncrementalQueryAnalyzer.java @@ -32,8 +32,11 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import javax.annotation.Nullable; @@ -108,11 +111,13 @@ * *

    IMPORTANT: the reader may optionally choose to fall back to reading the latest snapshot if there are files decoding the commit metadata are already cleaned. */ +@Slf4j public class IncrementalQueryAnalyzer { + public static final String START_COMMIT_EARLIEST = "earliest"; - private static final Logger LOG = LoggerFactory.getLogger(IncrementalQueryAnalyzer.class); private final HoodieTableMetaClient metaClient; + @Getter private final Option startCompletionTime; private final Option endCompletionTime; private final InstantRange.RangeType rangeType; @@ -143,10 +148,6 @@ private IncrementalQueryAnalyzer( this.limit = limit; } - public Option getStartCompletionTime() { - return startCompletionTime; - } - /** * Returns a builder. */ @@ -206,7 +207,7 @@ public QueryContext analyze() { String endInstant = endCompletionTime.isEmpty() ? null : lastInstant; return QueryContext.create(startInstant, endInstant, instants, archivedInstants, activeInstants, filteredTimeline, archivedReadTimeline); } catch (Exception ex) { - LOG.error("Got exception when generating incremental query info", ex); + log.error("Got exception when generating incremental query info", ex); throw new HoodieException(ex); } } @@ -284,6 +285,7 @@ private static HoodieTimeline filterInstantsAsPerUserConfigs( /** * Builder for {@link IncrementalQueryAnalyzer}. */ + @NoArgsConstructor public static class Builder { /** * Start completion time. @@ -304,9 +306,6 @@ public static class Builder { */ private int limit = -1; - public Builder() { - } - public Builder startCompletionTime(String startCompletionTime) { this.startCompletionTime = startCompletionTime; return this; @@ -361,9 +360,12 @@ public IncrementalQueryAnalyzer build() { /** * Represents the analyzed query context. */ + @AllArgsConstructor(access = AccessLevel.PRIVATE) + @Getter public static class QueryContext { + public static final QueryContext EMPTY = - new QueryContext(null, null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), null, null); + new QueryContext(Option.empty(), Option.empty(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), null, null); /** * An empty option indicates consumption from the earliest instant. @@ -373,6 +375,8 @@ public static class QueryContext { * An empty option indicates consumption to the latest instant. */ private final Option endInstant; + @Getter(AccessLevel.NONE) + private final List instants; private final List archivedInstants; private final List activeInstants; /** @@ -382,25 +386,9 @@ public static class QueryContext { /** * The archived timeline to read filtered by given configurations. */ + @Nullable + @Getter(AccessLevel.NONE) private final HoodieTimeline archivedTimeline; - private final List instants; - - private QueryContext( - @Nullable String startInstant, - @Nullable String endInstant, - List instants, - List archivedInstants, - List activeInstants, - HoodieTimeline activeTimeline, - @Nullable HoodieTimeline archivedTimeline) { - this.startInstant = Option.ofNullable(startInstant); - this.endInstant = Option.ofNullable(endInstant); - this.archivedInstants = archivedInstants; - this.activeInstants = activeInstants; - this.activeTimeline = activeTimeline; - this.archivedTimeline = archivedTimeline; - this.instants = instants; - } public static QueryContext create( @Nullable String startInstant, @@ -410,7 +398,7 @@ public static QueryContext create( List activeInstants, HoodieTimeline activeTimeline, @Nullable HoodieTimeline archivedTimeline) { - return new QueryContext(startInstant, endInstant, instants, archivedInstants, activeInstants, activeTimeline, archivedTimeline); + return new QueryContext(Option.ofNullable(startInstant), Option.ofNullable(endInstant), instants, archivedInstants, activeInstants, activeTimeline, archivedTimeline); } public boolean isEmpty() { @@ -421,14 +409,6 @@ public List getInstantTimeList() { return this.instants; } - public Option getStartInstant() { - return startInstant; - } - - public Option getEndInstant() { - return endInstant; - } - /** * Returns the latest instant time which should be included physically in reading. */ @@ -441,14 +421,6 @@ public List getInstants() { return Stream.concat(archivedInstants.stream(), activeInstants.stream()).collect(Collectors.toList()); } - public List getArchivedInstants() { - return archivedInstants; - } - - public List getActiveInstants() { - return activeInstants; - } - public boolean isConsumingFromEarliest() { return startInstant.isEmpty(); } @@ -489,10 +461,6 @@ public Option getInstantRange() { } } - public HoodieTimeline getActiveTimeline() { - return this.activeTimeline; - } - public @Nullable HoodieTimeline getArchivedTimeline() { return archivedTimeline; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java index e84921c080ae7..e8ff9add0ad85 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/DefaultFileGroupRecordBufferLoader.java @@ -33,6 +33,9 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.storage.HoodieStorage; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.util.List; /** @@ -40,16 +43,15 @@ * * @param the engine specific record type */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) class DefaultFileGroupRecordBufferLoader extends LogScanningRecordBufferLoader implements FileGroupRecordBufferLoader { + private static final DefaultFileGroupRecordBufferLoader INSTANCE = new DefaultFileGroupRecordBufferLoader<>(); static DefaultFileGroupRecordBufferLoader getInstance() { return INSTANCE; } - private DefaultFileGroupRecordBufferLoader() { - } - @Override public Pair, List> getRecordBuffer(HoodieReaderContext readerContext, HoodieStorage storage, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java index 702ecf5041e09..8d5d967ec2567 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java @@ -50,6 +50,9 @@ import org.apache.hudi.internal.schema.action.InternalSchemaMerger; import org.apache.hudi.internal.schema.convert.InternalSchemaConverter; +import lombok.Getter; +import lombok.Setter; + import java.io.IOException; import java.io.Serializable; import java.util.Iterator; @@ -77,8 +80,10 @@ abstract class FileGroupRecordBuffer implements HoodieFileGroupRecordBuffer> payloadClasses; protected final TypedProperties props; protected final ExternalSpillableMap> records; + @Getter protected final DeleteContext deleteContext; protected final BufferedRecordConverter bufferedRecordConverter; + @Setter protected ClosableIterator baseFileIterator; protected UpdateProcessor updateProcessor; protected Iterator> logRecordIterator; @@ -87,6 +92,7 @@ abstract class FileGroupRecordBuffer implements HoodieFileGroupRecordBuffer bufferedRecordMerger; + @Getter protected long totalLogRecords = 0; protected FileGroupRecordBuffer(HoodieReaderContext readerContext, @@ -131,15 +137,6 @@ protected ExternalSpillableMap> initializeRecord readerContext.getRecordSizeEstimator(), diskMapType, readerContext.getRecordSerializer(), isBitCaskDiskMapCompressionEnabled, getClass().getSimpleName()); } - @Override - public void setBaseFileIterator(ClosableIterator baseFileIterator) { - this.baseFileIterator = baseFileIterator; - } - - public DeleteContext getDeleteContext() { - return deleteContext; - } - /** * This allows hasNext() to be called multiple times without incrementing the iterator by more than 1 * record. It does come with the caveat that hasNext() must be called every time before next(). But @@ -169,10 +166,6 @@ public int size() { return records.size(); } - public long getTotalLogRecords() { - return totalLogRecords; - } - @Override public ClosableIterator> getLogRecordIterator() { return new LogRecordIterator<>(this); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java index 73dd5dda5e870..5eaa54e033c83 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java @@ -41,9 +41,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieKeyException; +import lombok.extern.slf4j.Slf4j; import org.roaringbitmap.longlong.Roaring64NavigableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -61,8 +60,8 @@ * Here the position means that record position in the base file. The records from the base file is accessed from an iterator object. These records are merged when the * {@link #hasNext} method is called. */ +@Slf4j public class PositionBasedFileGroupRecordBuffer extends KeyBasedFileGroupRecordBuffer { - private static final Logger LOG = LoggerFactory.getLogger(PositionBasedFileGroupRecordBuffer.class); private static final String ROW_INDEX_COLUMN_NAME = "row_index"; public static final String ROW_INDEX_TEMPORARY_COLUMN_NAME = "_tmp_metadata_" + ROW_INDEX_COLUMN_NAME; @@ -96,7 +95,7 @@ public void processDataBlock(HoodieDataBlock dataBlock, Option keySpecO // Extract positions from data block. List recordPositions = extractRecordPositions(dataBlock, baseFileInstantTime); if (recordPositions == null) { - LOG.debug("Falling back to key based merge for data block"); + log.debug("Falling back to key based merge for data block"); fallbackToKeyBasedBuffer(); super.processDataBlock(dataBlock, keySpecOpt); return; @@ -181,7 +180,7 @@ public void processDeleteBlock(HoodieDeleteBlock deleteBlock) throws IOException List recordPositions = extractRecordPositions(deleteBlock, baseFileInstantTime); if (recordPositions == null) { - LOG.debug("Falling back to key based merging for delete block"); + log.debug("Falling back to key based merging for delete block"); fallbackToKeyBasedBuffer(); super.processDeleteBlock(deleteBlock); return; @@ -291,7 +290,7 @@ protected static List extractRecordPositions(HoodieLogBlock logBlock, String blockBaseFileInstantTime = logBlock.getBaseFileInstantTimeOfPositions(); if (StringUtils.isNullOrEmpty(blockBaseFileInstantTime) || !baseFileInstantTime.equals(blockBaseFileInstantTime)) { - LOG.debug("The record positions cannot be used because the base file instant time " + log.debug("The record positions cannot be used because the base file instant time " + "is either missing or different from the base file to merge. " + "Instant time in the header: {}, base file instant time of the file group: {}.", blockBaseFileInstantTime, baseFileInstantTime); @@ -299,7 +298,7 @@ protected static List extractRecordPositions(HoodieLogBlock logBlock, } Roaring64NavigableMap positions = logBlock.getRecordPositions(); if (positions == null || positions.isEmpty()) { - LOG.info("No record position info is found when attempting to do position based merge."); + log.info("No record position info is found when attempting to do position based merge."); return null; } @@ -309,7 +308,7 @@ protected static List extractRecordPositions(HoodieLogBlock logBlock, } if (blockPositions.isEmpty()) { - LOG.info("No positions are extracted."); + log.info("No positions are extracted."); return null; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/HoodieSourceSplitSerializer.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/HoodieSourceSplitSerializer.java index 7f6771ba39408..57b0a30c4cb1b 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/HoodieSourceSplitSerializer.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/HoodieSourceSplitSerializer.java @@ -103,13 +103,13 @@ public byte[] serialize(HoodieSourceSplit obj) throws IOException { out.writeBoolean(false); } - out.writeBoolean(instantRange.getStartInstant().isPresent()); - if (instantRange.getStartInstant().isPresent()) { - out.writeUTF(instantRange.getStartInstant().get()); + out.writeBoolean(instantRange.getStartInstantOpt().isPresent()); + if (instantRange.getStartInstantOpt().isPresent()) { + out.writeUTF(instantRange.getStartInstantOpt().get()); } - out.writeBoolean(instantRange.getEndInstant().isPresent()); - if (instantRange.getEndInstant().isPresent()) { - out.writeUTF(instantRange.getEndInstant().get()); + out.writeBoolean(instantRange.getEndInstantOpt().isPresent()); + if (instantRange.getEndInstantOpt().isPresent()) { + out.writeUTF(instantRange.getEndInstantOpt().get()); } if (instantRange.getRangeType().equals(InstantRange.RangeType.EXACT_MATCH)) { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java index deaa492ce15dd..ce97b2e70f753 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java @@ -438,11 +438,11 @@ void testInputSplitsWithSpeedLimit() throws Exception { IncrementalInputSplits.Result result = iis.inputSplits(metaClient, firstInstant.getCompletionTime(), false); String minStartCommit = result.getInputSplits().stream() - .map(split -> split.getInstantRange().get().getStartInstant().get()) + .map(split -> split.getInstantRange().get().getStartInstantOpt().get()) .min((commit1,commit2) -> compareTimestamps(commit1, LESSER_THAN, commit2) ? 1 : 0) .orElse(null); String maxEndCommit = result.getInputSplits().stream() - .map(split -> split.getInstantRange().get().getEndInstant().get()) + .map(split -> split.getInstantRange().get().getEndInstantOpt().get()) .max((commit1,commit2) -> compareTimestamps(commit1, GREATER_THAN, commit2) ? 1 : 0) .orElse(null); assertEquals(0, intervalBetween2Instants(commitsTimeline, minStartCommit, maxEndCommit), "Should read 1 instant"); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadMonitoringFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadMonitoringFunction.java index ec6f3b863cafb..bb28ee92e2dff 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadMonitoringFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadMonitoringFunction.java @@ -505,8 +505,8 @@ public void testCheckpointRestoreWithLimit() throws Exception { private static boolean isPointInstantRange(InstantRange instantRange, String timestamp) { return instantRange != null - && Objects.equals(timestamp, instantRange.getStartInstant().get()) - && Objects.equals(timestamp, instantRange.getEndInstant().get()); + && Objects.equals(timestamp, instantRange.getStartInstantOpt().get()) + && Objects.equals(timestamp, instantRange.getEndInstantOpt().get()); } private AbstractStreamOperatorTestHarness createHarness( diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplit.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplit.java index 1cb244245ce82..8dd32af3adf3b 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplit.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplit.java @@ -360,8 +360,8 @@ public void testInstantRangePresent() { ); assertTrue(split.getInstantRange().isPresent()); - assertEquals("20230101000000000", split.getInstantRange().get().getStartInstant().get()); - assertEquals("20230131235959999", split.getInstantRange().get().getEndInstant().get()); + assertEquals("20230101000000000", split.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230131235959999", split.getInstantRange().get().getEndInstantOpt().get()); } @Test @@ -402,9 +402,9 @@ public void testInstantRangeWithOnlyStart() { ); assertTrue(split.getInstantRange().isPresent()); - assertTrue(split.getInstantRange().get().getStartInstant().isPresent()); - assertFalse(split.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230101000000000", split.getInstantRange().get().getStartInstant().get()); + assertTrue(split.getInstantRange().get().getStartInstantOpt().isPresent()); + assertFalse(split.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230101000000000", split.getInstantRange().get().getStartInstantOpt().get()); } @Test @@ -476,7 +476,7 @@ public void testClosedClosedInstantRange() { ); assertTrue(split.getInstantRange().isPresent()); - assertTrue(split.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(split.getInstantRange().get().getEndInstant().isPresent()); + assertTrue(split.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(split.getInstantRange().get().getEndInstantOpt().isPresent()); } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplitSerializer.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplitSerializer.java index fc30da2193d7c..04b526a805420 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplitSerializer.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/split/TestHoodieSourceSplitSerializer.java @@ -509,10 +509,10 @@ public void testSerializeWithInstantRangeStartAndEnd() throws IOException { assertNotNull(deserialized); assertTrue(deserialized.getInstantRange().isPresent()); - assertTrue(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstant().get()); - assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstant().get()); + assertTrue(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstantOpt().get()); } @Test @@ -540,9 +540,9 @@ public void testSerializeWithInstantRangeOnlyStart() throws IOException { assertNotNull(deserialized); assertTrue(deserialized.getInstantRange().isPresent()); - assertTrue(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertFalse(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstant().get()); + assertTrue(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertFalse(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstantOpt().get()); } @Test @@ -570,10 +570,10 @@ public void testSerializeWithClosedClosedInstantRange() throws IOException { assertNotNull(deserialized); assertTrue(deserialized.getInstantRange().isPresent()); - assertTrue(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstant().get()); - assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstant().get()); + assertTrue(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstantOpt().get()); } @Test @@ -605,8 +605,8 @@ public void testSerializeWithInstantRangeAndConsumedState() throws IOException { assertTrue(deserialized.getInstantRange().isPresent()); assertEquals(10, deserialized.getFileOffset()); assertEquals(500L, deserialized.getConsumed()); - assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstant().get()); - assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstant().get()); + assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstantOpt().get()); } @Test @@ -637,13 +637,13 @@ public void testSerializeMultipleSplitsWithInstantRange() throws IOException { // Verify split1 assertTrue(deserialized1.getInstantRange().isPresent()); - assertEquals("20230101000000000", deserialized1.getInstantRange().get().getStartInstant().get()); - assertEquals("20230131235959999", deserialized1.getInstantRange().get().getEndInstant().get()); + assertEquals("20230101000000000", deserialized1.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230131235959999", deserialized1.getInstantRange().get().getEndInstantOpt().get()); // Verify split2 assertTrue(deserialized2.getInstantRange().isPresent()); - assertEquals("20230201000000000", deserialized2.getInstantRange().get().getStartInstant().get()); - assertFalse(deserialized2.getInstantRange().get().getEndInstant().isPresent()); + assertEquals("20230201000000000", deserialized2.getInstantRange().get().getStartInstantOpt().get()); + assertFalse(deserialized2.getInstantRange().get().getEndInstantOpt().isPresent()); // Verify split3 assertFalse(deserialized3.getInstantRange().isPresent()); @@ -863,15 +863,15 @@ public void testSerializeMultipleSplitsWithDifferentRangeTypes() throws IOExcept assertTrue(deserialized2.getInstantRange().isPresent()); assertEquals(InstantRange.RangeType.OPEN_CLOSED, deserialized2.getInstantRange().get().getRangeType()); - assertEquals("20230201000000000", deserialized2.getInstantRange().get().getStartInstant().get()); - assertEquals("20230228235959999", deserialized2.getInstantRange().get().getEndInstant().get()); + assertEquals("20230201000000000", deserialized2.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230228235959999", deserialized2.getInstantRange().get().getEndInstantOpt().get()); // Verify split3 (CLOSED_CLOSED) assertTrue(deserialized3.getInstantRange().isPresent()); assertEquals(InstantRange.RangeType.CLOSED_CLOSED, deserialized3.getInstantRange().get().getRangeType()); - assertEquals("20230301000000000", deserialized3.getInstantRange().get().getStartInstant().get()); - assertEquals("20230331235959999", deserialized3.getInstantRange().get().getEndInstant().get()); + assertEquals("20230301000000000", deserialized3.getInstantRange().get().getStartInstantOpt().get()); + assertEquals("20230331235959999", deserialized3.getInstantRange().get().getEndInstantOpt().get()); } @Test @@ -954,9 +954,9 @@ public void testSerializeWithClosedClosedRangeOnlyStart() throws IOException { assertTrue(deserialized.getInstantRange().isPresent()); assertEquals(InstantRange.RangeType.CLOSED_CLOSED, deserialized.getInstantRange().get().getRangeType()); - assertTrue(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertFalse(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstant().get()); + assertTrue(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertFalse(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230101000000000", deserialized.getInstantRange().get().getStartInstantOpt().get()); // Verify range behavior - start is inclusive, no end boundary assertTrue(deserialized.getInstantRange().get().isInRange("20230101000000000")); // start inclusive @@ -992,9 +992,9 @@ public void testSerializeWithClosedClosedRangeOnlyEnd() throws IOException { assertTrue(deserialized.getInstantRange().isPresent()); assertEquals(InstantRange.RangeType.CLOSED_CLOSED, deserialized.getInstantRange().get().getRangeType()); - assertFalse(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstant().get()); + assertFalse(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstantOpt().get()); // Verify range behavior - no start boundary, end is inclusive assertTrue(deserialized.getInstantRange().get().isInRange("19700101000000000")); @@ -1029,9 +1029,9 @@ public void testSerializeWithOpenClosedRangeOnlyEnd() throws IOException { assertTrue(deserialized.getInstantRange().isPresent()); assertEquals(InstantRange.RangeType.OPEN_CLOSED, deserialized.getInstantRange().get().getRangeType()); - assertFalse(deserialized.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized.getInstantRange().get().getEndInstant().isPresent()); - assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstant().get()); + assertFalse(deserialized.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized.getInstantRange().get().getEndInstantOpt().isPresent()); + assertEquals("20230131235959999", deserialized.getInstantRange().get().getEndInstantOpt().get()); // Verify range behavior - no start boundary, end is inclusive assertTrue(deserialized.getInstantRange().get().isInRange("19700101000000000")); @@ -1087,23 +1087,23 @@ public void testSerializeWithAllRangeTypesAndNullableBoundaries() throws IOExcep HoodieSourceSplit deserialized4 = serializer.deserialize(serializer.getVersion(), serialized4); // Verify OPEN_CLOSED with only start - assertTrue(deserialized1.getInstantRange().get().getStartInstant().isPresent()); - assertFalse(deserialized1.getInstantRange().get().getEndInstant().isPresent()); + assertTrue(deserialized1.getInstantRange().get().getStartInstantOpt().isPresent()); + assertFalse(deserialized1.getInstantRange().get().getEndInstantOpt().isPresent()); assertEquals(InstantRange.RangeType.OPEN_CLOSED, deserialized1.getInstantRange().get().getRangeType()); // Verify OPEN_CLOSED with only end - assertFalse(deserialized2.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized2.getInstantRange().get().getEndInstant().isPresent()); + assertFalse(deserialized2.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized2.getInstantRange().get().getEndInstantOpt().isPresent()); assertEquals(InstantRange.RangeType.OPEN_CLOSED, deserialized2.getInstantRange().get().getRangeType()); // Verify CLOSED_CLOSED with only start - assertTrue(deserialized3.getInstantRange().get().getStartInstant().isPresent()); - assertFalse(deserialized3.getInstantRange().get().getEndInstant().isPresent()); + assertTrue(deserialized3.getInstantRange().get().getStartInstantOpt().isPresent()); + assertFalse(deserialized3.getInstantRange().get().getEndInstantOpt().isPresent()); assertEquals(InstantRange.RangeType.CLOSED_CLOSED, deserialized3.getInstantRange().get().getRangeType()); // Verify CLOSED_CLOSED with only end - assertFalse(deserialized4.getInstantRange().get().getStartInstant().isPresent()); - assertTrue(deserialized4.getInstantRange().get().getEndInstant().isPresent()); + assertFalse(deserialized4.getInstantRange().get().getStartInstantOpt().isPresent()); + assertTrue(deserialized4.getInstantRange().get().getEndInstantOpt().isPresent()); assertEquals(InstantRange.RangeType.CLOSED_CLOSED, deserialized4.getInstantRange().get().getRangeType()); } From 9b112cd3078f388876c03b48c3bfb57144a696d8 Mon Sep 17 00:00:00 2001 From: ericyuan915 <77124531+ericyuan915@users.noreply.github.com> Date: Thu, 11 Jun 2026 23:44:02 -0700 Subject: [PATCH 064/255] fix(flink): relocate org.apache.flink.dropwizard (#18982) * Update pom.xml to resolve the bug (cherry picked from commit 1f20fd42e2b94641e3b3838e42d8e4544ae36551) --- packaging/hudi-flink-bundle/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml index 25c3dff166c1e..0c995e939c233 100644 --- a/packaging/hudi-flink-bundle/pom.xml +++ b/packaging/hudi-flink-bundle/pom.xml @@ -196,6 +196,15 @@ com.codahale.metrics. org.apache.hudi.com.codahale.metrics. + + + org.apache.flink.dropwizard. + ${flink.bundle.shade.prefix}org.apache.flink.dropwizard. + com.beust.jcommander. ${flink.bundle.shade.prefix}com.beust.jcommander. From cce4bbebed56ca805f9750406e4e844da06908e8 Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Sat, 13 Jun 2026 07:12:44 +0800 Subject: [PATCH 065/255] feat: add more metrics for delta streamer (#18085) * Add more metrics for hudi streamer Summary: Add the following metrics 1. Emit success/failure metrics when streamer job is finished. 2. Emit the number of commits to process in the streamer job. 3. Emit the number of unprocessed commits in the source table timeline. --------- Co-authored-by: Jintao Guan Co-authored-by: Claude Opus 4.6 Co-authored-by: sivabalan Co-authored-by: Lokesh Jain (cherry picked from commit a032431c81aa71d5d627b13cb5189d9a9c54ba46) --- .../ingestion/HoodieIngestionMetrics.java | 4 ++ .../utilities/sources/SqlFileBasedSource.java | 6 ++- .../hudi/utilities/sources/SqlSource.java | 10 ++-- .../utilities/streamer/HoodieStreamer.java | 4 +- .../streamer/HoodieStreamerMetrics.java | 15 ++++++ .../hudi/utilities/streamer/StreamSync.java | 8 ++++ .../TestHoodieDeltaStreamer.java | 16 ++++--- ...odieDeltaStreamerSchemaEvolutionQuick.java | 9 +++- ...estHoodieDeltaStreamerWithMultiWriter.java | 5 +- .../multisync/TestMultipleMetaSync.java | 13 +++-- .../sources/TestSqlFileBasedSource.java | 13 +++-- .../hudi/utilities/sources/TestSqlSource.java | 15 +++--- .../TestStreamerSourceCheckpointVersion.java | 2 +- .../TestHoodieIncrSourceE2EAutoUpgrade.java | 7 ++- .../streamer/TestHoodieStreamerMetrics.java | 47 +++++++++++++++++++ .../utilities/streamer/TestStreamSync.java | 33 +++++++++++++ 16 files changed, 173 insertions(+), 34 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java index d58e7877f5e3b..44871f8c45fb9 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java @@ -48,6 +48,10 @@ public HoodieIngestionMetrics(HoodieMetricsConfig writeConfig) { public abstract void updateStreamerMetrics(long durationNanos); + public abstract void emitStreamerJobSuccessMetrics(); + + public abstract void emitStreamerJobFailedMetrics(); + public abstract void updateStreamerMetaSyncMetrics(String syncClassShortName, long syncTimeNanos); public abstract void updateStreamerSyncMetrics(long syncEpochTimeMs); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java index 9138064203233..5ca569136e7ce 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlFileBasedSource.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; import org.apache.hudi.utilities.schema.SchemaProvider; import lombok.extern.slf4j.Slf4j; @@ -65,16 +66,19 @@ public class SqlFileBasedSource extends RowSource { private final String sourceSqlFile; private final boolean shouldEmitCheckPoint; + private HoodieIngestionMetrics metrics; public SqlFileBasedSource( TypedProperties props, JavaSparkContext sparkContext, SparkSession sparkSession, - SchemaProvider schemaProvider) { + SchemaProvider schemaProvider, + HoodieIngestionMetrics metrics) { super(props, sparkContext, sparkSession, schemaProvider); checkRequiredConfigProperties(props, Collections.singletonList(SOURCE_SQL_FILE)); sourceSqlFile = getStringWithAltKeys(props, SOURCE_SQL_FILE); shouldEmitCheckPoint = getBooleanWithAltKeys(props, EMIT_EPOCH_CHECKPOINT); + this.metrics = metrics; } @Override diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlSource.java index a345e937003f1..21f1230db4697 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SqlSource.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.utilities.config.SqlSourceConfig; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; import org.apache.hudi.utilities.schema.SchemaProvider; import lombok.extern.slf4j.Slf4j; @@ -61,17 +62,20 @@ public class SqlSource extends RowSource { private static final long serialVersionUID = 1L; private final String sourceSql; private final SparkSession spark; + private final HoodieIngestionMetrics metrics; public SqlSource( TypedProperties props, JavaSparkContext sparkContext, SparkSession sparkSession, - SchemaProvider schemaProvider) { + SchemaProvider schemaProvider, + HoodieIngestionMetrics metrics) { super(props, sparkContext, sparkSession, schemaProvider); checkRequiredConfigProperties( props, Collections.singletonList(SqlSourceConfig.SOURCE_SQL)); - sourceSql = getStringWithAltKeys(props, SqlSourceConfig.SOURCE_SQL); - spark = sparkSession; + this.sourceSql = getStringWithAltKeys(props, SqlSourceConfig.SOURCE_SQL); + this.spark = sparkSession; + this.metrics = metrics; } @Override diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java index 0186e5fd05840..9fc9dbff66a9d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java @@ -924,7 +924,9 @@ private void shutdownAsyncServices(boolean error) { public void ingestOnce() { try { streamSync.syncOnce(); - } catch (IOException e) { + streamSync.reportSuccessMetrics(); + } catch (Exception e) { + streamSync.reportFailureMetrics(); throw new HoodieIngestionException(String.format("Ingestion via %s failed with exception.", this.getClass()), e); } finally { close(); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java index 5813533e2184d..0e0db770012a3 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java @@ -108,6 +108,20 @@ public void updateStreamerMetrics(long durationInNs) { } } + @Override + public void emitStreamerJobSuccessMetrics() { + if (writeConfig.isMetricsOn()) { + metrics.registerGauge(getMetricsName("deltastreamer", "success"), 1); + } + } + + @Override + public void emitStreamerJobFailedMetrics() { + if (writeConfig.isMetricsOn()) { + metrics.registerGauge(getMetricsName("deltastreamer", "failure"), 1); + } + } + @Override public void updateStreamerMetaSyncMetrics(String syncClassShortName, long syncNs) { if (writeConfig.isMetricsOn()) { @@ -198,6 +212,7 @@ public void updateStreamerSourceParallelism(int sourceParallelism) { } } + @Override public void updateStreamerSourceBytesToBeIngestedInSyncRound(long sourceBytesToBeIngested) { if (writeConfig.isMetricsOn()) { metrics.registerGauge(getMetricsName("deltastreamer", "sourceBytesToBeIngestedInSyncRound"), sourceBytesToBeIngested); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 7f38d80f83080..e79dbece90b72 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -589,6 +589,14 @@ private void initializeWriteClientAndRetryTableServices(InputBatch inputBatch, H } } + public void reportSuccessMetrics() { + metrics.emitStreamerJobSuccessMetrics(); + } + + public void reportFailureMetrics() { + metrics.emitStreamerJobFailedMetrics(); + } + private Option getLastPendingClusteringInstant(Option commitTimelineOpt) { if (commitTimelineOpt.isPresent()) { Option pendingClusteringInstant = commitTimelineOpt.get().getLastPendingClusterInstant(); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 0584a1ba26755..684352d033096 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -103,6 +103,7 @@ import org.apache.hudi.utilities.UtilHelpers; import org.apache.hudi.utilities.config.HoodieStreamerConfig; import org.apache.hudi.utilities.config.SourceTestConfig; +import org.apache.hudi.utilities.ingestion.HoodieIngestionException; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor; import org.apache.hudi.utilities.schema.SchemaProvider; @@ -2608,11 +2609,12 @@ public void testNullSchemaProvider() { HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.BULK_INSERT, Collections.singletonList(SqlQueryBasedTransformer.class.getName()), PROPS_FILENAME_TEST_SOURCE, true, false, false, null, null); - Exception e = assertThrows(HoodieException.class, () -> { + Exception e = assertThrows(HoodieIngestionException.class, () -> { syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf())); }, "Should error out when schema provider is not provided"); log.debug("Expected error during reading data from source ", e); - assertTrue(e.getMessage().contains("Schema provider is required for this operation and for the source of interest. " + String errorMsg = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + assertTrue(errorMsg.contains("Schema provider is required for this operation and for the source of interest. " + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. " + "Based on the schema provider class chosen, additional configs might be required. " + "For eg, if you choose 'org.apache.hudi.utilities.schema.SchemaRegistryProvider', " @@ -3532,16 +3534,18 @@ public void testCsvDFSSourceNoHeaderWithoutSchemaProviderAndWithTransformer() th // Target schema is determined based on the Dataframe after transformation // No CSV header and no schema provider at the same time are not recommended, // as the transformer behavior may be unexpected - Exception e = assertThrows(AnalysisException.class, () -> { + Exception e = assertThrows(HoodieIngestionException.class, () -> { testCsvDFSSource(false, '\t', false, Collections.singletonList(TripsWithDistanceTransformer.class.getName())); }, "Should error out when doing the transformation."); log.debug("Expected error during transformation", e); + Throwable cause = e.getCause(); + assertTrue(cause instanceof AnalysisException, "Expected cause to be AnalysisException but was: " + cause.getClass()); // First message for Spark 3.4 and above, second message for Spark 3.3, third message for Spark 3.2 and below assertTrue( - e.getMessage().contains("[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column or function parameter " + cause.getMessage().contains("[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column or function parameter " + "with name `begin_lat` cannot be resolved. Did you mean one of the following?") - || e.getMessage().contains("Column 'begin_lat' does not exist. Did you mean one of the following?") - || e.getMessage().contains("cannot resolve 'begin_lat' given input columns:")); + || cause.getMessage().contains("Column 'begin_lat' does not exist. Did you mean one of the following?") + || cause.getMessage().contains("cannot resolve 'begin_lat' given input columns:")); } @Test diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java index e2fc16dc6a2f2..5c07ff9d12949 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.MissingSchemaFieldException; import org.apache.hudi.utilities.UtilHelpers; +import org.apache.hudi.utilities.ingestion.HoodieIngestionException; import org.apache.hudi.utilities.streamer.HoodieStreamer; import org.apache.spark.sql.Column; @@ -231,7 +232,9 @@ public void testBase(String tableType, addData(df, false); deltaStreamer.sync(); assertTrue(allowNullForDeletedCols); - } catch (MissingSchemaFieldException e) { + } catch (HoodieIngestionException e) { + assertTrue(e.getCause() instanceof MissingSchemaFieldException, + "Expected cause to be MissingSchemaFieldException but was: " + e.getCause()); assertFalse(allowNullForDeletedCols); return; } @@ -421,7 +424,9 @@ public void testDroppedColumn(String tableType, assertTrue(riderFieldOpt.get().schema().getTypes() .stream().anyMatch(t -> HoodieSchemaType.STRING == t.getType())); assertTrue(metaClient.reloadActiveTimeline().lastInstant().get().compareTo(lastInstant) > 0); - } catch (MissingSchemaFieldException e) { + } catch (HoodieIngestionException e) { + assertTrue(e.getCause() instanceof MissingSchemaFieldException, + "Expected cause to be MissingSchemaFieldException but was: " + e.getCause()); assertFalse(allowNullForDeletedCols || targetSchemaSameAsTableSchema); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java index bfe5dbda81904..194b9b9bb9d0e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerWithMultiWriter.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.checkpoint.CheckpointUtils; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.JavaTestUtils; import org.apache.hudi.io.util.FileIOUtils; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieCompactionConfig; @@ -437,7 +438,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, * Need to perform getMessage().contains since the exception coming * from {@link org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.DeltaSyncService} gets wrapped many times into RuntimeExceptions. */ - if (expectConflict && backfillFailed.get() && e.getCause().getMessage().contains(ConcurrentModificationException.class.getName())) { + if (expectConflict && backfillFailed.get() && JavaTestUtils.checkNestedExceptionContains(e, ConcurrentModificationException.class.getName())) { // expected ConcurrentModificationException since ingestion & backfill will have overlapping writes if (!continuousFailed.get()) { // if backfill job failed, shutdown the continuous job. @@ -447,7 +448,7 @@ private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, // both backfill and ingestion job cannot fail. throw new HoodieException("Both backfilling and ingestion job failed ", e); } - } else if (expectConflict && continuousFailed.get() && e.getCause().getMessage().contains("Ingestion service was shut down with exception")) { + } else if (expectConflict && continuousFailed.get() && JavaTestUtils.checkNestedExceptionContains(e, "Ingestion service was shut down with exception")) { // incase of regular ingestion job failing, ConcurrentModificationException is not throw all the way. if (!backfillFailed.get()) { log.warn("Calling shutdown on backfill job since the ingstion/continuous job has failed for {}", jobId); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/multisync/TestMultipleMetaSync.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/multisync/TestMultipleMetaSync.java index d1e0f70b78f8c..9c332de1feba2 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/multisync/TestMultipleMetaSync.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/multisync/TestMultipleMetaSync.java @@ -22,6 +22,7 @@ import org.apache.hudi.exception.HoodieMetaSyncException; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamerTestBase; +import org.apache.hudi.utilities.ingestion.HoodieIngestionException; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.sources.TestDataSource; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; @@ -61,8 +62,9 @@ void testWithException(String syncClassNames) { MockSyncTool1.syncSuccess = false; MockSyncTool2.syncSuccess = false; HoodieDeltaStreamer.Config cfg = getConfig(tableBasePath, syncClassNames); - Exception e = assertThrows(HoodieMetaSyncException.class, () -> syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf()))); - assertTrue(e.getMessage().contains(MockSyncToolException1.class.getName())); + Exception e = assertThrows(HoodieIngestionException.class, () -> syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf()))); + assertTrue(e.getCause() instanceof HoodieMetaSyncException); + assertTrue(e.getCause().getMessage().contains(MockSyncToolException1.class.getName())); assertTrue(MockSyncTool1.syncSuccess); assertTrue(MockSyncTool2.syncSuccess); } @@ -73,9 +75,10 @@ void testMultipleExceptions() { MockSyncTool1.syncSuccess = false; MockSyncTool2.syncSuccess = false; HoodieDeltaStreamer.Config cfg = getConfig(tableBasePath, getSyncNames("MockSyncTool1", "MockSyncTool2", "MockSyncToolException1", "MockSyncToolException2")); - Exception e = assertThrows(HoodieMetaSyncException.class, () -> syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf()))); - assertTrue(e.getMessage().contains(MockSyncToolException1.class.getName())); - assertTrue(e.getMessage().contains(MockSyncToolException2.class.getName())); + Exception e = assertThrows(HoodieIngestionException.class, () -> syncOnce(new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf()))); + assertTrue(e.getCause() instanceof HoodieMetaSyncException); + assertTrue(e.getCause().getMessage().contains(MockSyncToolException1.class.getName())); + assertTrue(e.getCause().getMessage().contains(MockSyncToolException2.class.getName())); assertTrue(MockSyncTool1.syncSuccess); assertTrue(MockSyncTool2.syncSuccess); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java index f89552e62390b..72984e7247f96 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.streamer.SourceFormatAdapter; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; @@ -45,6 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; /** * Test against {@link SqlSource}. @@ -60,6 +62,7 @@ public class TestSqlFileBasedSource extends UtilitiesTestBase { private TypedProperties props; private SqlFileBasedSource sqlFileSource; private SourceFormatAdapter sourceFormatAdapter; + private final HoodieIngestionMetrics metrics = mock(HoodieIngestionMetrics.class); @BeforeAll public static void initClass() throws Exception { @@ -114,7 +117,7 @@ public void testSqlFileBasedSourceAvroFormat() throws IOException { UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); props.setProperty(sqlFileSourceConfig, UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); - sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider); + sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlFileSource); // Test fetching Avro format @@ -141,7 +144,7 @@ public void testSqlFileBasedSourceRowFormat() throws IOException { UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); props.setProperty(sqlFileSourceConfig, UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); - sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider); + sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlFileSource); // Test fetching Row format @@ -163,7 +166,7 @@ public void testSqlFileBasedSourceMoreRecordsThanSourceLimit() throws IOExceptio UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); props.setProperty(sqlFileSourceConfig, UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); - sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider); + sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlFileSource); InputBatch> fetch1AsRows = @@ -184,7 +187,7 @@ public void testSqlFileBasedSourceInvalidTable() throws IOException { UtilitiesTestBase.basePath + "/sql-file-based-source-invalid-table.sql"); props.setProperty(sqlFileSourceConfig, UtilitiesTestBase.basePath + "/sql-file-based-source-invalid-table.sql"); - sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider); + sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlFileSource); assertThrows( @@ -201,7 +204,7 @@ public void shouldSetCheckpointForSqlFileBasedSourceWithEpochCheckpoint() throws props.setProperty(sqlFileSourceConfig, UtilitiesTestBase.basePath + "/sql-file-based-source.sql"); props.setProperty(sqlFileSourceConfigEmitChkPointConf, "true"); - sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider); + sqlFileSource = new SqlFileBasedSource(props, jsc, sparkSession, schemaProvider, metrics); Pair>, Checkpoint> nextBatch = sqlFileSource.fetchNextBatch(Option.empty(), Long.MAX_VALUE); assertEquals(10000, nextBatch.getLeft().get().count()); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlSource.java index 146c5253d5f5b..8a393e8b982eb 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlSource.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.streamer.SourceFormatAdapter; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; @@ -43,6 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; /** * Test against {@link SqlSource}. @@ -57,6 +59,7 @@ public class TestSqlSource extends UtilitiesTestBase { private TypedProperties props; private SqlSource sqlSource; private SourceFormatAdapter sourceFormatAdapter; + private final HoodieIngestionMetrics metrics = mock(HoodieIngestionMetrics.class); @BeforeAll public static void initClass() throws Exception { @@ -105,7 +108,7 @@ private void generateTestTable(String filename, String instantTime, int n) throw @Test public void testSqlSourceAvroFormat() throws IOException { props.setProperty(sqlSourceConfig, "select * from test_sql_table"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); // Test fetching Avro format @@ -128,7 +131,7 @@ public void testSqlSourceAvroFormat() throws IOException { @Test public void testSqlSourceRowFormat() throws IOException { props.setProperty(sqlSourceConfig, "select * from test_sql_table"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); // Test fetching Row format @@ -146,7 +149,7 @@ public void testSqlSourceRowFormat() throws IOException { @Test public void testSqlSourceCheckpoint() throws IOException { props.setProperty(sqlSourceConfig, "select * from test_sql_table where 1=0"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); InputBatch> fetch1AsRows = @@ -163,7 +166,7 @@ public void testSqlSourceCheckpoint() throws IOException { @Test public void testSqlSourceMoreRecordsThanSourceLimit() throws IOException { props.setProperty(sqlSourceConfig, "select * from test_sql_table"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); InputBatch> fetch1AsRows = @@ -180,7 +183,7 @@ public void testSqlSourceMoreRecordsThanSourceLimit() throws IOException { @Test public void testSqlSourceZeroRecord() throws IOException { props.setProperty(sqlSourceConfig, "select * from test_sql_table where 1=0"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); InputBatch> fetch1AsRows = @@ -197,7 +200,7 @@ public void testSqlSourceZeroRecord() throws IOException { @Test public void testSqlSourceInvalidTable() throws IOException { props.setProperty(sqlSourceConfig, "select * from not_exist_sql_table"); - sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider); + sqlSource = new SqlSource(props, jsc, sparkSession, schemaProvider, metrics); sourceFormatAdapter = new SourceFormatAdapter(sqlSource); assertThrows( diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java index 32480348a5654..70ec4e708b9b3 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java @@ -168,7 +168,7 @@ void testSqlFileBasedSource(int writeTableVersion, InputCheckpointKind inputKind TypedProperties props = propsWith(writeTableVersion); props.setProperty("hoodie.streamer.source.sql.file", sqlFile.toString()); props.setProperty("hoodie.streamer.source.sql.checkpoint.emit", "true"); - SqlFileBasedSource source = new SqlFileBasedSource(props, jsc, spark, null); + SqlFileBasedSource source = new SqlFileBasedSource(props, jsc, spark, null, null); Pair>, Checkpoint> result = invokeRowSourceFetch(source, makeInputCheckpoint(inputKind, "k")); assertV1(result.getRight()); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java index 661216053b239..3fd2068c91003 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java @@ -37,6 +37,7 @@ import org.apache.hudi.testutils.HoodieClientTestUtils; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamerTestBase; +import org.apache.hudi.utilities.ingestion.HoodieIngestionException; import org.apache.hudi.utilities.sources.MockGeneralHoodieIncrSource; import org.apache.hudi.utilities.sources.S3EventsHoodieIncrSourceHarness; @@ -191,8 +192,10 @@ public void testSyncE2ENoPrevCkpThenSyncMultipleTimes() throws Exception { HoodieDeltaStreamer.Config cfg = createConfig(basePath(), null, sourceClass); cfg.checkpoint = "overrideWhenAutoUpgradingWouldFail"; ds = new HoodieDeltaStreamer(cfg, jsc, Option.of(props)); - Exception ex = assertThrows(HoodieUpgradeDowngradeException.class, ds::sync); - assertTrue(ex.getMessage().contains("When upgrade/downgrade is happening, please avoid setting --checkpoint option and --ignore-checkpoint for your delta streamers.")); + Exception ex = assertThrows(HoodieIngestionException.class, ds::sync); + Throwable cause = ex.getCause(); + assertTrue(cause instanceof HoodieUpgradeDowngradeException, "Expected cause to be HoodieUpgradeDowngradeException but was: " + cause.getClass()); + assertTrue(cause.getMessage().contains("When upgrade/downgrade is happening, please avoid setting --checkpoint option and --ignore-checkpoint for your delta streamers.")); // No changes to the timeline / table config. metaClient.reloadActiveTimeline(); assertEquals(metaClient.getActiveTimeline().lastInstant().get(), instantAfterFirstRound); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerMetrics.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerMetrics.java index 8a45c79e73486..4fb7db1cd634a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerMetrics.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerMetrics.java @@ -68,4 +68,51 @@ public void testHoodieStreamerMetricsForErrorTableIfDisabled() { metrics.updateErrorTableCommitDuration(0L); assertNull(metrics.getMetrics()); } + + @Test + public void testEmitStreamerJobSuccessMetrics() { + HoodieMetricsConfig metricsConfig = HoodieMetricsConfig.newBuilder() + .on(true) + .withPath("/tmp/path3") + .withReporterType("INMEMORY") + .build(); + HoodieStreamerMetrics metrics = new HoodieStreamerMetrics( + metricsConfig, HoodieStorageUtils.getStorage(getDefaultStorageConf())); + metrics.emitStreamerJobSuccessMetrics(); + MetricRegistry registry = metrics.getMetrics().getRegistry(); + assertEquals(1, registry.getGauges().size()); + assertEquals(".deltastreamer.success", registry.getGauges().firstKey()); + assertEquals(1L, registry.getGauges().get(".deltastreamer.success").getValue()); + } + + @Test + public void testEmitStreamerJobFailedMetrics() { + HoodieMetricsConfig metricsConfig = HoodieMetricsConfig.newBuilder() + .on(true) + .withPath("/tmp/path4") + .withReporterType("INMEMORY") + .build(); + HoodieStreamerMetrics metrics = new HoodieStreamerMetrics( + metricsConfig, HoodieStorageUtils.getStorage(getDefaultStorageConf())); + metrics.emitStreamerJobFailedMetrics(); + MetricRegistry registry = metrics.getMetrics().getRegistry(); + assertEquals(1, registry.getGauges().size()); + assertEquals(".deltastreamer.failure", registry.getGauges().firstKey()); + assertEquals(1L, registry.getGauges().get(".deltastreamer.failure").getValue()); + } + + @Test + public void testEmitStreamerJobMetricsIfDisabled() { + HoodieMetricsConfig metricsConfig = HoodieMetricsConfig.newBuilder() + .on(false) + .withPath("/tmp/path5") + .withReporterType("INMEMORY") + .build(); + HoodieStreamerMetrics metrics = new HoodieStreamerMetrics( + metricsConfig, HoodieStorageUtils.getStorage(getDefaultStorageConf())); + // Should not throw when metrics are disabled + metrics.emitStreamerJobSuccessMetrics(); + metrics.emitStreamerJobFailedMetrics(); + assertNull(metrics.getMetrics()); + } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java index 4a851e7c0a066..42499d8c76d48 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSync.java @@ -37,6 +37,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.hadoop.HoodieHadoopStorage; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; +import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics; import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.InputBatch; import org.apache.hudi.utilities.transform.Transformer; @@ -56,6 +57,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -76,6 +78,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -473,5 +476,35 @@ void testParseOverridingMergeConfigsWithMixedConfigs() { assertEquals("org.apache.hudi.common.model.OverwriteWithLatestPayload", triple.getMiddle()); assertEquals("any_id", triple.getRight()); } + + @Test + void testReportSuccessMetricsDelegatesToMetrics() throws Exception { + StreamSync streamSync = mock(StreamSync.class); + HoodieIngestionMetrics mockMetrics = mock(HoodieIngestionMetrics.class); + Field metricsField = StreamSync.class.getDeclaredField("metrics"); + metricsField.setAccessible(true); + metricsField.set(streamSync, mockMetrics); + doCallRealMethod().when(streamSync).reportSuccessMetrics(); + + streamSync.reportSuccessMetrics(); + + verify(mockMetrics, times(1)).emitStreamerJobSuccessMetrics(); + verify(mockMetrics, never()).emitStreamerJobFailedMetrics(); + } + + @Test + void testReportFailureMetricsDelegatesToMetrics() throws Exception { + StreamSync streamSync = mock(StreamSync.class); + HoodieIngestionMetrics mockMetrics = mock(HoodieIngestionMetrics.class); + Field metricsField = StreamSync.class.getDeclaredField("metrics"); + metricsField.setAccessible(true); + metricsField.set(streamSync, mockMetrics); + doCallRealMethod().when(streamSync).reportFailureMetrics(); + + streamSync.reportFailureMetrics(); + + verify(mockMetrics, times(1)).emitStreamerJobFailedMetrics(); + verify(mockMetrics, never()).emitStreamerJobSuccessMetrics(); + } } } From aca664d5abc59787e686bedc43ba9e350df81c0b Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Fri, 12 Jun 2026 16:14:30 -0700 Subject: [PATCH 066/255] feat: Add HUDI version and engine properties to commit metadata (#18183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add hudi.version and engine (SPARK/FLINK/JAVA) to every commit's extraMetadata so post-hoc debugging can answer "which writer wrote this commit?". Engine- specific properties (spark.application.id, java.version, os.name, etc.) and a configurable allowlist of HoodieWriteConfig values are also embeddable under new opt-in configs: - hoodie.commit.metadata.engine.properties.embed.enable (default false) - hoodie.write.config.keys.to.serialize.to.commit.metadata (default: small allowlist of operationally useful keys) Enrichment is centralized in CommitMetadataProperties.enrich() and invoked once via BaseHoodieClient.updateExtraMetadata() from the write-client commit path and the table-service schedule path (after early-return checks). Also enrich the hoodie.properties file comment line with hostname and hudi version. Fires only on properties (re)writes — table creation, schema updates, upgrades — not on every commit. Hostname is resolved once per JVM with a "unknown" fallback for restricted environments. HoodieVersion.get() memoizes the manifest read. No storage-format changes; no public-API breakage (HoodieEngineContext .getEngineProperties() ships with a default empty-map implementation). Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: sivabalan (cherry picked from commit cde86ffd64a4aadb55ec049940713d199d92eb03) --- .../apache/hudi/client/BaseHoodieClient.java | 4 + .../client/BaseHoodieTableServiceClient.java | 2 + .../hudi/client/BaseHoodieWriteClient.java | 1 + .../hudi/client/CommitMetadataProperties.java | 145 +++++++++++++++ .../client/TestCommitMetadataProperties.java | 170 ++++++++++++++++++ .../common/HoodieJavaEngineContext.java | 25 +++ .../common/HoodieSparkEngineContext.java | 14 ++ .../java/org/apache/hudi/HoodieVersion.java | 43 +++-- .../common/engine/HoodieEngineContext.java | 16 ++ .../hudi/common/table/HoodieTableConfig.java | 34 +++- .../streamer/TestHoodieIncrSourceE2E.java | 6 +- .../TestHoodieIncrSourceE2EAutoUpgrade.java | 6 +- 12 files changed, 452 insertions(+), 14 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java index 002ac1e7454f8..c04454bf08570 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java @@ -458,4 +458,8 @@ private static Map collectRollingMetadataFromTimeline( return foundRollingMetadata; } + + protected Option> updateExtraMetadata(Option> extraMetadata) { + return CommitMetadataProperties.enrich(extraMetadata, config, context); + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index 17106d8d940e5..01105c8bd8d76 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -726,6 +726,8 @@ Option scheduleTableServiceInternal(Option providedInstantTime, // so it is handled differently to avoid locking for planning. return scheduleCleaning(createTable(config, storageConf), providedInstantTime); } + // Only enrich metadata after early-return checks, when we're actually going to use it + extraMetadata = updateExtraMetadata(extraMetadata); Option lastCompletedInstant = lastCompletedTxnAndMetadata.isPresent() ? Option.of(lastCompletedTxnAndMetadata.get().getLeft()) : Option.empty(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 76ffe51a256d2..8284f61baa15f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -254,6 +254,7 @@ public boolean commitStats(String instantTime, TableWriteStats tableWriteStats, if (!config.allowEmptyCommit() && tableWriteStats.isEmptyDataTableWriteStats()) { return true; } + extraMetadata = updateExtraMetadata(extraMetadata); log.info("Committing {} action {}", instantTime, commitActionType); // Create a Hoodie table which encapsulated the commits and files visible HoodieTable table = hoodieTableOpt.orElse(createTable(config)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java new file mode 100644 index 0000000000000..3a7b79812c8e7 --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.HoodieVersion; +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Enriches the {@code extraMetadata} map persisted with every commit, with version, engine, and + * (optionally) engine-specific properties and a configurable subset of {@link HoodieWriteConfig} + * values. + * + *

    Key namespacing: + *

    + */ +public class CommitMetadataProperties { + + static final String HUDI_VERSION_KEY = "hudi.version"; + static final String ENGINE_KEY = "engine"; + static final String CONFIG_KEY_PREFIX = "config."; + + /** + * Default allowlist of write-config keys serialized into commit metadata. These are values that + * change across jobs/runs but aren't already captured in {@code hoodie.properties}, so they're + * useful for after-the-fact debugging. Intentionally excludes immutable table identity + * (already in {@code hoodie.properties}) and per-record/sensitive values. + */ + private static final String DEFAULT_WRITE_CONFIG_KEYS = String.join(",", + Arrays.asList( + "hoodie.datasource.write.operation", + "hoodie.insert.shuffle.parallelism", + "hoodie.upsert.shuffle.parallelism", + "hoodie.bulkinsert.shuffle.parallelism", + "hoodie.delete.shuffle.parallelism", + "hoodie.write.concurrency.mode", + "hoodie.metadata.enable")); + + /** + * When enabled, engine-specific properties supplied by + * {@link HoodieEngineContext#getEngineProperties()} are embedded into commit metadata for + * debugging (e.g. {@code spark.application.id}, {@code spark.user}). {@code hudi.version} and + * {@code engine} are always embedded regardless of this flag. + * + *

    Default is {@code false} since these add per-commit growth to the timeline. Long-running + * ingestion workloads writing many commits should leave this off unless debugging. + */ + public static final ConfigProperty EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA = + ConfigProperty + .key("hoodie.commit.metadata.engine.properties.embed.enable") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("When enabled, engine-specific properties (e.g. spark.application.id, " + + "spark.user, java.version) are embedded into commit metadata for debugging. " + + "hudi.version and engine name are always embedded regardless of this flag."); + + /** + * Comma-separated list of {@link HoodieWriteConfig} keys whose values should be serialized into + * commit metadata under the {@code config.} prefix. Use with care: every key listed here + * adds an entry to every commit, which lives forever in the active and archived timeline. + * + *

    Empty value disables config-key serialization entirely (only {@code hudi.version} and + * {@code engine} are emitted). + */ + public static final ConfigProperty WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA = + ConfigProperty + .key("hoodie.write.config.keys.to.serialize.to.commit.metadata") + .defaultValue(DEFAULT_WRITE_CONFIG_KEYS) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Comma-separated list of write-config keys whose values are " + + "serialized into the extraMetadata map of every commit (under the 'config.' " + + "prefix). Set to empty to skip config-key serialization entirely. Avoid adding " + + "keys whose values may contain credentials or large payloads, since commit " + + "metadata is persisted in the timeline."); + + public static Option> enrich(Option> extraMetadata, + HoodieWriteConfig config, + HoodieEngineContext context) { + Map newMetadata = new HashMap<>(); + if (extraMetadata.isPresent()) { + newMetadata.putAll(extraMetadata.get()); + } + + newMetadata.put(HUDI_VERSION_KEY, HoodieVersion.get()); + newMetadata.put(ENGINE_KEY, config.getEngineType().name()); + + if (config.getBoolean(EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA)) { + newMetadata.putAll(context.getEngineProperties()); + } + + for (String key : parseConfigKeys(config.getString(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA))) { + String value = config.getString(key); + if (!StringUtils.isNullOrEmpty(value)) { + newMetadata.put(CONFIG_KEY_PREFIX + key, value); + } + } + + return Option.of(newMetadata); + } + + private static List parseConfigKeys(String csv) { + if (StringUtils.isNullOrEmpty(csv)) { + return Collections.emptyList(); + } + return Arrays.stream(csv.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java new file mode 100644 index 0000000000000..acd83424ff3ea --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.hudi.client.CommitMetadataProperties.CONFIG_KEY_PREFIX; +import static org.apache.hudi.client.CommitMetadataProperties.EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA; +import static org.apache.hudi.client.CommitMetadataProperties.ENGINE_KEY; +import static org.apache.hudi.client.CommitMetadataProperties.HUDI_VERSION_KEY; +import static org.apache.hudi.client.CommitMetadataProperties.WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestCommitMetadataProperties { + + /** Always-emitted keys (hudi.version, engine) are present even when input is empty. */ + @Test + void enrich_emptyInput_emitsVersionAndEngine() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Passing an immutable map must not throw — yihua's defensive-copy fix. */ + @Test + void enrich_immutableInputMap_doesNotThrow() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + Map input = Collections.unmodifiableMap( + Collections.singletonMap("caller.key", "caller.value")); + + Map result = CommitMetadataProperties.enrich(Option.of(input), config, context).get(); + + assertEquals("caller.value", result.get("caller.key")); + assertTrue(input.equals(Collections.singletonMap("caller.key", "caller.value")), + "Input map must not be mutated"); + } + + /** Default (opt-in flag = false) suppresses engine-supplied keys. */ + @Test + void enrich_engineEmbedFlagOff_omitsEngineSuppliedKeys() { + HoodieWriteConfig config = newConfig(new Properties()); + Map engineProps = new HashMap<>(); + engineProps.put("spark.application.id", "app-123"); + HoodieEngineContext context = newContext(engineProps); + + Map result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertFalse(result.containsKey("spark.application.id"), + "Engine-supplied keys must be omitted when embed flag is off"); + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Opt-in flag = true emits engine-supplied keys. */ + @Test + void enrich_engineEmbedFlagOn_emitsEngineSuppliedKeys() { + Properties props = new Properties(); + props.put(EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA.key(), "true"); + HoodieWriteConfig config = newConfig(props); + Map engineProps = new HashMap<>(); + engineProps.put("spark.application.id", "app-123"); + engineProps.put("spark.user", "sivabalan"); + HoodieEngineContext context = newContext(engineProps); + + Map result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertEquals("app-123", result.get("spark.application.id")); + assertEquals("sivabalan", result.get("spark.user")); + } + + /** Config-key allowlist serializes present values; skips truly absent keys (no default). */ + @Test + void enrich_writeConfigKeyAllowlist_emitsPresentNonEmptyValues() { + Properties props = new Properties(); + props.put(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA.key(), + "hoodie.metadata.enable,hoodie.does.not.exist"); + props.put("hoodie.metadata.enable", "true"); + HoodieWriteConfig config = newConfig(props); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertEquals("true", result.get(CONFIG_KEY_PREFIX + "hoodie.metadata.enable")); + assertFalse(result.containsKey(CONFIG_KEY_PREFIX + "hoodie.does.not.exist"), + "Keys absent from the config must not be serialized"); + } + + /** Empty allowlist disables config-key serialization entirely. */ + @Test + void enrich_emptyConfigKeyList_emitsNoConfigKeys() { + Properties props = new Properties(); + props.put(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA.key(), ""); + props.put("hoodie.metadata.enable", "true"); + HoodieWriteConfig config = newConfig(props); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + long configKeyCount = result.keySet().stream().filter(k -> k.startsWith(CONFIG_KEY_PREFIX)).count(); + assertEquals(0L, configKeyCount, "No config.* keys when allowlist is empty"); + // hudi.version and engine still always emitted + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Caller-provided extra metadata is preserved alongside enrichment. */ + @Test + void enrich_preservesCallerProvidedKeys() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + Map input = new HashMap<>(); + input.put("caller.key1", "caller.value1"); + input.put("caller.key2", "caller.value2"); + + Map result = CommitMetadataProperties.enrich(Option.of(input), config, context).get(); + + assertEquals("caller.value1", result.get("caller.key1")); + assertEquals("caller.value2", result.get("caller.key2")); + assertNotNull(result.get(HUDI_VERSION_KEY)); + } + + private static HoodieWriteConfig newConfig(Properties overrides) { + Properties props = new Properties(); + props.put(HoodieWriteConfig.BASE_PATH.key(), "/tmp/test-commit-metadata-properties"); + props.putAll(overrides); + return HoodieWriteConfig.newBuilder().withProperties(props).build(); + } + + private static HoodieEngineContext newContext(Map engineProperties) { + HoodieEngineContext context = mock(HoodieEngineContext.class); + when(context.getEngineProperties()).thenReturn(engineProperties); + return context; + } +} diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java index 8da9e2aca0e15..a24d6fb4c25d9 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -192,6 +193,30 @@ public void cancelAllJobs() { // no operation for now } + // Allowlist of safe system properties to include in commit metadata. Avoid wildcarding system + // properties since callers may pass credentials via -D flags (e.g. -Ddb.password=...). + private static final String[] SAFE_SYSTEM_PROPERTIES = { + "java.version", + "java.vendor", + "java.vm.name", + "java.vm.version", + "os.name", + "os.version", + "os.arch" + }; + + @Override + public Map getEngineProperties() { + Map info = new HashMap<>(); + for (String property : SAFE_SYSTEM_PROPERTIES) { + String value = System.getProperty(property); + if (value != null) { + info.put(property, value); + } + } + return info; + } + @Override public O aggregate(HoodieData data, O zeroValue, Functions.Function2 seqOp, Functions.Function2 combOp) { return data.collectAsList().stream().reduce(zeroValue, seqOp::apply, combOp::apply); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java index d764dcd3cd54b..0e4fae4f70e6e 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java @@ -408,4 +408,18 @@ public , V extends Comparable> HoodiePairData r new ConditionalRangePartitioner.CompositeKeyComparator<>()) .mapToPair(e -> e._1)); } + + @Override + public Map getEngineProperties() { + Map info = new HashMap<>(); + info.put("spark.application.id", javaSparkContext.sc().applicationId()); + info.put("spark.user", javaSparkContext.sparkUser()); + info.put("spark.master", javaSparkContext.master()); + info.put("spark.application", javaSparkContext.appName()); + info.put("spark.version", javaSparkContext.version()); + info.put("spark.defaultParallelism", String.valueOf(javaSparkContext.defaultParallelism())); + info.put("spark.defaultMinPartitions", String.valueOf(javaSparkContext.defaultMinPartitions())); + info.put("spark.executor.instances", javaSparkContext.getConf().get("spark.executor.instances", "")); + return info; + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java b/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java index dd88836468c92..23cee10158417 100644 --- a/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java @@ -33,23 +33,46 @@ public final class HoodieVersion { public static final String HOODIE_WRITER_VERSION = "hudi_writer_version"; + // Cached result of reading version from the manifest. Null means "not loaded yet". An empty + // string means "manifest absent or unreadable" — fall back to HOODIE_DEFAULT_VERSION so tests + // that swap the default via setVersionOverride continue to work. + private static volatile String cachedManifestVersion = null; + /** * Returns the complete version of HUDI code * Example: 0.12.2 or 0.12.3-snapshot */ public static String get() { - String hudiPropertiesFilePath = "META-INF/maven/org.apache.hudi/hudi-common/pom.properties"; - try (InputStream inputStream = HoodieVersion.class.getClassLoader().getResourceAsStream(hudiPropertiesFilePath)) { - Properties properties = new Properties(); - if (inputStream != null) { - properties.load(inputStream); - // Access properties - return properties.getProperty("version"); + String fromManifest = loadManifestVersion(); + return fromManifest.isEmpty() ? HOODIE_DEFAULT_VERSION : fromManifest; + } + + private static String loadManifestVersion() { + String local = cachedManifestVersion; + if (local != null) { + return local; + } + synchronized (HoodieVersion.class) { + if (cachedManifestVersion != null) { + return cachedManifestVersion; + } + String hudiPropertiesFilePath = "META-INF/maven/org.apache.hudi/hudi-common/pom.properties"; + String resolved = ""; + try (InputStream inputStream = HoodieVersion.class.getClassLoader().getResourceAsStream(hudiPropertiesFilePath)) { + if (inputStream != null) { + Properties properties = new Properties(); + properties.load(inputStream); + String version = properties.getProperty("version"); + if (version != null) { + resolved = version; + } + } + } catch (Exception ignored) { + // Ignoring the exception as there is a fallback to default version } - } catch (Exception ignored) { - // Ignoring the exception as there is as fallback to default version + cachedManifestVersion = resolved; + return resolved; } - return HOODIE_DEFAULT_VERSION; } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java index b4663509002d1..eea766b56dda0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java @@ -45,6 +45,7 @@ import lombok.Getter; import java.io.IOException; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -117,6 +118,21 @@ public abstract List reduceByKey( public abstract void cancelAllJobs(); + /** + * Returns engine-specific properties to be included in commit metadata for debugging. + *

    Contract: + *

      + *
    • Implementations must only return safe, non-sensitive values (no credentials, no PII).
      • + *
    • This is invoked on the driver, on every commit. It must be cheap and free of side effects.
      • + *
    • Must not reach into checkpoint / runtime state (e.g. for streaming engines like Flink, + * per-checkpoint metadata is set up via the coordinator, not here).
      • + *
    + * Default returns an empty map so external subclasses are not forced to implement this. + */ + public Map getEngineProperties() { + return Collections.emptyMap(); + } + /** * Returns the application id of the engine (e.g. Spark application id). * Used to populate lock metadata so lock holders can be identified. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 73703d70d85a4..aef84c9f8adc9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -18,6 +18,7 @@ package org.apache.hudi.common.table; +import org.apache.hudi.HoodieVersion; import org.apache.hudi.common.HoodieTableFormat; import org.apache.hudi.common.NativeTableFormat; import org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex; @@ -51,6 +52,7 @@ import org.apache.hudi.common.util.BinaryUtil; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.HoodieTableConfigUtils; +import org.apache.hudi.common.util.NetworkUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; @@ -126,6 +128,9 @@ + " initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.") public class HoodieTableConfig extends HoodieConfig { + // Cached hostname to avoid repeated synchronized network calls + private static volatile String cachedHostname = null; + public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; public static final String HOODIE_PROPERTIES_FILE_BACKUP = "hoodie.properties.backup"; public static final String HOODIE_WRITE_TABLE_NAME_KEY = "hoodie.datasource.write.table.name"; @@ -500,10 +505,10 @@ private static String storeProperties(Properties props, OutputStream outputStrea final String checksum; if (isValidChecksum(props)) { checksum = props.getProperty(TABLE_CHECKSUM.key()); - props.store(outputStream, "Updated at " + Instant.now()); + props.store(outputStream, getFileComment()); } else { Properties propsWithChecksum = getOrderedPropertiesWithTableChecksum(props); - propsWithChecksum.store(outputStream, "Properties saved on " + Instant.now()); + propsWithChecksum.store(outputStream, getFileComment()); checksum = propsWithChecksum.getProperty(TABLE_CHECKSUM.key()); props.setProperty(TABLE_CHECKSUM.key(), checksum); } @@ -1385,6 +1390,31 @@ public Map propsMap() { .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue()))); } + /** + * Returns the cached hostname, fetching it lazily on first call. + * Falls back to "unknown" if network resolution fails. + */ + private static String getHostnameSafe() { + if (cachedHostname == null) { + synchronized (HoodieTableConfig.class) { + if (cachedHostname == null) { + try { + cachedHostname = NetworkUtils.getHostname(); + } catch (Exception e) { + log.warn("Failed to resolve hostname, using 'unknown'", e); + cachedHostname = "unknown"; + } + } + } + } + return cachedHostname; + } + + public static String getFileComment() { + return String.format("Updated at %s, host=%s, hudi_version=%s", + Instant.now(), getHostnameSafe(), HoodieVersion.get()); + } + /** * @deprecated Use {@link #BASE_FILE_FORMAT} and its methods. */ diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java index 1b010efab7cbc..64c432474953a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java @@ -108,7 +108,11 @@ public void verifyLastInstantCommitMetadata(Map expectedMetadata Option metadata = HoodieClientTestUtils.getCommitMetadataForInstant( metaClient, metaClient.getActiveTimeline().lastInstant().get()); assertFalse(metadata.isEmpty()); - assertEquals(metadata.get().getExtraMetadata(), expectedMetadata); + // Assert expected entries are a subset of the actual extra metadata. CommitMetadataProperties + // also enriches commit metadata with hudi.version, engine, and config.* entries on every write, + // so the actual map is a superset. + Map actual = metadata.get().getExtraMetadata(); + expectedMetadata.forEach((k, v) -> assertEquals(v, actual.get(k), "extraMetadata[" + k + "]")); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java index 3fd2068c91003..52a6734ec3317 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java @@ -137,7 +137,11 @@ public void verifyLastInstantCommitMetadata(Map expectedMetadata Option metadata = HoodieClientTestUtils.getCommitMetadataForInstant( metaClient, metaClient.getActiveTimeline().lastInstant().get()); assertFalse(metadata.isEmpty()); - assertEquals(expectedMetadata, metadata.get().getExtraMetadata()); + // Assert expected entries are a subset of the actual extra metadata. CommitMetadataProperties + // also enriches commit metadata with hudi.version, engine, and config.* entries on every write, + // so the actual map is a superset. + Map actual = metadata.get().getExtraMetadata(); + expectedMetadata.forEach((k, v) -> assertEquals(v, actual.get(k), "extraMetadata[" + k + "]")); } /** From d33079932ac1db8ed0ded6e93b504871e869e289 Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Fri, 12 Jun 2026 16:34:13 -0700 Subject: [PATCH 067/255] Fix NPE in getInputFileSlices when RO path filter returns empty partition (#18639) generatePartitionFileSlicesPostROTablePathFilter built its result map by iterating over the file list, so a partition with no files produced no entry. The caller getInputFileSlices then did Collectors.toMap(identity, cache::get) where cache::get returned null for the missing partition, and Collectors.toMap rejects null values via Objects.requireNonNull. Pre-populate the result map with empty file-slice lists for every input partition before processing files. This restores the contract already honored by filterFiles (the non-RO path), which iterates over partitions and naturally returns an entry per partition. Co-authored-by: Claude Opus 4.7 (cherry picked from commit fedf24ffc933f7f2c962ae091420fddbb543b53b) --- .../apache/hudi/BaseHoodieTableFileIndex.java | 11 ++- .../hudi/BaseHoodieTableFileIndexTest.java | 69 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java index cbe0e3ccf64a1..ff3e7dbbbe001 100644 --- a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java @@ -314,6 +314,11 @@ private Map> generatePartitionFileSlicesPostROTab Map partitionsMap = new HashMap<>(); partitions.forEach(p -> partitionsMap.put(p.path, p)); Map> partitionToFileSlices = new HashMap<>(); + // Pre-populate so partitions with no files still appear in the result map. + // Without this, the caller's Collectors.toMap(identity, cache::get) NPEs on empty partitions + // because cache.get returns null and toMap rejects null values. This matches the contract + // already honored by filterFiles, which iterates over partitions rather than over files. + partitions.forEach(p -> partitionToFileSlices.put(p, Collections.emptyList())); for (StoragePathInfo pathInfo : allFiles) { // Create FileSlice obj from StoragePathInfo. @@ -326,7 +331,11 @@ private Map> generatePartitionFileSlicesPostROTab // Add the FileSlice to partitionToFileSlices PartitionPath partitionPathObj = partitionsMap.get(relPartitionPath); if (partitionPathObj != null) { - List fileSlices = partitionToFileSlices.computeIfAbsent(partitionPathObj, k -> new ArrayList<>()); + List fileSlices = partitionToFileSlices.get(partitionPathObj); + if (fileSlices.isEmpty()) { + fileSlices = new ArrayList<>(); + partitionToFileSlices.put(partitionPathObj, fileSlices); + } fileSlices.add(fileSlice); } else { log.warn("Could not find partition path object for relative path: {}. Skipping file: {}", diff --git a/hudi-common/src/test/java/org/apache/hudi/BaseHoodieTableFileIndexTest.java b/hudi-common/src/test/java/org/apache/hudi/BaseHoodieTableFileIndexTest.java index a6edb8c64a88d..285e7698c5f51 100644 --- a/hudi-common/src/test/java/org/apache/hudi/BaseHoodieTableFileIndexTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/BaseHoodieTableFileIndexTest.java @@ -18,15 +18,25 @@ package org.apache.hudi; +import org.apache.hudi.BaseHoodieTableFileIndex.PartitionPath; import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; public class BaseHoodieTableFileIndexTest { @@ -58,4 +68,63 @@ public void testGetMetadataConfigReturnsFieldValue() throws Exception { assertEquals(true, result.isBloomFilterIndexEnabled(), "Bloom filter index should be enabled"); assertEquals(true, result.isColumnStatsIndexEnabled(), "Column stats index should be enabled"); } + + /** + * Regression test for the empty-partition NPE that surfaces in {@code getInputFileSlices} + * when the {@code hoodie.datasource.read.file.index.list.file.statuses.using.ro.path.filter} + * code path is exercised on a COW (or READ_OPTIMIZED) table that contains a partition + * holding zero base files. + * + *

    Before the fix, {@link BaseHoodieTableFileIndex#generatePartitionFileSlicesPostROTablePathFilter} + * built its result map by iterating over the file list, so a partition with no files received + * no entry. The downstream {@code Collectors.toMap(identity, p -> cache.get(p))} in + * {@code getInputFileSlices} then dereferenced a null value and threw NPE inside + * {@code Collectors.uniqKeysMapAccumulator}. + * + *

    After the fix, every input partition appears in the returned map (with an empty list + * for empty partitions), preserving the contract already honored by the non-RO path + * ({@code filterFiles}). + */ + @Test + public void testGeneratePartitionFileSlicesPostROTablePathFilterIncludesEmptyPartitions() throws Exception { + BaseHoodieTableFileIndex fileIndex = mock(BaseHoodieTableFileIndex.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + + StoragePath basePath = new StoragePath("/tmp/hudi_empty_partition_test"); + Field basePathField = BaseHoodieTableFileIndex.class.getDeclaredField("basePath"); + basePathField.setAccessible(true); + basePathField.set(fileIndex, basePath); + + PartitionPath partitionWithFiles = new PartitionPath("dt=2026-01-01", new Object[]{"2026-01-01"}); + PartitionPath emptyPartition = new PartitionPath("dt=2026-01-02", new Object[]{"2026-01-02"}); + PartitionPath anotherEmpty = new PartitionPath("dt=2026-01-03", new Object[]{"2026-01-03"}); + List partitions = Arrays.asList(partitionWithFiles, emptyPartition, anotherEmpty); + + StoragePathInfo file = new StoragePathInfo( + new StoragePath(basePath, "dt=2026-01-01/file-0_0-0-0_20260101000000001.parquet"), + 100L, false, (short) 1, 1024L, 0L); + List allFiles = Collections.singletonList(file); + + Method generateMethod = BaseHoodieTableFileIndex.class.getDeclaredMethod( + "generatePartitionFileSlicesPostROTablePathFilter", List.class, List.class); + generateMethod.setAccessible(true); + @SuppressWarnings("unchecked") + Map> result = + (Map>) generateMethod.invoke(fileIndex, partitions, allFiles); + + assertNotNull(result, "Result map must not be null"); + assertEquals(3, result.size(), + "Result map must contain an entry for every input partition, including empty ones"); + assertTrue(result.containsKey(partitionWithFiles)); + assertTrue(result.containsKey(emptyPartition), + "Empty partition must appear in the result so getInputFileSlices does not NPE"); + assertTrue(result.containsKey(anotherEmpty), + "Empty partition must appear in the result so getInputFileSlices does not NPE"); + assertEquals(1, result.get(partitionWithFiles).size(), + "Partition with files should retain its file slice"); + assertTrue(result.get(emptyPartition).isEmpty(), + "Empty partition's file slice list must be present and empty (not null, not missing)"); + assertTrue(result.get(anotherEmpty).isEmpty(), + "Empty partition's file slice list must be present and empty (not null, not missing)"); + } } \ No newline at end of file From 73ec4812c2f6e6259c57d23b4308f482881403c9 Mon Sep 17 00:00:00 2001 From: ashokkumar-allu <65997235+ashokkumar-allu@users.noreply.github.com> Date: Sat, 13 Jun 2026 10:09:16 +0800 Subject: [PATCH 068/255] fix: Isolate classloader-aware parallel execution in HoodiePreCommitValidatorEngineContext (#18585) Move the classloader-aware parallel execution logic out of HoodieLocalEngineContext into a new dedicated ExecutorServiceBasedEngineContext that extends HoodieLocalEngineContext and overrides map() using an ExecutorService-backed thread pool. This avoids polluting HoodieLocalEngineContext with pre-commit-specific concerns and follows the reviewer's suggestion. Co-authored-by: gallu (cherry picked from commit 0915f7776be2a7d08ea0f23000deeceff7caa8c6) --- .../client/utils/SparkValidatorUtils.java | 38 +-- .../validator/SparkPreCommitValidator.java | 12 + .../client/utils/TestSparkValidatorUtils.java | 211 ++++++++++++- .../TestSparkPreCommitValidator.java | 143 +++++++++ .../ExecutorServiceBasedEngineContext.java | 290 ++++++++++++++++++ ...TestExecutorServiceBasedEngineContext.java | 112 +++++++ .../engine/TestHoodieLocalEngineContext.java | 12 +- 7 files changed, 793 insertions(+), 25 deletions(-) create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/validator/TestSparkPreCommitValidator.java create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/engine/ExecutorServiceBasedEngineContext.java create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestExecutorServiceBasedEngineContext.java diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java index 40ee7d8cefff9..233f622beff68 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java @@ -23,6 +23,7 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.client.validator.SparkPreCommitValidator; import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.engine.ExecutorServiceBasedEngineContext; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.BaseFile; import org.apache.hudi.common.model.HoodieWriteStat; @@ -50,7 +51,6 @@ import java.util.Arrays; import java.util.List; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -83,7 +83,7 @@ public static void runValidators(HoodieWriteConfig config, Dataset afterState = getRecordsFromPendingCommits(sqlContext, partitionsModified, writeMetadata, table, instantTime); Dataset beforeState = getRecordsFromCommittedFiles(sqlContext, partitionsModified, table, afterState.schema()); - Stream validators = Arrays.stream(config.getPreCommitValidators().split(",")) + List validators = Arrays.stream(config.getPreCommitValidators().split(",")) .map(String::trim) .filter(s -> !s.isEmpty()) .flatMap(validatorClass -> { @@ -105,10 +105,12 @@ public static void runValidators(HoodieWriteConfig config, } catch (ReflectiveOperationException e) { throw new HoodieValidationException("Failed to instantiate validator: " + validatorClass, e); } - }); + }) + .collect(Collectors.toList()); - boolean allSuccess = validators.map(v -> runValidatorAsync(v, writeMetadata, beforeState, afterState, instantTime)).map(CompletableFuture::join) - .reduce(true, Boolean::logicalAnd); + boolean allSuccess = new ExecutorServiceBasedEngineContext(context.getStorageConf()) + .map(validators, v -> runValidator(v, writeMetadata, beforeState, afterState, instantTime), validators.size()) + .stream().reduce(true, Boolean::logicalAnd); if (allSuccess) { LOG.info("All validations succeeded"); @@ -120,20 +122,20 @@ public static void runValidators(HoodieWriteConfig config, } /** - * Run validators in a separate thread pool for parallelism. Each of validator can submit a distributed spark job if needed. + * Run a single validator synchronously in the calling thread; parallelism across validators is + * provided by the {@link ExecutorServiceBasedEngineContext#map} call site. Each validator may submit a distributed Spark + * job if needed. */ - private static CompletableFuture runValidatorAsync(SparkPreCommitValidator validator, HoodieWriteMetadata writeMetadata, - Dataset beforeState, Dataset afterState, String instantTime) { - return CompletableFuture.supplyAsync(() -> { - try { - validator.validate(instantTime, writeMetadata, beforeState, afterState); - LOG.info("validation complete for {}", validator.getClass().getName()); - return true; - } catch (HoodieValidationException e) { - LOG.error("validation failed for {}", validator.getClass().getName(), e); - return false; - } - }); + private static boolean runValidator(SparkPreCommitValidator validator, HoodieWriteMetadata> writeMetadata, + Dataset beforeState, Dataset afterState, String instantTime) { + try { + validator.validate(instantTime, writeMetadata, beforeState, afterState); + LOG.info("validation complete for {}", validator.getClass().getName()); + return true; + } catch (HoodieValidationException e) { + LOG.error("validation failed for {}", validator.getClass().getName(), e); + return false; + } } /** diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java index 7e1da34c24426..b06e80bf63922 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java @@ -82,6 +82,18 @@ public void validate(String instantTime, HoodieWriteMetadata writeResult, Dat HoodieTimer timer = HoodieTimer.start(); try { validateRecordsBeforeAndAfter(before, after, getPartitionsModified(writeResult)); + } catch (HoodieValidationException e) { + throw e; + } catch (RuntimeException e) { + // Unexpected bug (NPE, ClassCastException, etc.) — re-throw as-is so it propagates + // crash-loud with the original stack trace instead of being silently swallowed as a + // generic "validation failed" message. + log.error("Validator {} threw unexpected exception for instant {}", getClass().getName(), instantTime, e); + throw e; + } catch (Exception e) { + // Checked exception — promote to RuntimeException so it propagates crash-loud. + log.error("Validator {} threw unexpected checked exception for instant {}", getClass().getName(), instantTime, e); + throw new RuntimeException(e); } finally { long duration = timer.endTimer(); log.info(getClass() + " validator took " + duration + " ms" + ", metrics on? " + getWriteConfig().isMetricsOn()); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/utils/TestSparkValidatorUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/utils/TestSparkValidatorUtils.java index c53b5ceb6d1c0..ac54baea9c34c 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/utils/TestSparkValidatorUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/utils/TestSparkValidatorUtils.java @@ -20,18 +20,30 @@ import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.WriteClientTestUtils; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.validator.SparkPreCommitValidator; import org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodiePreCommitValidatorConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieValidationException; +import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.testutils.HoodieClientTestBase; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.apache.spark.api.java.JavaRDD; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests for {@link SparkValidatorUtils}. @@ -95,4 +107,201 @@ public void testSqlQueryValidatorWithNoRecords() throws Exception { "Should have 2 commits (one with data, one empty)"); } } + + /** + * Verifies that two custom validators are both invoked in parallel via ExecutorServiceBasedEngineContext + * without ClassNotFoundException, confirming that classloader-loaded user validator classes execute correctly. + */ + @Test + public void testTwoValidatorsBothInvoked() throws Exception { + CountingValidator.INVOCATION_COUNT.set(0); + + HoodieWriteConfig configWithTwoValidators = getConfigBuilder() + .withPreCommitValidatorConfig( + HoodiePreCommitValidatorConfig.newBuilder() + .withPreCommitValidator( + CountingValidator.class.getName() + "," + CountingValidator.class.getName()) + .build()) + .build(); + + try (SparkRDDWriteClient writeClient = getHoodieWriteClient(configWithTwoValidators)) { + String commit = "001"; + writeBatch( + writeClient, + commit, + "000", + Option.empty(), + "000", + 5, + generateWrapRecordsFn(false, configWithTwoValidators, dataGen::generateInserts), + SparkRDDWriteClient::bulkInsert, + true, + 5, + 5, + 1, + false, + INSTANT_GENERATOR); + } + + Assertions.assertEquals(2, CountingValidator.INVOCATION_COUNT.get(), + "Both configured validators must have been invoked in parallel"); + } + + /** + * Verifies that when a validator throws {@link HoodieValidationException}, it surfaces somewhere + * in the exception cause chain of the write operation. + * The write client wraps validator exceptions in a {@code HoodieInsertException}, so we walk + * the cause chain rather than asserting the top-level type. + */ + @Test + public void testValidatorFailurePropagatesException() throws Exception { + HoodieWriteConfig configWithFailingValidator = getConfigBuilder() + .withPreCommitValidatorConfig( + HoodiePreCommitValidatorConfig.newBuilder() + .withPreCommitValidator(FailingValidator.class.getName()) + .build()) + .build(); + + try (SparkRDDWriteClient writeClient = getHoodieWriteClient(configWithFailingValidator)) { + String commit = "001"; + Exception thrown = assertThrows(Exception.class, () -> + writeBatch( + writeClient, + commit, + "000", + Option.empty(), + "000", + 5, + generateWrapRecordsFn(false, configWithFailingValidator, dataGen::generateInserts), + SparkRDDWriteClient::bulkInsert, + true, + 5, + 5, + 1, + false, + INSTANT_GENERATOR), + "A failing validator must cause the write operation to throw"); + + // Walk the cause chain: bulkInsert wraps the HoodieValidationException in HoodieInsertException. + Throwable cause = thrown; + while (cause != null && !(cause instanceof HoodieValidationException)) { + cause = cause.getCause(); + } + Assertions.assertNotNull(cause, + "HoodieValidationException must appear somewhere in the exception cause chain"); + Assertions.assertInstanceOf(HoodieValidationException.class, cause); + } + } + + /** + * Verifies that when a validator throws an unexpected RuntimeException (e.g. NPE or + * IllegalStateException — a validator bug), the exception is NOT silently swallowed as a + * generic "validation failed" message. The original exception must appear in the cause chain + * so operators can diagnose the real problem. + */ + @Test + public void testUnexpectedValidatorExceptionIsNotSilenced() throws Exception { + HoodieWriteConfig configWithBuggyValidator = getConfigBuilder() + .withPreCommitValidatorConfig( + HoodiePreCommitValidatorConfig.newBuilder() + .withPreCommitValidator(BuggyValidator.class.getName()) + .build()) + .build(); + + try (SparkRDDWriteClient writeClient = getHoodieWriteClient(configWithBuggyValidator)) { + String commit = "001"; + Exception thrown = assertThrows(Exception.class, () -> + writeBatch( + writeClient, + commit, + "000", + Option.empty(), + "000", + 5, + generateWrapRecordsFn(false, configWithBuggyValidator, dataGen::generateInserts), + SparkRDDWriteClient::bulkInsert, + true, + 5, + 5, + 1, + false, + INSTANT_GENERATOR), + "A buggy validator must still cause the write to fail"); + + // The original IllegalStateException must be visible somewhere in the cause chain. + // It must NOT be silently converted into a plain "At least one pre-commit validation failed". + Throwable cause = thrown; + boolean foundOriginal = false; + while (cause != null) { + if (cause instanceof IllegalStateException + && "simulated bug in validator".equals(cause.getMessage())) { + foundOriginal = true; + break; + } + cause = cause.getCause(); + } + Assertions.assertTrue(foundOriginal, + "The original IllegalStateException from the buggy validator must appear in the cause chain, " + + "not be buried under a generic 'validation failed' message. Full exception: " + thrown); + } + } + + /** + * Minimal validator that records each invocation. Must be a public static class so that + * ReflectionUtils can instantiate it by name during runValidators. + */ + public static class CountingValidator> + extends SparkPreCommitValidator { + + static final AtomicInteger INVOCATION_COUNT = new AtomicInteger(0); + + public CountingValidator(HoodieSparkTable table, HoodieEngineContext context, + HoodieWriteConfig config) { + super(table, context, config); + } + + @Override + protected void validateRecordsBeforeAndAfter(Dataset before, Dataset after, + Set partitionsAffected) { + INVOCATION_COUNT.incrementAndGet(); + } + } + + /** + * Validator that always fails with {@link HoodieValidationException}. Must be a public static + * class so that ReflectionUtils can instantiate it by name during runValidators. + */ + public static class FailingValidator> + extends SparkPreCommitValidator { + + public FailingValidator(HoodieSparkTable table, HoodieEngineContext context, + HoodieWriteConfig config) { + super(table, context, config); + } + + @Override + protected void validateRecordsBeforeAndAfter(Dataset before, Dataset after, + Set partitionsAffected) { + throw new HoodieValidationException("intentional failure from FailingValidator"); + } + } + + /** + * Validator that throws an unexpected RuntimeException (simulates a validator bug such as NPE). + * Must be a public static class so that ReflectionUtils can instantiate it by name. + */ + public static class BuggyValidator> + extends SparkPreCommitValidator { + + public BuggyValidator(HoodieSparkTable table, HoodieEngineContext context, + HoodieWriteConfig config) { + super(table, context, config); + } + + @Override + protected void validateRecordsBeforeAndAfter(Dataset before, Dataset after, + Set partitionsAffected) { + throw new IllegalStateException("simulated bug in validator"); + } + } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/validator/TestSparkPreCommitValidator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/validator/TestSparkPreCommitValidator.java new file mode 100644 index 0000000000000..83ee85302cbe6 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/validator/TestSparkPreCommitValidator.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.validator; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieValidationException; +import org.apache.hudi.table.HoodieSparkTable; +import org.apache.hudi.table.action.HoodieWriteMetadata; + +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Collections; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +/** + * Unit tests for exception-handling behavior in {@link SparkPreCommitValidator#validate}. + */ +@ExtendWith(MockitoExtension.class) +public class TestSparkPreCommitValidator { + + @Mock + @SuppressWarnings("rawtypes") + private HoodieSparkTable table; + + @Mock + private HoodieEngineContext engineContext; + + @Mock + private HoodieWriteConfig writeConfig; + + @Mock + @SuppressWarnings("rawtypes") + private HoodieWriteMetadata writeMetadata; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + when(writeConfig.getTableName()).thenReturn("test-table"); + when(writeConfig.isMetricsOn()).thenReturn(false); + when(writeMetadata.getWriteStats()).thenReturn(Option.of(Collections.emptyList())); + } + + @Test + @SuppressWarnings("unchecked") + void testValidateSucceeds() { + SparkPreCommitValidator> validator = + new NoOpValidator(table, engineContext, writeConfig); + assertDoesNotThrow(() -> validator.validate("001", writeMetadata, null, null), + "validate must not throw when validateRecordsBeforeAndAfter completes normally"); + } + + @Test + @SuppressWarnings("unchecked") + void testValidateRethrowsUnexpectedRuntimeException() { + RuntimeException cause = new RuntimeException("disk full"); + SparkPreCommitValidator> validator = + new ThrowingValidator(table, engineContext, writeConfig, cause); + + RuntimeException ex = assertThrows(RuntimeException.class, + () -> validator.validate("001", writeMetadata, null, null)); + assertSame(cause, ex, + "unexpected RuntimeException must propagate as-is so the operator sees the original stack trace, " + + "not a generic 'validation failed' message"); + } + + @Test + @SuppressWarnings("unchecked") + void testValidateReThrowsValidationException() { + HoodieValidationException original = new HoodieValidationException("bad data"); + SparkPreCommitValidator> validator = + new ThrowingValidator(table, engineContext, writeConfig, original); + + HoodieValidationException ex = assertThrows(HoodieValidationException.class, + () -> validator.validate("001", writeMetadata, null, null)); + assertSame(original, ex, + "HoodieValidationException must be rethrown as-is without additional wrapping"); + } + + /** Minimal concrete validator that completes normally. */ + private static class NoOpValidator> + extends SparkPreCommitValidator { + + NoOpValidator(HoodieSparkTable table, HoodieEngineContext context, HoodieWriteConfig config) { + super(table, context, config); + } + + @Override + protected void validateRecordsBeforeAndAfter(Dataset before, Dataset after, + Set partitionsAffected) { + // no-op — validation always passes + } + } + + /** Minimal concrete validator that throws a fixed exception from validateRecordsBeforeAndAfter. */ + private static class ThrowingValidator> + extends SparkPreCommitValidator { + + private final RuntimeException toThrow; + + ThrowingValidator(HoodieSparkTable table, HoodieEngineContext context, + HoodieWriteConfig config, RuntimeException toThrow) { + super(table, context, config); + this.toThrow = toThrow; + } + + @Override + protected void validateRecordsBeforeAndAfter(Dataset before, Dataset after, + Set partitionsAffected) { + throw toThrow; + } + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/ExecutorServiceBasedEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/ExecutorServiceBasedEngineContext.java new file mode 100644 index 0000000000000..c6ba0d736acf2 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/ExecutorServiceBasedEngineContext.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.engine; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.data.HoodieAccumulator; +import org.apache.hudi.common.data.HoodieAtomicLongAccumulator; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.data.HoodieData.HoodieDataCacheKey; +import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.data.HoodieListPairData; +import org.apache.hudi.common.data.HoodiePairData; +import org.apache.hudi.common.function.FunctionWrapper; +import org.apache.hudi.common.function.SerializableBiFunction; +import org.apache.hudi.common.function.SerializableConsumer; +import org.apache.hudi.common.function.SerializableFunction; +import org.apache.hudi.common.function.SerializablePairFlatMapFunction; +import org.apache.hudi.common.function.SerializablePairFunction; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.Functions; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ImmutablePair; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.keygen.KeyGenerator; +import org.apache.hudi.storage.StorageConfiguration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A general-purpose {@link HoodieEngineContext} that executes all parallel operations on a + * dedicated classloader-aware {@link ExecutorService}, so classes resolved by the application + * classloader remain visible to worker threads on Java 11+. + * + *

    The pool is lazily created once per JVM (JLS §12.4.2) and shared across all instances. + * Worker threads are daemon threads and carry the classloader of this class, preventing + * {@code ClassNotFoundException} that occurs when the common {@link ForkJoinPool} is used + * because its workers do not inherit the submitting thread's context classloader. + */ +public class ExecutorServiceBasedEngineContext extends HoodieEngineContext { + + private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceBasedEngineContext.class); + + // Lazy-initialized, daemon, fixed thread pool whose workers carry the correct classloader. + // JLS §12.4.2 guarantees thread-safe initialization via class-loading locks. + private static class PoolHolder { + static final ExecutorService INSTANCE = createExecutorService(); + } + + private static ExecutorService createExecutorService() { + int parallelism = ForkJoinPool.commonPool().getParallelism(); + ClassLoader cl = ExecutorServiceBasedEngineContext.class.getClassLoader(); + ExecutorService executor = Executors.newFixedThreadPool(parallelism, r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setContextClassLoader(cl); + t.setDaemon(true); + return t; + }); + LOG.info("Created ExecutorServiceBasedEngineContext pool with {} threads", parallelism); + return executor; + } + + public ExecutorServiceBasedEngineContext(StorageConfiguration conf) { + super(conf, new LocalTaskContextSupplier()); + } + + // ---- Core parallel helpers ---- + + /** + * Submits each element to the executor pool and collects results in input order. + * RuntimeExceptions / Errors from {@code func} are re-thrown as-is after unwrapping + * the {@link CompletionException} wrapper. + */ + private List mapAsync(List data, Function func) { + List> futures = data.stream() + .map(item -> CompletableFuture.supplyAsync(() -> func.apply(item), PoolHolder.INSTANCE)) + .collect(Collectors.toList()); + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } catch (CompletionException e) { + throw rethrowUnwrapped(e); + } + return futures.stream().map(CompletableFuture::join).collect(Collectors.toList()); + } + + private static RuntimeException rethrowUnwrapped(CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw e; + } + + // ---- HoodieEngineContext implementations ---- + + @Override + public HoodieAccumulator newAccumulator() { + return HoodieAtomicLongAccumulator.create(); + } + + @Override + public HoodieData emptyHoodieData() { + return HoodieListData.eager(Collections.emptyList()); + } + + @Override + public HoodiePairData emptyHoodiePairData() { + return HoodieListPairData.eager(Collections.emptyList()); + } + + @Override + public HoodieData parallelize(List data, int parallelism) { + return HoodieListData.eager(data); + } + + @Override + public List map(List data, SerializableFunction func, int parallelism) { + return mapAsync(data, FunctionWrapper.throwingMapWrapper(func)); + } + + @Override + public List mapToPairAndReduceByKey(List data, SerializablePairFunction mapToPairFunc, + SerializableBiFunction reduceFunc, int parallelism) { + List> pairs = mapAsync(data, FunctionWrapper.throwingMapToPairWrapper(mapToPairFunc)); + return pairs.stream() + .collect(Collectors.groupingBy(Pair::getKey)).values().stream() + .map(list -> list.stream().map(Pair::getValue) + .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @Override + public Stream> mapPartitionsToPairAndReduceByKey( + Stream data, SerializablePairFlatMapFunction, K, V> flatMapToPairFunc, + SerializableBiFunction reduceFunc, int parallelism) { + try { + return CompletableFuture.supplyAsync(() -> + FunctionWrapper.throwingFlatMapToPairWrapper(flatMapToPairFunc).apply(data.iterator()) + .collect(Collectors.groupingBy(Pair::getKey)).entrySet().stream() + .map(entry -> new ImmutablePair<>(entry.getKey(), + entry.getValue().stream().map(Pair::getValue) + .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null))) + .filter(Objects::nonNull), + PoolHolder.INSTANCE).join(); + } catch (CompletionException e) { + throw rethrowUnwrapped(e); + } + } + + @Override + public List reduceByKey(List> data, SerializableBiFunction reduceFunc, + int parallelism) { + // Group by key (sequential), then reduce each group in parallel on the executor. + Map> grouped = data.stream() + .collect(Collectors.groupingBy(Pair::getKey, + Collectors.mapping(Pair::getValue, Collectors.toList()))); + return mapAsync(new ArrayList<>(grouped.entrySet()), + entry -> entry.getValue().stream() + .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null)) + .stream().filter(Objects::nonNull).collect(Collectors.toList()); + } + + @Override + public List flatMap(List data, SerializableFunction> func, int parallelism) { + return mapAsync(data, FunctionWrapper.throwingFlatMapWrapper(func)) + .stream().flatMap(s -> s).collect(Collectors.toList()); + } + + @Override + public void foreach(List data, SerializableConsumer consumer, int parallelism) { + List> futures = data.stream() + .map(item -> CompletableFuture.runAsync( + () -> FunctionWrapper.throwingForeachWrapper(consumer).accept(item), PoolHolder.INSTANCE)) + .collect(Collectors.toList()); + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } catch (CompletionException e) { + throw rethrowUnwrapped(e); + } + } + + @Override + public Map mapToPair(List data, SerializablePairFunction func, Integer parallelism) { + return mapAsync(data, FunctionWrapper.throwingMapToPairWrapper(func)) + .stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (oldVal, newVal) -> newVal)); + } + + @Override + public void setProperty(EngineProperty key, String value) { + // no operation + } + + @Override + public Option getProperty(EngineProperty key) { + return Option.empty(); + } + + @Override + public void setJobStatus(String activeModule, String activityDescription) { + // no operation + } + + @Override + public void clearJobStatus() { + // no operation + } + + @Override + public void putCachedDataIds(HoodieDataCacheKey cacheKey, int... ids) { + // no operation + } + + @Override + public List getCachedDataIds(HoodieDataCacheKey cacheKey) { + return Collections.emptyList(); + } + + @Override + public List removeCachedDataIds(HoodieDataCacheKey cacheKey) { + return Collections.emptyList(); + } + + @Override + public void cancelJob(String jobId) { + // no operation + } + + @Override + public void cancelAllJobs() { + // no operation + } + + @Override + public O aggregate(HoodieData data, O zeroValue, Functions.Function2 seqOp, + Functions.Function2 combOp) { + return data.collectAsList().stream().reduce(zeroValue, seqOp::apply, combOp::apply); + } + + @Override + @SuppressWarnings("unchecked") + public ReaderContextFactory getReaderContextFactory(HoodieTableMetaClient metaClient) { + return (ReaderContextFactory) getEngineReaderContextFactory(metaClient); + } + + @Override + public ReaderContextFactory getEngineReaderContextFactory(HoodieTableMetaClient metaClient) { + return new AvroReaderContextFactory(metaClient, new TypedProperties()); + } + + @Override + public KeyGenerator createKeyGenerator(TypedProperties props) throws IOException { + throw new UnsupportedOperationException("Not yet implemented"); + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestExecutorServiceBasedEngineContext.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestExecutorServiceBasedEngineContext.java new file mode 100644 index 0000000000000..475aa4c9d2dc8 --- /dev/null +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestExecutorServiceBasedEngineContext.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.engine; + +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.storage.HoodieStorage; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorage; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestExecutorServiceBasedEngineContext { + + private ExecutorServiceBasedEngineContext context; + + @BeforeEach + void setUp() { + HoodieStorage storage = getDefaultStorage(); + context = new ExecutorServiceBasedEngineContext(storage.getConf()); + } + + @Test + void testMapHappyPath() { + List result = context.map(Arrays.asList(1, 2, 3), x -> x * 2, 3); + assertEquals(Arrays.asList(2, 4, 6), result.stream().sorted().collect(Collectors.toList())); + } + + @Test + void testMapEmptyList() { + List result = context.map(java.util.Collections.emptyList(), x -> x * 2, 0); + assertTrue(result.isEmpty(), "map over empty list must return empty list"); + } + + @Test + void testMapPreservesInputOrder() { + List input = IntStream.range(0, 100).boxed().collect(Collectors.toList()); + List result = context.map(input, x -> x, 100); + assertEquals(input, result, "map must return results in the same order as the input list"); + } + + @Test + void testMapPropagatesRuntimeException() { + RuntimeException original = new RuntimeException("boom"); + RuntimeException thrown = assertThrows(RuntimeException.class, () -> + context.map(java.util.Collections.singletonList(1), x -> { + throw original; + }, 1)); + // throwingMapWrapper wraps ALL exceptions in HoodieException; original is the direct cause + assertInstanceOf(HoodieException.class, thrown); + assertSame(original, thrown.getCause()); + } + + @Test + void testMapPropagatesHoodieException() { + HoodieException original = new HoodieException("hoodie-boom"); + RuntimeException thrown = assertThrows(RuntimeException.class, () -> + context.map(java.util.Collections.singletonList(1), x -> { + throw original; + }, 1)); + assertInstanceOf(HoodieException.class, thrown); + assertSame(original, thrown.getCause()); + } + + @Test + void testMapWrapsCheckedException() { + Exception checkedCause = new Exception("checked!"); + RuntimeException thrown = assertThrows(RuntimeException.class, () -> + context.map(java.util.Collections.singletonList(1), x -> { + throw checkedCause; + }, 1)); + assertInstanceOf(HoodieException.class, thrown); + assertSame(checkedCause, thrown.getCause()); + } + + @Test + void testWorkerThreadClassloader() { + ClassLoader[] captured = new ClassLoader[1]; + context.map(java.util.Collections.singletonList(1), x -> { + captured[0] = Thread.currentThread().getContextClassLoader(); + return x; + }, 1); + assertEquals(ExecutorServiceBasedEngineContext.class.getClassLoader(), captured[0], + "Worker threads must use ExecutorServiceBasedEngineContext classloader to avoid ClassNotFoundException on Java 11+"); + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java index bdb858c763e4f..c4fc1e285f7c1 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java @@ -150,24 +150,24 @@ void testProcessKeyGroupsWithSingleValue() { ImmutablePair.of("key1", 42), ImmutablePair.of("key2", 17) ); - + HoodiePairData pairData = HoodieListPairData.lazy(singleValuePairs); - + // Create a function that just returns the values SerializableFunction, Iterator> func = iterator -> { List values = new ArrayList<>(); iterator.forEachRemaining(values::add); return values.iterator(); }; - + List shardIndices = Arrays.asList("key1", "key2"); HoodieData result = context.mapGroupsByKey(pairData, func, shardIndices, false); - + List resultList = result.collectAsList(); - + // Verify the results assertEquals(2, resultList.size()); assertTrue(resultList.contains(42)); assertTrue(resultList.contains(17)); } -} \ No newline at end of file +} From df35de41ee78d5aff7349b05a7da379214953068 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 13 Jun 2026 12:04:56 +0800 Subject: [PATCH 069/255] perf(spark): Resolve drop-partition-columns projection once per writer instead of per row (#18972) BulkInsertDataInternalWriterHelper#write redid constant work for every row when hoodie.datasource.write.drop.partition.columns is enabled: resolving the config flag, instantiating a key generator via constructor reflection through getPartitionPathCols, recomputing the partition-column ordinals into a fresh HashSet, and round-tripping the whole row through toSeq/fromSeq (boxing every column). The flag is now resolved once in the constructor, and the retained (non-partition) field ordinals and types are computed once on the first write(). The lazy initialization keeps the partition-column resolution unreachable for the bucket-index subclasses, which override write() and never drop columns, and for tasks that write no rows, matching the previous reachability exactly. write() copies the retained fields into a fresh GenericInternalRow, which is value-identical to the previous toSeq/filter/fromSeq output. (cherry picked from commit 6428cb258366df5fdba79dad35c7c8b9933aa8a2) --- .../BulkInsertDataInternalWriterHelper.java | 64 +++++++++++++------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertDataInternalWriterHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertDataInternalWriterHelper.java index 57847faedb82e..712a27f81833e 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertDataInternalWriterHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertDataInternalWriterHelper.java @@ -32,6 +32,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; @@ -69,6 +70,13 @@ public class BulkInsertDataInternalWriterHelper { protected final boolean simpleKeyGen; protected final int simplePartitionFieldIndex; protected final DataType simplePartitionFieldDataType; + protected final boolean shouldDropPartitionColumns; + // Ordinals and types of the non-partition fields, computed once on the first write() instead of + // in the constructor: bucket-index subclasses override write() and never drop columns, and the + // partition-column resolution must stay unreachable for them (and for tasks that write no rows) + // exactly as before. The helper is confined to a single task thread, so plain lazy init is safe. + private int[] retainedOrdinals; + private DataType[] retainedTypes; /** * NOTE: This is stored as Catalyst's internal {@link UTF8String} to avoid * conversion (deserialization) b/w {@link UTF8String} and {@link String} @@ -114,6 +122,36 @@ public BulkInsertDataInternalWriterHelper(HoodieTable hoodieTable, HoodieWriteCo this.simplePartitionFieldIndex = -1; this.simplePartitionFieldDataType = null; } + + this.shouldDropPartitionColumns = writeConfig.shouldDropPartitionColumns(); + } + + /** + * Resolves the ordinals and types of the non-partition fields. The partition columns are a pure + * function of the write config and schema, both immutable for the helper's lifetime, so this + * runs once per helper instead of once per row (getPartitionPathCols instantiates a key + * generator reflectively). + */ + private void initRetainedFields() { + List partitionCols = JavaScalaConverters.convertScalaListToJavaList( + HoodieDatasetBulkInsertHelper.getPartitionPathCols(this.writeConfig)); + Set partitionIdx = new HashSet<>(); + for (String col : partitionCols) { + partitionIdx.add(this.structType.fieldIndex(col)); + } + int numRetained = structType.fields().length - partitionIdx.size(); + int[] ordinals = new int[numRetained]; + DataType[] types = new DataType[numRetained]; + int retained = 0; + for (int i = 0; i < structType.fields().length; i++) { + if (!partitionIdx.contains(i)) { + ordinals[retained] = i; + types[retained] = structType.fields()[i].dataType(); + retained++; + } + } + this.retainedOrdinals = ordinals; + this.retainedTypes = types; } public void write(InternalRow row) throws IOException { @@ -126,27 +164,17 @@ public void write(InternalRow row) throws IOException { lastKnownPartitionPath = partitionPath.clone(); } - boolean shouldDropPartitionColumns = writeConfig.shouldDropPartitionColumns(); if (shouldDropPartitionColumns) { - // Drop the partition columns from the row - List partitionCols = JavaScalaConverters.convertScalaListToJavaList(HoodieDatasetBulkInsertHelper.getPartitionPathCols(this.writeConfig)); - Set partitionIdx = new HashSet<>(); - for (String col : partitionCols) { - partitionIdx.add(this.structType.fieldIndex(col)); + if (retainedOrdinals == null) { + initRetainedFields(); } - - // Relies on InternalRow::toSeq(...) preserving the column ordering based on the supplied schema - List cols = JavaScalaConverters.convertScalaListToJavaList(row.toSeq(structType)); - int idx = 0; - List newCols = new ArrayList<>(); - for (Object o : cols) { - if (!partitionIdx.contains(idx)) { - newCols.add(o); - } - idx += 1; + // Drop the partition columns from the row by copying the retained fields; a fresh row is + // allocated per record so values keep the same aliasing behavior as InternalRow.fromSeq + Object[] values = new Object[retainedOrdinals.length]; + for (int i = 0; i < retainedOrdinals.length; i++) { + values[i] = row.get(retainedOrdinals[i], retainedTypes[i]); } - InternalRow newRow = InternalRow.fromSeq(JavaScalaConverters.convertJavaListToScalaSeq(newCols)); - handle.write(newRow); + handle.write(new GenericInternalRow(values)); } else { handle.write(row); } From 87b86fef303de044ac3d74e1663c69243dc7f9e5 Mon Sep 17 00:00:00 2001 From: Aditya Goenka <63430370+ad1happy2go@users.noreply.github.com> Date: Sat, 13 Jun 2026 09:54:01 +0530 Subject: [PATCH 070/255] fix(spark): strip _hoodie_* meta columns from CDC before/after images (#18948) * [HUDI-14363] Strip _hoodie_* meta columns from CDC before/after images CDC before/after images inferred directly from base/log data files (e.g. the BASE_FILE_INSERT case hit by an insert-only commit that writes no CDC log file) leaked the _hoodie_* meta columns, while images served from the supplemental CDC log already have them stripped at write time. This produced an inconsistent, alternating-per-commit image schema. --------- Co-authored-by: Claude Opus 4.8 (cherry picked from commit 97c03f75c737ac88d8ac7f5234a9cf3856d05cfb) --- .../hudi/cdc/CDCFileGroupIterator.scala | 22 +++++-- .../cdc/TestCDCDataFrameSuite.scala | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala index 4970be3c249b3..cc316243d2664 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala @@ -62,6 +62,7 @@ import org.apache.spark.unsafe.types.UTF8String import java.io.Closeable import java.util import java.util.{Collections, Locale} +import java.util.function.UnaryOperator import java.util.stream.Collectors import scala.annotation.tailrec @@ -228,7 +229,9 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit, props.getBoolean(DISK_MAP_BITCASK_COMPRESSION_ENABLED.key(), DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue()), getClass.getSimpleName) - private val internalRowToJsonStringConverterMap: mutable.Map[Integer, InternalRowToJsonStringConverter] = mutable.Map.empty + // Per-schema cache of the (image projection, json converter) used to build CDC before/after + // images. Keyed by the record's schema id so schema evolution is handled correctly. + private val cdcImageConverterMap: mutable.Map[Integer, (UnaryOperator[InternalRow], InternalRowToJsonStringConverter)] = mutable.Map.empty private def needLoadNextFile: Boolean = { !recordIter.hasNext && @@ -559,9 +562,20 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit, * Convert InternalRow to json string. */ private def convertBufferedRecordToJsonString(record: BufferedRecord[InternalRow]): UTF8String = { - internalRowToJsonStringConverterMap.getOrElseUpdate(record.getSchemaId, - new InternalRowToJsonStringConverter(HoodieInternalRowUtils.getCachedSchema(readerContext.getRecordContext.decodeAvroSchema(record.getSchemaId)))) - .convert(record.getRecord) + val (imageProjection, converter) = cdcImageConverterMap.getOrElseUpdate(record.getSchemaId, { + val recordSchema = readerContext.getRecordContext.decodeAvroSchema(record.getSchemaId) + // CDC before/after images must contain only business columns. Records read from base/log + // files carry the _hoodie_* meta columns (kept on the InternalRow because they are needed + // internally for record keying and merging), while images served from the supplemental CDC + // log already have them stripped at write time (HoodieCDCLogger). Project each record onto + // the meta-stripped image schema so every inference case produces a schema-consistent, + // business-columns-only image. + val imageSchema = HoodieSchemaUtils.removeMetadataFields(recordSchema) + val projection = readerContext.getRecordContext.projectRecord(recordSchema, imageSchema) + val converter = new InternalRowToJsonStringConverter(HoodieInternalRowUtils.getCachedSchema(imageSchema)) + (projection, converter) + }) + converter.convert(imageProjection.apply(record.getRecord)) } /** diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala index 51905c00caa4e..4bada04ab29d7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala @@ -21,6 +21,7 @@ package org.apache.hudi.functional.cdc import org.apache.hudi.DataSourceWriteOptions import org.apache.hudi.DataSourceWriteOptions.{MOR_TABLE_TYPE_OPT_VAL, PARTITIONPATH_FIELD_OPT_KEY, PRECOMBINE_FIELD_OPT_KEY, RECORDKEY_FIELD_OPT_KEY} import org.apache.hudi.QuickstartUtils.getQuickstartWriteConfigs +import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver} import org.apache.hudi.common.table.cdc.{HoodieCDCOperation, HoodieCDCSupplementalLoggingMode} import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode.OP_KEY_ONLY @@ -959,4 +960,64 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { } assertTrue(newRecordFound, "Should have found the new record with complex data types in CDC") } + + /** + * Regression test for HUDI-14363: the CDC incremental query must produce before/after images + * that contain only business columns, never the _hoodie_* meta columns. Before the fix, images + * inferred directly from base/log data files (e.g. the BASE_FILE_INSERT case, which is hit by an + * insert-only commit that writes no CDC log file) leaked the meta columns, while images served + * from the supplemental CDC log did not - producing an inconsistent, alternating-per-commit + * schema. This reproduces the reporter's scenario (MOR + inline compaction every delta commit + + * upsert inserts) and asserts no image carries meta fields for any supplemental logging mode. + */ + @ParameterizedTest + @EnumSource(classOf[HoodieCDCSupplementalLoggingMode]) + def testCDCImagesExcludeHoodieMetaFields(loggingMode: HoodieCDCSupplementalLoggingMode): Unit = { + val options = commonOpts ++ Map( + DataSourceWriteOptions.TABLE_TYPE.key() -> DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL, + HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE.key -> loggingMode.name(), + "hoodie.compact.inline" -> "true", + "hoodie.compact.inline.max.delta.commits" -> "1" + ) + + // 1. Insert - this commit writes no CDC log file, so its change data is inferred from the base + // file via the BASE_FILE_INSERT case (the path that previously leaked _hoodie_* meta columns). + val records1 = recordsToStrings(dataGen.generateInserts("000", 100)).asScala.toList + spark.read.json(spark.sparkContext.parallelize(records1, 2)) + .write.format("org.apache.hudi") + .options(options) + .mode(SaveMode.Overwrite) + .save(basePath) + metaClient = createMetaClient(spark, basePath) + val instant1 = metaClient.reloadActiveTimeline.lastInstant().get() + assertFalse(hasCDCLogFile(instant1)) + val commitTime1 = instant1.requestedTime + + // 2. Upsert (updates + new inserts) - exercises the supplemental CDC log (AS_IS) path too. + val updates = recordsToStrings(dataGen.generateUniqueUpdates("001", 30)).asScala.toList + val inserts = recordsToStrings(dataGen.generateInserts("001", 20)).asScala.toList + spark.read.json(spark.sparkContext.parallelize(updates ++ inserts, 2)) + .write.format("org.apache.hudi") + .options(options) + .mode(SaveMode.Append) + .save(basePath) + + // Read all change data and assert no before/after image contains a Hudi meta column. Note we + // check the actual meta-column names rather than the "_hoodie_" prefix: _hoodie_is_deleted is a + // business/payload field (the soft-delete marker carried in the record schema), not a meta + // column, so it is expected to remain in the image. + val allCDCData = cdcDataFrame((commitTime1.toLong - 1).toString).collect() + assertTrue(allCDCData.nonEmpty, "Expected some CDC rows") + allCDCData.foreach { row => + Seq("before", "after").foreach { col => + val json = row.getAs[String](col) + if (json != null) { + HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.asScala.foreach { metaCol => + assertFalse(json.contains(metaCol), + s"$col image should not contain meta column $metaCol, but was: $json") + } + } + } + } + } } From fc56bedbaf2c75fad82f86bdb37a5cf62d107ff2 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 13 Jun 2026 14:54:46 +0800 Subject: [PATCH 071/255] =?UTF-8?q?perf(spark):=20Parse=20bucket=20index?= =?UTF-8?q?=20hash-field=20config=20once=20instead=20of=20per=E2=80=A6=20(?= =?UTF-8?q?#18979)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(spark): Parse bucket index hash-field config once instead of per record The String overload of BucketIdentifier.getBucketId re-parses the comma-separated hash-field config on every call (split, per-token trim, empty filter, new list). Precompute the field list once per partitioner or writer with KeyGenUtils.getIndexKeyFields, the exact parser the String overload uses, and call the existing List overloads: bucket ids are bit-identical since the downstream chain is unchanged, mirroring the precedent in HoodieBucketIndex. Covers the upsert shuffle (SparkBucketIndexPartitioner), the row-writer repartition closure (BucketPartitionUtils) and the bucket bulk-insert write path (BucketBulkInsertDataInternalWriterHelper and the consistent-hashing variant). Measured on the two existing overloads (JDK 17, single-field key, 10M iterations): ~62 ns/op with the per-call parse vs ~4 ns/op with the precomputed list. * review: Apply parse-once fix to sibling Spark bucket-index partitioners Mirror the SparkBucketIndexPartitioner change to the remaining Spark write paths that still re-split the comma-separated hash-field config per record: SparkPartitionBucketIndexPartitioner (partition-level simple bucket index), ConsistentBucketIndexBulkInsertPartitionerWithRows and SingleSparkJobConsistentHashingExecutionStrategy (consistent hashing). Each precomputes KeyGenUtils.getIndexKeyFields(...) once and calls the existing List overload. Behavior-preserving. (cherry picked from commit 27daf3df0757855b274bc5ebbeeee6e9c8179b45) --- ...gleSparkJobConsistentHashingExecutionStrategy.java | 9 ++++++--- ...stentBucketIndexBulkInsertPartitionerWithRows.java | 9 ++++++--- .../BucketBulkInsertDataInternalWriterHelper.java | 11 ++++++++--- ...stentBucketBulkInsertDataInternalWriterHelper.java | 2 +- .../action/commit/SparkBucketIndexPartitioner.java | 9 ++++++--- .../commit/SparkPartitionBucketIndexPartitioner.java | 9 ++++++--- .../org/apache/spark/sql/BucketPartitionUtils.scala | 6 +++++- 7 files changed, 38 insertions(+), 17 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java index 7b49e7972f473..33b4a8d08ec5b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java @@ -44,6 +44,7 @@ import org.apache.hudi.io.HoodieWriteHandle; import org.apache.hudi.io.IOUtils; import org.apache.hudi.io.WriteHandleFactory; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.cluster.strategy.BaseConsistentHashingBucketClusteringPlanStrategy; import org.apache.hudi.util.ExecutorFactory; @@ -69,13 +70,15 @@ @Slf4j public class SingleSparkJobConsistentHashingExecutionStrategy extends SingleSparkJobExecutionStrategy { - private final String indexKeyFields; + // parsed once; the per-record bucket lookup uses the List overload of getBucket so the + // comma-separated config string is not re-split per record + private final List indexKeyFieldList; private final HoodieSchema readerSchema; public SingleSparkJobConsistentHashingExecutionStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) { super(table, engineContext, writeConfig); - this.indexKeyFields = table.getConfig().getBucketIndexHashField(); + this.indexKeyFieldList = KeyGenUtils.getIndexKeyFields(table.getConfig().getBucketIndexHashField()); this.readerSchema = HoodieSchemaUtils.addMetadataFields(HoodieSchema.parse(writeConfig.getSchema())); } @@ -205,7 +208,7 @@ private List performBucketSplitForGroup(ReaderContextFactory rea ConsistentBucketIdentifier identifier = new ConsistentBucketIdentifier(metadata); ClusteringOperation operation = clusteringGroup.getOperations().get(0); ClosableIterator> iterator = getRecordIterator(readerContextFactory, operation, instantTime, IOUtils.getMaxMemoryPerCompaction(new SparkTaskContextSupplier(), writeConfig)); - Function, String> fileIdPrefixExtractor = record -> identifier.getBucket(record.getRecordKey(), this.indexKeyFields).getFileIdPrefix(); + Function, String> fileIdPrefixExtractor = record -> identifier.getBucket(record.getRecordKey(), this.indexKeyFieldList).getFileIdPrefix(); HoodieConsumer, List> insertHandler = new InsertHandler(writeConfig, instantTime, getHoodieTable(), taskContextSupplier, new FixedIdSuffixCreateHandleFactory(), false, fileIdPrefixExtractor, readerSchema); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/ConsistentBucketIndexBulkInsertPartitionerWithRows.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/ConsistentBucketIndexBulkInsertPartitionerWithRows.java index 24ef7fd18716a..b681c03f4f033 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/ConsistentBucketIndexBulkInsertPartitionerWithRows.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/ConsistentBucketIndexBulkInsertPartitionerWithRows.java @@ -29,6 +29,7 @@ import org.apache.hudi.index.bucket.ConsistentBucketIndexUtils; import org.apache.hudi.index.bucket.HoodieSparkConsistentBucketIndex; import org.apache.hudi.keygen.BuiltinKeyGenerator; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory; import org.apache.hudi.table.BucketSortBulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; @@ -54,7 +55,9 @@ */ public class ConsistentBucketIndexBulkInsertPartitionerWithRows extends BucketSortBulkInsertPartitioner> { - private final String indexKeyFields; + // parsed once; the per-record getBucketId path uses the List overload of getBucket so the + // comma-separated config string is not re-split per record + private final List indexKeyFieldList; private final List fileIdPfxList = new ArrayList<>(); @@ -83,7 +86,7 @@ public ConsistentBucketIndexBulkInsertPartitionerWithRows(HoodieTable table, Map strategyParams, boolean populateMetaFields, Map> hashingChildrenNodes) { super(table, strategyParams.getOrDefault(PLAN_STRATEGY_SORT_COLUMNS.key(), "")); - this.indexKeyFields = table.getConfig().getBucketIndexHashField(); + this.indexKeyFieldList = KeyGenUtils.getIndexKeyFields(table.getConfig().getBucketIndexHashField()); this.populateMetaFields = populateMetaFields; if (!populateMetaFields) { this.keyGeneratorOpt = HoodieSparkKeyGeneratorFactory.getKeyGenerator(table.getConfig().getProps()); @@ -179,7 +182,7 @@ private Map initializeBucketIdentifier(JavaR private int getBucketId(Row row) { String recordKey = extractor.getRecordKey(row); String partitionPath = extractor.getPartitionPath(row); - ConsistentHashingNode node = partitionToIdentifier.get(partitionPath).getBucket(recordKey, indexKeyFields); + ConsistentHashingNode node = partitionToIdentifier.get(partitionPath).getBucket(recordKey, indexKeyFieldList); return partitionToFileIdPfxIdxMap.get(partitionPath).get(node.getFileIdPrefix()); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java index 15d973743fd16..6a3c5dd4912c4 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java @@ -25,6 +25,7 @@ import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.partition.NumBucketsFunction; import org.apache.hudi.io.storage.row.HoodieRowCreateHandle; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.table.HoodieTable; @@ -35,6 +36,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -47,7 +49,9 @@ public class BucketBulkInsertDataInternalWriterHelper extends BulkInsertDataInte private Pair lastFileId; // for efficient code path // p -> (fileId -> handle) private final Map, HoodieRowCreateHandle> handles; - protected final String indexKeyFields; + // parsed once; the per-row write path uses the List overloads so the comma-separated config + // string is not re-split per row + protected final List indexKeyFieldList; protected final int bucketNum; private final boolean isNonBlockingConcurrencyControl; private final NumBucketsFunction numBucketsFunction; @@ -62,7 +66,8 @@ public BucketBulkInsertDataInternalWriterHelper(HoodieTable hoodieTable, HoodieW String instantTime, int taskPartitionId, long taskId, long taskEpochId, StructType structType, boolean populateMetaFields, boolean arePartitionRecordsSorted, boolean shouldPreserveHoodieMetadata) { super(hoodieTable, writeConfig, instantTime, taskPartitionId, taskId, taskEpochId, structType, populateMetaFields, arePartitionRecordsSorted, shouldPreserveHoodieMetadata); - this.indexKeyFields = writeConfig.getStringOrDefault(HoodieIndexConfig.BUCKET_INDEX_HASH_FIELD, writeConfig.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())); + this.indexKeyFieldList = KeyGenUtils.getIndexKeyFields( + writeConfig.getStringOrDefault(HoodieIndexConfig.BUCKET_INDEX_HASH_FIELD, writeConfig.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()))); this.bucketNum = writeConfig.getInt(HoodieIndexConfig.BUCKET_INDEX_NUM_BUCKETS); this.handles = new HashMap<>(); this.isNonBlockingConcurrencyControl = writeConfig.isNonBlockingConcurrencyControl(); @@ -73,7 +78,7 @@ public void write(InternalRow row) throws IOException { try { UTF8String partitionPath = extractPartitionPath(row); UTF8String recordKey = extractRecordKey(row); - int bucketId = BucketIdentifier.getBucketId(String.valueOf(recordKey), indexKeyFields, numBucketsFunction.getNumBuckets(partitionPath.toString())); + int bucketId = BucketIdentifier.getBucketId(String.valueOf(recordKey), indexKeyFieldList, numBucketsFunction.getNumBuckets(partitionPath.toString())); if (lastFileId == null || !Objects.equals(lastFileId.getKey(), partitionPath) || !Objects.equals(lastFileId.getValue(), bucketId)) { // NOTE: It's crucial to make a copy here, since [[UTF8String]] could be pointing into // a mutable underlying buffer diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/ConsistentBucketBulkInsertDataInternalWriterHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/ConsistentBucketBulkInsertDataInternalWriterHelper.java index 9072e32939d70..19c11caa13690 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/ConsistentBucketBulkInsertDataInternalWriterHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/ConsistentBucketBulkInsertDataInternalWriterHelper.java @@ -75,7 +75,7 @@ public void write(InternalRow row) throws IOException { private HoodieRowCreateHandle getBucketRowCreateHandle(String partitionPath, String recordKey) { ConsistentBucketIdentifier identifier = getBucketIdentifier(partitionPath); - final ConsistentHashingNode node = identifier.getBucket(recordKey, indexKeyFields); + final ConsistentHashingNode node = identifier.getBucket(recordKey, indexKeyFieldList); String fileId = FSUtils.createNewFileId(node.getFileIdPrefix(), 0); ValidationUtils.checkArgument(node.getTag() != ConsistentHashingNode.NodeTag.NORMAL diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java index a0f778f17e6f4..db7261eb8286a 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java @@ -28,6 +28,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.HoodieBucketIndex; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.table.WorkloadStat; @@ -52,7 +53,9 @@ public class SparkBucketIndexPartitioner extends SparkHoodiePartitioner { private final int numBuckets; - private final String indexKeyField; + // parsed once; the per-record getPartition path uses the List overload of getBucketId so the + // comma-separated config string is not re-split per record + private final List indexKeyFieldList; private final int totalPartitionPaths; private final List partitionPaths; /** @@ -80,7 +83,7 @@ public SparkBucketIndexPartitioner(WorkloadProfile profile, + table.getIndex().getClass().getSimpleName()); } this.numBuckets = ((HoodieBucketIndex) table.getIndex()).getNumBuckets(); - this.indexKeyField = config.getBucketIndexHashField(); + this.indexKeyFieldList = KeyGenUtils.getIndexKeyFields(config.getBucketIndexHashField()); this.totalPartitionPaths = profile.getPartitionPaths().size(); partitionPaths = new ArrayList<>(profile.getPartitionPaths()); partitionPathOffset = new HashMap<>(); @@ -129,7 +132,7 @@ public int getPartition(Object key) { Option location = keyLocation._2; int bucketId = location.isPresent() ? BucketIdentifier.bucketIdFromFileId(location.get().getFileId()) - : BucketIdentifier.getBucketId(keyLocation._1.getRecordKey(), indexKeyField, numBuckets); + : BucketIdentifier.getBucketId(keyLocation._1.getRecordKey(), indexKeyFieldList, numBuckets); return partitionPathOffset.get(partitionPath) + bucketId; } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionBucketIndexPartitioner.java index 3873835989877..292410d35d771 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionBucketIndexPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionBucketIndexPartitioner.java @@ -29,6 +29,7 @@ import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.HoodieBucketIndex; import org.apache.hudi.index.bucket.partition.NumBucketsFunction; +import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.table.WorkloadStat; @@ -57,7 +58,9 @@ public class SparkPartitionBucketIndexPartitioner extends SparkHoodiePartitio private final int totalPartitions; private final NumBucketsFunction numBucketsFunction; - private final String indexKeyField; + // parsed once; the per-record getPartition path uses the List overload of getBucketId so the + // comma-separated config string is not re-split per record + private final List indexKeyFieldList; private final int totalPartitionPaths; private final List partitionPaths; /** @@ -96,7 +99,7 @@ public SparkPartitionBucketIndexPartitioner(WorkloadProfile profile, HoodieWriteConfig writeConfig = table.getConfig(); this.numBucketsFunction = NumBucketsFunction.fromWriteConfig(writeConfig); - this.indexKeyField = config.getBucketIndexHashField(); + this.indexKeyFieldList = KeyGenUtils.getIndexKeyFields(config.getBucketIndexHashField()); this.totalPartitionPaths = profile.getPartitionPaths().size(); partitionPaths = new ArrayList<>(profile.getPartitionPaths()); partitionPathOffset = new HashMap<>(); @@ -160,7 +163,7 @@ public int getPartition(Object key) { Option location = keyLocation._2; int bucketId = location.isPresent() ? BucketIdentifier.bucketIdFromFileId(location.get().getFileId()) - : BucketIdentifier.getBucketId(keyLocation._1.getRecordKey(), indexKeyField, numBucketsFunction.getNumBuckets(partitionPath)); + : BucketIdentifier.getBucketId(keyLocation._1.getRecordKey(), indexKeyFieldList, numBucketsFunction.getNumBuckets(partitionPath)); return partitionPathOffset.get(partitionPath) + bucketId; } } diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala index da7e8c682e4e6..14aff571c238c 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala @@ -25,16 +25,20 @@ import org.apache.hudi.common.util.{Functions, RemotePartitionHelper} import org.apache.hudi.common.util.hash.BucketIndexUtil import org.apache.hudi.index.bucket.BucketIdentifier import org.apache.hudi.index.bucket.partition.NumBucketsFunction +import org.apache.hudi.keygen.KeyGenUtils import org.apache.spark.Partitioner import org.apache.spark.sql.catalyst.InternalRow object BucketPartitionUtils extends SparkAdapterSupport { def createDataFrame(df: DataFrame, indexKeyFields: String, numBucketsFunction: NumBucketsFunction, partitioner: Partitioner): DataFrame = { + // parse the comma-separated config once outside the per-row closure; the list is a + // serializable java.util.List, safe to capture + val indexKeyFieldList = KeyGenUtils.getIndexKeyFields(indexKeyFields) def getPartitionKeyExtractor(): InternalRow => (String, Int) = row => { val partition = row.getString(HoodieRecord.PARTITION_PATH_META_FIELD_ORD) val kb = BucketIdentifier - .getBucketId(row.getString(HoodieRecord.RECORD_KEY_META_FIELD_ORD), indexKeyFields, numBucketsFunction.getNumBuckets(partition)) + .getBucketId(row.getString(HoodieRecord.RECORD_KEY_META_FIELD_ORD), indexKeyFieldList, numBucketsFunction.getNumBuckets(partition)) if (partition == null || partition.trim.isEmpty) { ("", kb) From 1588e027e91d09279f89a87cdceef342a4c45b77 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 13 Jun 2026 17:37:16 +0800 Subject: [PATCH 072/255] =?UTF-8?q?perf(metadata):=20Parse=20RLI=20instant?= =?UTF-8?q?=20time=20once=20per=20batch=20instead=20of=20per=20=E2=80=A6?= =?UTF-8?q?=20(#18965)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(metadata): Parse RLI instant time once per batch instead of per record createRecordIndexUpdate parsed the commit instant time string into epoch millis for every record although the instant is constant for the whole commit. Add a millis-based overload plus a public parseRecordIndexInstantTime helper and hoist the parse out of the per-record loops in base-file RLI generation, revived-key processing, record key iterators, RLI initialization and the Flink index write path. RecordIndexMapper memoizes per write status since its delegate locations can carry different instants. On the read side, getLocationFromRecordIndexInfo formatted the instant millis back to a string per looked-up record. Cache the formatting in a per-thread bounded map that is invalidated when the JVM default time zone changes, since formatDate depends on it. * review: Print raw epoch millis in the invalid-fileId error message Reformatting the millis through HoodieInstantTimeGenerator.formatDate raised a timezone question and is not always byte-identical to the original instant string (legacy second-granularity instants gain the compatibility-parse millis suffix), so print the unambiguous raw millis instead on this error path. * review: Drop the read-side formatted-instant cache Restore the direct per-call formatting in getLocationFromRecordIndexInfo; the cache needed zone-change invalidation to stay correct, which is more machinery than the formatting cost justifies. The PR is now write-side only (parse hoisting). * review: Print instantTime, not raw millis, in the invalid-fileId error Log the human-readable instant time in the error message instead of raw epoch millis. Reconstruct it only on the cold catch path via TimelineUtils.formatDate(new Date(instantTimeMillis)) (the inverse of parseDateFromInstantTime), so the hot per-record path keeps using the pre-parsed millis and pays nothing. * docs(metadata): Document how instantTimeMillis must be parsed and its timezone The millis overload of createRecordIndexUpdate now documents that instantTimeMillis must come from parseRecordIndexInstantTime(String) (delegating to TimelineUtils.parseDateFromInstantTime), not hand-constructed, and that the instant-time string is interpreted in the JVM default time zone (ZoneId.systemDefault()), yielding epoch millis since 1970-01-01T00:00:00Z. (cherry picked from commit 73ff4ea3aa4fb0c6b7b3e5be1c0210c72faa532c) --- .../HoodieBackedTableMetadataWriter.java | 4 +- .../hudi/metadata/RecordIndexMapper.java | 9 ++- .../metadata/BaseFileRecordParsingUtils.java | 3 +- .../hudi/metadata/HoodieMetadataPayload.java | 42 ++++++++++++-- .../metadata/HoodieTableMetadataUtil.java | 10 +++- .../metadata/TestHoodieTableMetadataUtil.java | 58 +++++++++++++++++++ .../sink/partitioner/index/IndexRowUtils.java | 4 +- .../partitioner/index/IndexWriteFunction.java | 4 +- 8 files changed, 120 insertions(+), 14 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 546ce10bdfbed..4789accc8caa9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1011,11 +1011,11 @@ private static HoodieData readRecordKeysFromFileSliceSnapshot( .withShouldUseRecordPosition(false) .withProps(metaClient.getTableConfig().getProps()) .build(); - String baseFileInstantTime = fileSlice.getBaseInstantTime(); + long baseFileInstantTimeMillis = HoodieMetadataPayload.parseRecordIndexInstantTime(fileSlice.getBaseInstantTime()); return new CloseableMappingIterator<>(fileGroupReader.getClosableIterator(), record -> { String recordKey = readerContext.getRecordContext().getRecordKey(record, requestedSchema); return HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partition, fileId, - baseFileInstantTime, 0); + baseFileInstantTimeMillis, 0); }); }); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java index 329e74e60676e..4b736768ebc25 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java @@ -30,7 +30,9 @@ import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Mapper for Record Level Index (RLI). @@ -45,6 +47,8 @@ public RecordIndexMapper(HoodieWriteConfig dataWriteConfig) { @Override protected List generateRecords(WriteStatus writeStatus) { List allRecords = new ArrayList<>(); + // delegates of one write status share at most a few distinct instants, so memoize the parse + Map instantTimeMillisCache = new HashMap<>(); for (HoodieRecordDelegate recordDelegate : writeStatus.getIndexStats().getWrittenRecordDelegates()) { if (!writeStatus.isErrored(recordDelegate.getHoodieKey())) { if (recordDelegate.isIgnoreIndexUpdate()) { @@ -68,7 +72,10 @@ protected List generateRecords(WriteStatus writeStatus) { // Insert new record case hoodieRecord = HoodieMetadataPayload.createRecordIndexUpdate( recordDelegate.getRecordKey(), recordDelegate.getPartitionPath(), - newLocation.get().getFileId(), newLocation.get().getInstantTime(), dataWriteConfig.getWritesFileIdEncoding()); + newLocation.get().getFileId(), + instantTimeMillisCache.computeIfAbsent( + newLocation.get().getInstantTime(), HoodieMetadataPayload::parseRecordIndexInstantTime), + dataWriteConfig.getWritesFileIdEncoding()); allRecords.add(hoodieRecord); } } else { diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseFileRecordParsingUtils.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseFileRecordParsingUtils.java index e4d83111ae883..6b92f9dcf39bb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseFileRecordParsingUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseFileRecordParsingUtils.java @@ -77,9 +77,10 @@ public static Iterator generateRLIMetadataHoodieRecordsForBaseFile recordStatuses); List hoodieRecords = new ArrayList<>(); if (recordStatusListMap.containsKey(RecordStatus.INSERT)) { + long instantTimeMillis = HoodieMetadataPayload.parseRecordIndexInstantTime(instantTime); hoodieRecords.addAll(recordStatusListMap.get(RecordStatus.INSERT).stream() .map(recordKey -> (HoodieRecord) HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partition, fileId, - instantTime, writesFileIdEncoding)).collect(toList())); + instantTimeMillis, writesFileIdEncoding)).collect(toList())); } if (recordStatusListMap.containsKey(RecordStatus.DELETE)) { diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java index ac509fce962ef..325a7a5af6baa 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java @@ -60,6 +60,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -649,14 +650,46 @@ public static Stream createPartitionStatsRecords(String partitionP */ public static HoodieRecord createRecordIndexUpdate(String recordKey, String partition, String fileId, String instantTime, int fileIdEncoding) { + return createRecordIndexUpdate(recordKey, partition, fileId, parseRecordIndexInstantTime(instantTime), fileIdEncoding); + } - HoodieKey key = new HoodieKey(recordKey, MetadataPartitionType.RECORD_INDEX.getPartitionPath()); - long instantTimeMillis = -1; + /** + * Parses an instant time into the epoch millis stored in record index entries. + *

    + * Callers creating record index updates for many records of the same commit should parse the + * instant time once with this method and use the millis-based + * {@link #createRecordIndexUpdate(String, String, String, long, int)} overload per record. + */ + public static long parseRecordIndexInstantTime(String instantTime) { try { - instantTimeMillis = TimelineUtils.parseDateFromInstantTime(instantTime).getTime(); + return TimelineUtils.parseDateFromInstantTime(instantTime).getTime(); } catch (Exception e) { throw new HoodieMetadataException("Failed to create metadata payload for record index. Instant time parsing for " + instantTime + " failed ", e); } + } + + /** + * Create and return a {@code HoodieMetadataPayload} to insert or update an entry for the record index. + *

    + * Same as {@link #createRecordIndexUpdate(String, String, String, String, int)} but takes the + * instant time already parsed to epoch millis, so per-commit callers parse it only once. + *

    + * {@code instantTimeMillis} must be obtained from {@link #parseRecordIndexInstantTime(String)} (which + * delegates to {@link TimelineUtils#parseDateFromInstantTime(String)}); it should not be constructed by + * hand, so that the value stored here matches what the String overload would write. The instant-time + * string is interpreted in the JVM default time zone ({@link java.time.ZoneId#systemDefault()}) and the + * result is epoch milliseconds (since 1970-01-01T00:00:00Z). + * + * @param recordKey Key of the record + * @param partition Name of the partition which contains the record + * @param fileId fileId which contains the record + * @param instantTimeMillis epoch millis of the instant when the record was added, as returned by + * {@link #parseRecordIndexInstantTime(String)} + */ + public static HoodieRecord createRecordIndexUpdate(String recordKey, String partition, + String fileId, long instantTimeMillis, int fileIdEncoding) { + + HoodieKey key = new HoodieKey(recordKey, MetadataPartitionType.RECORD_INDEX.getPartitionPath()); if (fileIdEncoding == 0) { // Data file names have a -D suffix to denote the index (D = integer) of the file written // In older HUID versions the file index was missing @@ -672,8 +705,9 @@ public static HoodieRecord createRecordIndexUpdate(String fileIndex = Integer.parseInt(fileId.substring(index + 1)); } } catch (Exception e) { + // reconstruct the instant time only on this cold error path; the hot per-record path keeps the pre-parsed millis throw new HoodieMetadataException(String.format("Invalid UUID or index: fileID=%s, partition=%s, instantTime=%s", - fileId, partition, instantTime), e); + fileId, partition, TimelineUtils.formatDate(new Date(instantTimeMillis))), e); } HoodieMetadataPayload payload = new HoodieMetadataPayload(recordKey, diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index 0ede3ae5015e5..23f20fa4efea9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -144,6 +144,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -940,8 +941,9 @@ public static HoodieData convertMetadataToRecordIndexRecords(H Set revivedKeys = revivedAndDeletedKeys.getLeft(); Set deletedKeys = revivedAndDeletedKeys.getRight(); // Process revived keys to create updates + long instantTimeMillis = HoodieMetadataPayload.parseRecordIndexInstantTime(instantTime); List revivedRecords = revivedKeys.stream() - .map(recordKey -> HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partitionPath, fileId, instantTime, writesFileIdEncoding)) + .map(recordKey -> HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partitionPath, fileId, instantTimeMillis, writesFileIdEncoding)) .collect(Collectors.toList()); // Process deleted keys to create deletes List deletedRecords = deletedKeys.stream() @@ -2499,7 +2501,7 @@ public static HoodieRecordGlobalLocation getLocationFromRecordIndexInfo( fileId = originalFileId; } - final java.util.Date instantDate = new java.util.Date(instantTime); + final Date instantDate = new Date(instantTime); return new HoodieRecordGlobalLocation(partition, HoodieInstantTimeGenerator.formatDate(instantDate), fileId); } @@ -2615,6 +2617,8 @@ private static ClosableIterator getHoodieRecordIterator(ClosableIt String instantTime, boolean isPartitionedRLI ) { + // the delete iterator never reads the instant time, so only the update path pays the parse + final long instantTimeMillis = forDelete ? -1L : HoodieMetadataPayload.parseRecordIndexInstantTime(instantTime); return new ClosableIterator() { @Override public void close() { @@ -2630,7 +2634,7 @@ public boolean hasNext() { public HoodieRecord next() { return forDelete ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next(), partition, isPartitionedRLI) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); + : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTimeMillis, 0); } }; } diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java index 95023eebe6954..6cab9b9067c34 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java @@ -21,22 +21,27 @@ import org.apache.hudi.common.function.SerializableBiFunction; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; +import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; import org.apache.hudi.common.util.Option; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TimeZone; import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS; import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS; @@ -352,4 +357,57 @@ void testVariantBlobVectorColumnsAreNotSupportedForV1ColumnStats() { "STRING should remain supported for record type " + recordType); } } + + @Test + void testCreateRecordIndexUpdateMillisOverloadMatchesStringOverload() { + String instantTime = "20260610153045678"; + long instantTimeMillis = HoodieMetadataPayload.parseRecordIndexInstantTime(instantTime); + + // uuid-encoded fileId (encoding 0) + HoodieRecord fromString = HoodieMetadataPayload.createRecordIndexUpdate( + "rk1", "p1", "49b8b3c8-9e5d-4731-9d51-a2d8e9b5c7f3-0", instantTime, 0); + HoodieRecord fromMillis = HoodieMetadataPayload.createRecordIndexUpdate( + "rk1", "p1", "49b8b3c8-9e5d-4731-9d51-a2d8e9b5c7f3-0", instantTimeMillis, 0); + assertEquals(fromString.getKey(), fromMillis.getKey()); + assertEquals(fromString.getData(), fromMillis.getData()); + + // raw fileId (encoding 1) + fromString = HoodieMetadataPayload.createRecordIndexUpdate( + "rk1", "p1", "some-raw-file-id", instantTime, 1); + fromMillis = HoodieMetadataPayload.createRecordIndexUpdate( + "rk1", "p1", "some-raw-file-id", instantTimeMillis, 1); + assertEquals(fromString.getKey(), fromMillis.getKey()); + assertEquals(fromString.getData(), fromMillis.getData()); + } + + @Test + void testGetLocationFromRecordIndexInfoFormatsInstantConsistently() { + long instantMillis1 = HoodieMetadataPayload.parseRecordIndexInstantTime("20260610153045678"); + long instantMillis2 = HoodieMetadataPayload.parseRecordIndexInstantTime("20260610163045678"); + String expected1 = HoodieInstantTimeGenerator.formatDate(new Date(instantMillis1)); + String expected2 = HoodieInstantTimeGenerator.formatDate(new Date(instantMillis2)); + // repeated and alternating instants must format consistently + for (long instantMillis : new long[] {instantMillis1, instantMillis1, instantMillis2, instantMillis1}) { + HoodieRecordGlobalLocation location = HoodieTableMetadataUtil.getLocationFromRecordIndexInfo( + "p1", 1, -1L, -1L, -1, "some-raw-file-id", instantMillis); + assertEquals(instantMillis == instantMillis1 ? expected1 : expected2, location.getInstantTime()); + assertEquals("p1", location.getPartitionPath()); + assertEquals("some-raw-file-id", location.getFileId()); + } + + // formatDate follows the JVM default time zone, so the decoded location must track a zone + // change; the two switches differ in offset, so at least one changes the formatted string + TimeZone originalTimeZone = TimeZone.getDefault(); + try { + for (String zoneId : new String[] {"UTC", "Asia/Kolkata"}) { + TimeZone.setDefault(TimeZone.getTimeZone(zoneId)); + String expectedInZone = HoodieInstantTimeGenerator.formatDate(new Date(instantMillis1)); + HoodieRecordGlobalLocation location = HoodieTableMetadataUtil.getLocationFromRecordIndexInfo( + "p1", 1, -1L, -1L, -1, "some-raw-file-id", instantMillis1); + assertEquals(expectedInZone, location.getInstantTime()); + } + } finally { + TimeZone.setDefault(originalTimeZone); + } + } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexRowUtils.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexRowUtils.java index 4ea6f06e75519..5e8a04c2e15c5 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexRowUtils.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexRowUtils.java @@ -70,7 +70,7 @@ public static RowData createRecordIndexRow(HoodieFlinkInternalRow internalRow) { return indexRow; } - public static HoodieRecord convertToHoodieRecord(String instant, RowData indexRow, HoodieWriteConfig dataWriteConfig) { + public static HoodieRecord convertToHoodieRecord(long instantMillis, RowData indexRow, HoodieWriteConfig dataWriteConfig) { if (indexRow.getByte(INDEX_TYPE_ORD) == RLI_TYPE) { switch (indexRow.getRowKind()) { case INSERT: @@ -78,7 +78,7 @@ public static HoodieRecord convertToHoodieRecord(String instant, RowData indexRo String.valueOf(indexRow.getString(KEY_ORD)), String.valueOf(indexRow.getString(PARTITION_ORD)), String.valueOf(indexRow.getString(FILE_ID_ORD)), - instant, + instantMillis, dataWriteConfig.getWritesFileIdEncoding()); case DELETE: return HoodieMetadataPayload.createRecordIndexDelete( diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexWriteFunction.java index 8e64d58a97b37..066f5aa62eb6a 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexWriteFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/IndexWriteFunction.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.metadata.HoodieMetadataPayload; import org.apache.hudi.sink.common.AbstractStreamWriteFunction; import org.apache.hudi.sink.event.WriteMetadataEvent; import org.apache.hudi.sink.exception.MemoryPagesExhaustedException; @@ -174,11 +175,12 @@ private Pair, Set> prepareIndexRecordsAndPartitions(B HoodieWriteConfig writeConfig = writeClient.getConfig(); // deduplicate the index records using commit time ordering. Map keyAndRecordMap = new LinkedHashMap<>(); + long currentInstantMillis = HoodieMetadataPayload.parseRecordIndexInstantTime(this.currentInstant); while (rowItr.hasNext()) { RowData indexRow = rowItr.next(); keyAndRecordMap.put( dedupKeyExtractor.apply(indexRow), - IndexRowUtils.convertToHoodieRecord(this.currentInstant, indexRow, writeConfig)); + IndexRowUtils.convertToHoodieRecord(currentInstantMillis, indexRow, writeConfig)); dataPartitions.add(IndexRowUtils.getPartition(indexRow)); } return Pair.of(new ArrayList<>(keyAndRecordMap.values()), dataPartitions); From 23d50905d6e8fd8bc07504a926d2dfd590a591f0 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Sun, 14 Jun 2026 07:56:32 +0700 Subject: [PATCH 073/255] test(trino): de-flake TestHudi*FileOperations by disabling async table statistics (#18995) (cherry picked from commit b75a5acc8d10848a14265b19bfe7b280343635bc) --- .../TestHudiAlluxioCacheFileOperations.java | 48 +++++-------------- .../TestHudiMemoryCacheFileOperations.java | 48 +++++-------------- .../hudi/TestHudiNoCacheFileOperations.java | 48 +++++-------------- 3 files changed, 36 insertions(+), 108 deletions(-) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java index 4395a9c1ddfff..89507721afa59 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java @@ -66,6 +66,11 @@ protected DistributedQueryRunner createQueryRunner() .put("fs.cache.directories", cacheDirectory.toAbsolutePath().toString()) .put("fs.cache.max-sizes", "100MB") .put("hudi.metadata.cache.enabled", "false") + // Disable async table-statistics refresh: it reads the metadata table on a + // background executor whose spans can outlive the query and leak into the next + // test's measurement (the symmetric off-by-N flake). Disabling it makes the + // file-operation counts deterministic right after the query returns. + .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); return HudiQueryRunner.builder() @@ -77,7 +82,6 @@ protected DistributedQueryRunner createQueryRunner() @Test public void testSelectWithFilter() - throws InterruptedException { @Language("SQL") String query = "SELECT * FROM " + HUDI_MULTI_FG_PT_V8_MOR + " WHERE country='SG'"; assertFileSystemAccesses( @@ -115,7 +119,6 @@ public void testSelectWithFilter() @Test public void testJoin() - throws InterruptedException { @Language("SQL") String query = "SELECT t1.id, t1.name, t1.price, t1.ts FROM " + HUDI_MULTI_FG_PT_V8_MOR + " t1 " + @@ -125,16 +128,16 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("Alluxio.readCached", DATA), 6) - .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 288) + .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 215) .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 8) .addCopies(new FileOperation("Alluxio.readCached", LOG), 30) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 39) - .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 93) + .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) + .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 69) .addCopies(new FileOperation("InputFile.length", TIMELINE), 4) .addCopies(new FileOperation("InputFile.length", LOG), 2) - .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 5) - .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 3) - .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 5) + .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) + .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) + .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) .build()); assertFileSystemAccesses(query, @@ -154,37 +157,10 @@ public void testJoin() } private void assertFileSystemAccesses(@Language("SQL") String query, Multiset expectedCacheAccesses) - throws InterruptedException { DistributedQueryRunner queryRunner = getDistributedQueryRunner(); queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); - // Async table-stats computation can outlive the synchronous query and emit spans into - // the exporter after execute returns. A fixed Thread.sleep races with this — when - // stats from query N is still running while query N+1's measurement happens, spans - // leak across the boundary and counts get scrambled (the symmetric off-by-N failure - // across paired tests). Poll until the span set is stable for two consecutive reads. - Multiset actual = waitForStableSpans(queryRunner); - assertMultisetsEqual(actual, expectedCacheAccesses); - } - - /** - * Returns the file-operation span set once two consecutive reads (200ms apart) agree. - * Bounded by a 30-second ceiling so a runaway test fails loudly instead of hanging. - */ - private static Multiset waitForStableSpans(QueryRunner queryRunner) - throws InterruptedException - { - long deadlineMillis = System.currentTimeMillis() + 30_000L; - Multiset previous = null; - while (System.currentTimeMillis() < deadlineMillis) { - Thread.sleep(200L); - Multiset current = getFileOperations(queryRunner); - if (previous != null && current.equals(previous)) { - return current; - } - previous = current; - } - return previous != null ? previous : getFileOperations(queryRunner); + assertMultisetsEqual(getFileOperations(queryRunner), expectedCacheAccesses); } public static Multiset getFileOperations(QueryRunner queryRunner) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java index ed362391b017a..61867b1cde7dd 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java @@ -56,6 +56,11 @@ protected DistributedQueryRunner createQueryRunner() .put("hudi.metadata-enabled", "true") .put("hudi.metadata.cache.enabled", "true") .put("fs.cache.enabled", "false") + // Disable async table-statistics refresh: it reads the metadata table on a + // background executor whose spans can outlive the query and leak into the next + // test's measurement (the symmetric off-by-N flake). Disabling it makes the + // file-operation counts deterministic right after the query returns. + .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); return HudiQueryRunner.builder() @@ -67,7 +72,6 @@ protected DistributedQueryRunner createQueryRunner() @Test public void testSelectWithFilter() - throws InterruptedException { @Language("SQL") String query = "SELECT * FROM " + HUDI_MULTI_FG_PT_V8_MOR + " WHERE country='SG'"; assertFileSystemAccesses( @@ -101,7 +105,6 @@ public void testSelectWithFilter() @Test public void testJoin() - throws InterruptedException { @Language("SQL") String query = "SELECT t1.id, t1.name, t1.price, t1.ts FROM " + HUDI_MULTI_FG_PT_V8_MOR + " t1 " + @@ -111,14 +114,14 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6) - .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 39) - .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 54) + .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 29) + .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 40) .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4) .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 39) - .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 5) - .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 3) - .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 5) + .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) + .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) + .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) + .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) .build()); assertFileSystemAccesses(query, @@ -136,37 +139,10 @@ public void testJoin() } private void assertFileSystemAccesses(@Language("SQL") String query, Multiset expectedCacheAccesses) - throws InterruptedException { DistributedQueryRunner queryRunner = getDistributedQueryRunner(); queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); - // Async table-stats computation can outlive the synchronous query and emit spans into - // the exporter after execute returns. A fixed Thread.sleep races with this — when - // stats from query N is still running while query N+1's measurement happens, spans - // leak across the boundary and counts get scrambled (the symmetric off-by-N failure - // across paired tests). Poll until the span set is stable for two consecutive reads. - Multiset actual = waitForStableSpans(queryRunner); - assertMultisetsEqual(actual, expectedCacheAccesses); - } - - /** - * Returns the file-operation span set once two consecutive reads (200ms apart) agree. - * Bounded by a 30-second ceiling so a runaway test fails loudly instead of hanging. - */ - private static Multiset waitForStableSpans(QueryRunner queryRunner) - throws InterruptedException - { - long deadlineMillis = System.currentTimeMillis() + 30_000L; - Multiset previous = null; - while (System.currentTimeMillis() < deadlineMillis) { - Thread.sleep(200L); - Multiset current = getFileOperations(queryRunner); - if (previous != null && current.equals(previous)) { - return current; - } - previous = current; - } - return previous != null ? previous : getFileOperations(queryRunner); + assertMultisetsEqual(getFileOperations(queryRunner), expectedCacheAccesses); } private static Multiset getFileOperations(QueryRunner queryRunner) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java index 9d6a6a8a52002..71518b9fc67b6 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java @@ -56,6 +56,11 @@ protected DistributedQueryRunner createQueryRunner() .put("hudi.metadata-enabled", "true") .put("hudi.metadata.cache.enabled", "false") .put("fs.cache.enabled", "false") + // Disable async table-statistics refresh: it reads the metadata table on a + // background executor whose spans can outlive the query and leak into the next + // test's measurement (the symmetric off-by-N flake). Disabling it makes the + // file-operation counts deterministic right after the query returns. + .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); return HudiQueryRunner.builder() @@ -67,7 +72,6 @@ protected DistributedQueryRunner createQueryRunner() @Test public void testSelectWithFilter() - throws InterruptedException { @Language("SQL") String query = "SELECT * FROM " + HUDI_MULTI_FG_PT_V8_MOR + " WHERE country='SG'"; assertFileSystemAccesses( @@ -101,7 +105,6 @@ public void testSelectWithFilter() @Test public void testJoin() - throws InterruptedException { @Language("SQL") String query = "SELECT t1.id, t1.name, t1.price, t1.ts FROM " + HUDI_MULTI_FG_PT_V8_MOR + " t1 " + @@ -111,12 +114,12 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 6) - .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 39) - .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 39) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 5) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 54) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 3) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 5) + .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 29) + .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 29) + .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) + .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 40) + .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) + .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 2) .build()); @@ -136,37 +139,10 @@ public void testJoin() } private void assertFileSystemAccesses(@Language("SQL") String query, Multiset expectedCacheAccesses) - throws InterruptedException { DistributedQueryRunner queryRunner = getDistributedQueryRunner(); queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); - // Async table-stats computation can outlive the synchronous query and emit spans into - // the exporter after execute returns. A fixed Thread.sleep races with this — when - // stats from query N is still running while query N+1's measurement happens, spans - // leak across the boundary and counts get scrambled (the symmetric off-by-N failure - // across paired tests). Poll until the span set is stable for two consecutive reads. - Multiset actual = waitForStableSpans(queryRunner); - assertMultisetsEqual(actual, expectedCacheAccesses); - } - - /** - * Returns the file-operation span set once two consecutive reads (200ms apart) agree. - * Bounded by a 30-second ceiling so a runaway test fails loudly instead of hanging. - */ - private static Multiset waitForStableSpans(QueryRunner queryRunner) - throws InterruptedException - { - long deadlineMillis = System.currentTimeMillis() + 30_000L; - Multiset previous = null; - while (System.currentTimeMillis() < deadlineMillis) { - Thread.sleep(200L); - Multiset current = getFileOperations(queryRunner); - if (previous != null && current.equals(previous)) { - return current; - } - previous = current; - } - return previous != null ? previous : getFileOperations(queryRunner); + assertMultisetsEqual(getFileOperations(queryRunner), expectedCacheAccesses); } private static Multiset getFileOperations(QueryRunner queryRunner) From 709767094682c1c67813b491ad193512863cf538 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 14 Jun 2026 13:09:02 +0800 Subject: [PATCH 074/255] perf(metadata): Avoid per-record enum-array clone and string parse when materializing MDT records (#18997) * perf(metadata): Avoid per-record enum-array clone and string parse when materializing MDT records Two per-record costs on the metadata-table read path: - MetadataPartitionType.get(int) iterated values(), which clones the enum constant array on every call; it runs once per record materialized from the metadata table (RLI/SI/col-stats lookups, MDT log merges). Cache values() once and iterate the cached array; the linear scan and IllegalArgumentException for unknown types are unchanged. - RECORD_INDEX.constructMetadataPayload decoded the numeric record-index fields via Long.parseLong(x.toString()) / Integer.parseInt(...) even though they are long/int in HoodieMetadata.avsc. Read them directly with ((Number) x).longValue() / .intValue(), removing five String allocations and parses per materialized RLI record. String fields keep toString(). Behavior-preserving; reconstructed records are identical. Adds an avro write/read round-trip test over both fileId encodings. Closes #18996 * review: address nit (cherry picked from commit 097dd4a4c8df98414fcfd57375fb315ce844ae95) --- .../hudi/metadata/MetadataPartitionType.java | 20 +++++++++++------ .../metadata/TestHoodieTableMetadataUtil.java | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java index 04bd9bdab2643..1e12d05b1c229 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java @@ -176,14 +176,16 @@ public void constructMetadataPayload(HoodieMetadataPayload payload, GenericRecor if (recordIndexRecord.hasField(RECORD_INDEX_FIELD_POSITION)) { recordIndexPosition = recordIndexRecord.get(RECORD_INDEX_FIELD_POSITION); } + // Numeric RLI fields are long/int per HoodieMetadata.avsc, so read them directly instead of + // round-tripping through String (toString + parse) for every materialized record. payload.recordIndexMetadata = new HoodieRecordIndexInfo(recordIndexRecord.get(RECORD_INDEX_FIELD_PARTITION).toString(), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_HIGH_BITS).toString()), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_LOW_BITS).toString()), - Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILE_INDEX).toString()), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_HIGH_BITS)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_LOW_BITS)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILE_INDEX)).intValue(), recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID).toString(), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_INSTANT_TIME).toString()), - Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING).toString()), - recordIndexPosition != null ? Long.parseLong(recordIndexPosition.toString()) : null); + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_INSTANT_TIME)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING)).intValue(), + recordIndexPosition != null ? ((Number) recordIndexPosition).longValue() : null); } }, EXPRESSION_INDEX(PARTITION_NAME_EXPRESSION_INDEX_PREFIX, "expr-index-", -1) { @@ -427,11 +429,15 @@ public static boolean shouldDeletePartitionOnRestore(String partitionPath) { && partitionType != COLUMN_STATS; } + // Cache values() once; it clones the constant array on every call, and get(int) runs once per + // record materialized from the metadata table (RLI/SI/col-stats lookups, MDT log merges). + private static final MetadataPartitionType[] VALUES = values(); + /** * Get the metadata partition type for the given record type. */ public static MetadataPartitionType get(int type) { - for (MetadataPartitionType partitionType : values()) { + for (MetadataPartitionType partitionType : VALUES) { if (partitionType.getRecordType() == type) { return partitionType; } diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java index 6cab9b9067c34..cd35f382dedba 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java @@ -18,6 +18,8 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.common.function.SerializableBiFunction; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; @@ -33,6 +35,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; import org.apache.hudi.common.util.Option; +import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -380,6 +383,25 @@ void testCreateRecordIndexUpdateMillisOverloadMatchesStringOverload() { assertEquals(fromString.getData(), fromMillis.getData()); } + @Test + void testRecordIndexPayloadRoundTripsThroughAvro() throws Exception { + // both fileId encodings populate the numeric RLI fields; they must survive the avro read path + // (constructMetadataPayload now reads the long/int fields directly instead of via toString+parse) + assertRecordIndexRoundTrips("49b8b3c8-9e5d-4731-9d51-a2d8e9b5c7f3-0", 0); + assertRecordIndexRoundTrips("some-raw-file-id", 1); + } + + private static void assertRecordIndexRoundTrips(String fileId, int fileIdEncoding) throws Exception { + HoodieRecord written = + HoodieMetadataPayload.createRecordIndexUpdate("rk1", "p1", fileId, "20260610153045678", fileIdEncoding); + // serialize to avro bytes and back so the read path sees a GenericRecord with boxed Long/Integer fields + byte[] bytes = HoodieAvroUtils.avroToBytes(written.getData().getInsertValue(null).get()); + GenericRecord deserialized = HoodieAvroUtils.bytesToAvro(bytes, HoodieMetadataRecord.getClassSchema()); + HoodieMetadataPayload readBack = new HoodieMetadataPayload(Option.of(deserialized)); + assertEquals(written.getData().recordIndexMetadata, readBack.recordIndexMetadata, + "RLI metadata must survive the avro read path for fileId encoding " + fileIdEncoding); + } + @Test void testGetLocationFromRecordIndexInfoFormatsInstantConsistently() { long instantMillis1 = HoodieMetadataPayload.parseRecordIndexInstantTime("20260610153045678"); From 4159a94a3e88c92ea4dbae5f67c817bb6de7fe88 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 16 Jun 2026 10:34:49 +0700 Subject: [PATCH 075/255] [MINOR] Cap UT_FT_10 Azure install to -T 2 to avoid flaky compiler heap OOM (#19008) The UT_FT_10 Azure job builds the full reactor with `mvn clean install` under the shared MVN_OPTS_INSTALL `-T 3` inside a single `-Xmx8g` JVM, and intermittently OOMs during compilation (most recently hudi-kafka-connect-bundle) when several heavy concurrent builds align in time. Prepend `-T 2` to this job's install so Maven (first -T wins) caps its concurrency to 2 while every other job keeps the shared -T 3. Local measurement: the full install peaks at ~2.4 GB at -T 1 and -T 2 vs ~2.9 GB at -T 3, all well under the 8g ceiling, and -T 2 keeps essentially the same wall-clock as -T 3. Co-authored-by: Vova Kolmakov (cherry picked from commit 7eaa1c3630d275925e1aaee6925b8d585dd271ce) --- azure-pipelines-20230430.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 3edac6046b1a3..ce003a13b11f3 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -515,11 +515,17 @@ stages: containerRegistry: 'apachehudi-docker-hub' repository: 'apachehudi/hudi-ci-bundle-validation-base' command: 'run' + # UT_FT_10 builds the full reactor with `clean install`. Under the + # shared MVN_OPTS_INSTALL `-T 3`, concurrent module builds (notably + # parallel maven-shade of large bundles) can spike heap past the 8g + # ceiling and flakily OOM. Prepend `-T 2` below so Maven (first -T + # wins) caps concurrency for THIS job's install only; other jobs + # keep the shared -T 3. arguments: > -v $(Build.SourcesDirectory):/hudi -v /var/run/docker.sock:/var/run/docker.sock -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) - /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source + /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2 $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source && mvn test $(MVN_OPTS_TEST) -Punit-tests -Dsurefire.failIfNoSpecifiedTests=false $(JACOCO_AGENT_DESTFILE1_ARG) -pl $(JOB10_UT_MODULES) && mvn test $(MVN_OPTS_TEST) -Punit-tests $(JACOCO_AGENT_DESTFILE2_ARG) -Dtest="!TestHoodie*" -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -Dsurefire.failIfNoSpecifiedTests=false $(JACOCO_AGENT_DESTFILE3_ARG) -pl $(JOB10_FT_MODULES)" From e2af6db031d902a23c5ec5a437e6c8101957db95 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 16 Jun 2026 11:39:49 +0800 Subject: [PATCH 076/255] refactor(metadata): Replace misused stream reduce with a plain for-loop (#18532) - HoodieTableMetadataUtil.convertMetadataToFilesPartitionRecords aggregated per-partition write stats via writeStats.stream().reduce(new HashMap<>(), accumulator, CollectionUtils::combine). - The "identity" is a mutable HashMap that the accumulator mutates in place - a misuse of Stream.reduce. - It only works because the stream is sequential and the method runs on the driver (HoodieMetadataWriteUtils then wraps the result via context.parallelize(..., 1)). - A plain for-loop expresses the same aggregation directly and is idiomatic for mutable-accumulation sequential code. - No behavior change. No measurable perf impact - readability/idiom cleanup. (cherry picked from commit dcfe9d403ecd9ad7f782e6aad4560a2727f04cf7) --- .../metadata/HoodieTableMetadataUtil.java | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index 23f20fa4efea9..4823f041bae6d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -480,31 +480,29 @@ public static List convertMetadataToFilesPartitionRecords(HoodieCo String partitionStatName = entry.getKey(); List writeStats = entry.getValue(); - HashMap updatedFilesToSizesMapping = - writeStats.stream().reduce(new HashMap<>(writeStats.size()), - (map, stat) -> { - String pathWithPartition = stat.getPath(); - if (pathWithPartition == null) { - // Empty partition - log.warn("Unable to find path in write stat to update metadata table {}", stat); - return map; - } - - String fileName = FSUtils.getFileName(pathWithPartition, partitionStatName); - - // Since write-stats are coming in no particular order, if the same - // file have previously been appended to w/in the txn, we simply pick max - // of the sizes as reported after every write, since file-sizes are - // monotonically increasing (ie file-size never goes down, unless deleted) - map.merge(fileName, stat.getFileSizeInBytes(), Math::max); - - Map cdcPathAndSizes = stat.getCdcStats(); - if (cdcPathAndSizes != null && !cdcPathAndSizes.isEmpty()) { - cdcPathAndSizes.forEach((key, value) -> map.put(FSUtils.getFileName(key, partitionStatName), value)); - } - return map; - }, - CollectionUtils::combine); + HashMap updatedFilesToSizesMapping = new HashMap<>(writeStats.size()); + for (HoodieWriteStat stat : writeStats) { + String pathWithPartition = stat.getPath(); + if (pathWithPartition == null) { + // Empty partition + log.warn("Unable to find path in write stat to update metadata table {}", stat); + continue; + } + + String fileName = FSUtils.getFileName(pathWithPartition, partitionStatName); + + // Since write-stats are coming in no particular order, if the same + // file have previously been appended to w/in the txn, we simply pick max + // of the sizes as reported after every write, since file-sizes are + // monotonically increasing (ie file-size never goes down, unless deleted) + updatedFilesToSizesMapping.merge(fileName, stat.getFileSizeInBytes(), Math::max); + + Map cdcPathAndSizes = stat.getCdcStats(); + if (cdcPathAndSizes != null && !cdcPathAndSizes.isEmpty()) { + cdcPathAndSizes.forEach((key, value) -> + updatedFilesToSizesMapping.put(FSUtils.getFileName(key, partitionStatName), value)); + } + } newFileCount.add(updatedFilesToSizesMapping.size()); return HoodieMetadataPayload.createPartitionFilesRecord(partitionStatName, updatedFilesToSizesMapping, From d994e06674d594eab5d701b361541373241f6f76 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 16 Jun 2026 11:52:34 +0800 Subject: [PATCH 077/255] perf(io): Derive log file size from AppendResult on append-handle close (#19002) * perf(io): Derive log file size from AppendResult on append-handle close HoodieAppendHandle.close() called storage.getPathInfo().getLength() for every WriteStatus to record the final log file size -- a remote HEAD per log file per file group on object stores, purely to read a size the handle already knows. (cherry picked from commit e1193c385be66f46921e2e6ed3c45d4d5f54366a) --- .../apache/hudi/io/HoodieAppendHandle.java | 16 ++--- .../common/table/log/TestLogReaderUtils.java | 58 +++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index 3b825bbf1ade7..b6389d9122f07 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -558,14 +558,16 @@ public List close() { writer = null; } - // update final size, once for all log files - // TODO we can actually deduce file size purely from AppendResult (based on offset and size - // of the appended block) + // Set the final on-disk size of each log file. Appends within an append handle are contiguous, + // so a log file's length equals its start offset plus the total bytes appended to it. That is + // exactly what fs.getFileStatus().getLength() returns, and both values are already captured by + // the AppendResult stats (logOffset and the accumulated fileSizeInBytes). Deriving the size this + // way avoids a getPathInfo/HEAD per log file, which is a remote round trip per file group on + // object stores. for (WriteStatus status : statuses) { - long logFileSize = storage.getPathInfo( - new StoragePath(config.getBasePath(), status.getStat().getPath())) - .getLength(); - status.getStat().setFileSizeInBytes(logFileSize); + HoodieDeltaWriteStat stat = (HoodieDeltaWriteStat) status.getStat(); + long appendedBytes = stat.getFileSizeInBytes(); + stat.setFileSizeInBytes(stat.getLogOffset() + appendedBytes); } // generate Secondary index stats if streaming writes is enabled. diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java index 2ddffc27d8750..f1a3f877ab1b9 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java @@ -43,6 +43,8 @@ import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.List; @@ -50,6 +52,7 @@ import java.util.Properties; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.testutils.Assertions.assertFileSizesEqual; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -131,6 +134,61 @@ public void testGetAllLogFilesWithMaxCommit() throws Exception { } } + @ParameterizedTest + @ValueSource(ints = {6, 9}) // SIX appends to the existing log file (logOffset > 0); NINE writes fresh files (logOffset == 0) + public void testLogFileWriteStatSizeMatchesOnDisk(int writeTableVersion) throws Exception { + // HoodieAppendHandle derives each log file's on-disk size from the AppendResult + // (logOffset + accumulated appended bytes) instead of a getPathInfo per file. Validate that the + // derived size in the write stat matches the actual on-disk log file length. + // + // The "logOffset +" term only contributes when a delta commit appends to a pre-existing log file: + // - table version >= EIGHT (e.g. NINE): each delta commit writes a fresh instant-named log file + // from offset 0, so logOffset is always 0; + // - table version SIX: the second upsert appends to the first commit's log file, so logOffset + // is > 0 and the derived sum is actually exercised. + // Running both covers the derivation with and without a non-zero offset. + Properties props = new Properties(); + props.setProperty(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), String.valueOf(writeTableVersion)); + HoodieTableMetaClient metaClient = getHoodieMetaClient( + storageConf(), basePath(), props, HoodieTableType.MERGE_ON_READ); + + HoodieWriteConfig config = getConfigBuilder(true) + .withPath(basePath()) + .withWriteTableVersion(writeTableVersion) + .withAutoUpgradeVersion(false) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .compactionSmallFileSize(0) + .build()) + .build(); + + HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(); + + try (SparkRDDWriteClient client = getHoodieWriteClient(config)) { + // First commit - insert data (base files) + String firstCommit = "001"; + WriteClientTestUtils.startCommitWithTime(client, firstCommit); + JavaRDD insertRdd = client.insert(jsc().parallelize(dataGen.generateInserts(firstCommit, 100), 1), firstCommit); + assertNoWriteErrors(insertRdd.collect()); + client.commit(firstCommit, insertRdd); + + // Two upsert commits. Under version SIX the second commit appends to the first's log file + // (logOffset > 0); under version >= EIGHT each commit writes a fresh log file (logOffset == 0). + for (String commitTime : new String[] {"002", "003"}) { + WriteClientTestUtils.startCommitWithTime(client, commitTime); + JavaRDD upsertRdd = client.upsert(jsc().parallelize(dataGen.generateUpdates(commitTime, 50), 1), commitTime); + List statuses = upsertRdd.collect(); + assertNoWriteErrors(statuses); + assertLogFilesProduced(statuses); + client.commit(commitTime, upsertRdd); + // Derived log file size (logOffset + appended bytes) must equal the actual on-disk length + assertFileSizesEqual(statuses, status -> FSUtils.getFileSize( + metaClient.getStorage(), + new StoragePath(config.getBasePath(), status.getStat().getPath()))); + } + } + } + @Test public void testGetAllLogFilesWithMaxCommitEmptyPartitions() throws Exception { HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, new Properties()); From b38994a22b179e36462f970391004f2a56c32e88 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 16 Jun 2026 14:14:09 +0800 Subject: [PATCH 078/255] refactor: Add Lombok annotations to hudi-common module (part 8) (#18957) * refactor: Add Lombok annotations to hudi-common module (part 8) * refactor: Keep explicit equals/hashCode and toString implementations - Restore getClass() based equals/hashCode in HoodieInstant and TimelineLayoutVersion instead of Lombok's instanceof/canEqual semantics - Restore TimelineLayoutVersion#toString: its value is persisted to hoodie.properties via String.valueOf, Lombok's format would corrupt it (cherry picked from commit b140d249b6cc624131ee0c237aa2c46f6e2c442f) --- .../table/timeline/BaseHoodieTimeline.java | 13 ++--- .../common/table/timeline/HoodieInstant.java | 33 +++-------- .../common/table/timeline/LSMTimeline.java | 11 ++-- .../timeline/MetadataConversionUtils.java | 11 ++-- .../table/timeline/TimeGeneratorBase.java | 12 ++-- .../table/timeline/TimelineDiffHelper.java | 46 +++++---------- .../common/table/timeline/TimelineLayout.java | 58 ++----------------- .../common/table/timeline/TimelineUtils.java | 13 ++--- .../versioning/TimelineLayoutVersion.java | 8 +-- .../versioning/v1/ActiveTimelineV1.java | 45 ++++++-------- .../v1/ArchivedTimelineLoaderV1.java | 10 ++-- .../versioning/v1/ArchivedTimelineV1.java | 11 +--- .../versioning/v1/CommitMetadataSerDeV1.java | 5 +- .../v1/CompletionTimeQueryViewV1.java | 9 ++- .../versioning/v2/ActiveTimelineV2.java | 48 +++++++-------- .../v2/CompletionTimeQueryViewV2.java | 7 +-- .../view/AbstractTableFileSystemView.java | 52 +++++++---------- .../table/view/FileSystemViewManager.java | 27 +++++---- .../table/view/HoodieTableFileSystemView.java | 15 ++--- ...IncrementalTimelineSyncFileSystemView.java | 56 +++++++++--------- .../view/PriorityBasedFileSystemView.java | 25 ++++---- .../view/RemoteHoodieTableFileSystemView.java | 6 +- .../view/RocksDbBasedFileSystemView.java | 50 ++++++++-------- .../view/SpillableMapBasedFileSystemView.java | 8 +-- 24 files changed, 216 insertions(+), 363 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/BaseHoodieTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/BaseHoodieTimeline.java index f32cce52c4515..aaf24bf85feee 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/BaseHoodieTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/BaseHoodieTimeline.java @@ -27,6 +27,8 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import lombok.Getter; + import java.io.IOException; import java.io.InputStream; import java.io.Serializable; @@ -60,6 +62,7 @@ public abstract class BaseHoodieTimeline implements HoodieTimeline { private static final String HASHING_ALGORITHM = "SHA-256"; + @Getter protected transient HoodieInstantReader instantReader; private List instants; // for efficient #contains queries. @@ -70,6 +73,7 @@ public abstract class BaseHoodieTimeline implements HoodieTimeline { private transient volatile Option firstNonSavepointCommit; // for efficient #isBeforeTimelineStartsByCompletionTime private transient volatile Option firstNonSavepointCommitByCompletionTime; + @Getter private String timelineHash; protected TimelineFactory factory; @@ -495,11 +499,6 @@ public boolean containsOrBeforeTimelineStarts(String instant) { return containsInstant(instant) || isBeforeTimelineStarts(instant); } - @Override - public String getTimelineHash() { - return timelineHash; - } - @Override public Stream getInstantsAsStream() { return instants.stream(); @@ -665,10 +664,6 @@ private String computeTimelineHash(List instants) { return StringUtils.toHexString(md.digest()); } - public HoodieInstantReader getInstantReader() { - return instantReader; - } - /** * Merges the given instant list into one and keep the sequence. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java index 8d874708cbcb6..70935d7e40ef4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java @@ -20,6 +20,10 @@ import org.apache.hudi.common.util.StringUtils; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.io.Serializable; import java.util.Comparator; import java.util.Objects; @@ -28,6 +32,8 @@ * A Hoodie Instant represents a action done on a hoodie table. All actions start with a inflight instant and then * create a completed instant after done. */ +@AllArgsConstructor +@Getter public class HoodieInstant implements Serializable, Comparable { public static final String FILE_NAME_FORMAT_ERROR = "The provided file name %s does not conform to the required format"; @@ -36,10 +42,12 @@ public class HoodieInstant implements Serializable, Comparable { private final State state; private final String action; + @Getter(AccessLevel.NONE) private final String requestedTime; private final String completionTime; // Marker for older formats, we need the state transition time (pre table version 7) private boolean isLegacy = false; + @Getter(AccessLevel.NONE) private final Comparator comparator; public HoodieInstant(State state, String action, String requestTime, Comparator comparator) { @@ -50,15 +58,6 @@ public HoodieInstant(State state, String action, String requestTime, String comp this(state, action, requestTime, completionTime, false, comparator); } - public HoodieInstant(State state, String action, String requestedTime, String completionTime, boolean isLegacy, Comparator comparator) { - this.state = state; - this.action = action; - this.requestedTime = requestedTime; - this.completionTime = completionTime; - this.isLegacy = isLegacy; - this.comparator = comparator; - } - public boolean isCompleted() { return state == State.COMPLETED; } @@ -71,18 +70,10 @@ public boolean isRequested() { return state == State.REQUESTED; } - public String getAction() { - return action; - } - public String requestedTime() { return requestedTime; } - public boolean isLegacy() { - return isLegacy; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -95,14 +86,6 @@ public boolean equals(Object o) { return state == that.state && Objects.equals(action, that.action) && Objects.equals(requestedTime, that.requestedTime); } - public State getState() { - return state; - } - - public String getCompletionTime() { - return completionTime; - } - @Override public int hashCode() { return Objects.hash(state, action, requestedTime); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java index 19938f7cd7cc9..267e3f1585748 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java @@ -29,9 +29,8 @@ import org.apache.hudi.storage.StoragePathFilter; import org.apache.hudi.storage.StoragePathInfo; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; @@ -103,8 +102,8 @@ *

    Instants TTL

    * The timeline reader only reads instants of last limited days. We will by default skip the instants from LSM timeline that are generated long time ago. */ +@Slf4j public class LSMTimeline { - private static final Logger LOG = LoggerFactory.getLogger(LSMTimeline.class); public static final int LSM_TIMELINE_INSTANT_VERSION_1 = 1; @@ -158,7 +157,7 @@ public static int latestSnapshotVersion(HoodieTableMetaClient metaClient, Storag } } catch (Exception e) { // fallback to manifest file listing. - LOG.warn("Error reading version file {}", versionFilePath, e); + log.warn("Error reading version file {}", versionFilePath, e); } return allSnapshotVersions(metaClient, archivePath).stream().max(Integer::compareTo).orElse(-1); @@ -176,7 +175,7 @@ public static List allSnapshotVersions(HoodieTableMetaClient metaClient .map(LSMTimeline::getManifestVersion) .collect(Collectors.toList()); } catch (FileNotFoundException ex) { - LOG.debug("Archive path {} does not exist", archivePath); + log.debug("Archive path {} does not exist", archivePath); return Collections.emptyList(); } } @@ -256,7 +255,7 @@ public static int getFileLayer(String fileName) { } } catch (NumberFormatException e) { // log and ignore any format warnings - LOG.warn("error getting file layout for archived file: {}", fileName, e); + log.warn("error getting file layout for archived file: {}", fileName, e); } // return default value in case of any errors diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java index e28af75a1ef75..8ea9ccc03fb9a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java @@ -40,10 +40,9 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.avro.specific.SpecificRecordBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -52,8 +51,8 @@ /** * Helper class to convert between different action related payloads and {@link HoodieArchivedMetaEntry}. */ +@Slf4j public class MetadataConversionUtils { - private static final Logger LOG = LoggerFactory.getLogger(MetadataConversionUtils.class); public static HoodieArchivedMetaEntry createMetaWrapper(HoodieInstant hoodieInstant, HoodieTableMetaClient metaClient) { try { @@ -387,17 +386,17 @@ public static void removeNullKeyFromMapMembersForCommitMetadata(T metadata) if (metadata instanceof HoodieReplaceCommitMetadata) { HoodieReplaceCommitMetadata hoodieCommitMetadata = (HoodieReplaceCommitMetadata) metadata; if (hoodieCommitMetadata.getPartitionToWriteStats().containsKey(null)) { - LOG.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToWriteStats().get(null)); + log.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToWriteStats().get(null)); hoodieCommitMetadata.getPartitionToWriteStats().remove(null); } if (hoodieCommitMetadata.getPartitionToReplaceFileIds().containsKey(null)) { - LOG.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToReplaceFileIds().get(null)); + log.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToReplaceFileIds().get(null)); hoodieCommitMetadata.getPartitionToReplaceFileIds().remove(null); } } else if (metadata instanceof HoodieCommitMetadata) { HoodieCommitMetadata hoodieCommitMetadata = (HoodieCommitMetadata) metadata; if (hoodieCommitMetadata.getPartitionToWriteStats().containsKey(null)) { - LOG.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToWriteStats().get(null)); + log.info("partition path is null for {}", hoodieCommitMetadata.getPartitionToWriteStats().get(null)); hoodieCommitMetadata.getPartitionToWriteStats().remove(null); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java index b616640a90290..18d187e2fdf12 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java @@ -26,8 +26,7 @@ import org.apache.hudi.exception.HoodieLockException; import org.apache.hudi.storage.StorageConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.util.Arrays; @@ -42,10 +41,9 @@ /** * Base time generator facility that maintains lock-related utilities. */ +@Slf4j public abstract class TimeGeneratorBase implements TimeGenerator, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(TimeGeneratorBase.class); - /** * The lock provider. */ @@ -87,7 +85,7 @@ protected LockProvider getLockProvider() { synchronized (this) { if (lockProvider == null) { String lockProviderClass = lockConfiguration.getConfig().getString("hoodie.write.lock.provider"); - LOG.info("LockProvider for TimeGenerator: {}", lockProviderClass); + log.info("LockProvider for TimeGenerator: {}", lockProviderClass); lockProvider = (LockProvider) ReflectionUtils.loadClass(lockProviderClass, new Class[] {LockConfiguration.class, StorageConfiguration.class}, lockConfiguration, storageConf); @@ -121,10 +119,10 @@ private synchronized void closeQuietly() { if (lockProvider != null) { lockProvider.close(); lockProvider = null; - LOG.info("Released the connection of the timeGenerator lock"); + log.info("Released the connection of the timeGenerator lock"); } } catch (Exception e) { - LOG.info("Unable to release the connection of the timeGenerator lock"); + log.info("Unable to release the connection of the timeGenerator lock"); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineDiffHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineDiffHelper.java index 502565dd0221a..a0159dd2bc3ec 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineDiffHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineDiffHelper.java @@ -23,8 +23,12 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.List; @@ -37,13 +41,10 @@ /** * A helper class used to diff timeline. */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@Slf4j public class TimelineDiffHelper { - private static final Logger LOG = LoggerFactory.getLogger(TimelineDiffHelper.class); - - private TimelineDiffHelper() { - } - public static TimelineDiffResult getNewInstantsForIncrementalSync(HoodieTableMetaClient metaClient, HoodieTimeline oldTimeline, HoodieTimeline newTimeline) { @@ -72,7 +73,7 @@ public static TimelineDiffResult getNewInstantsForIncrementalSync(HoodieTableMet if (!lostPendingCompactions.isEmpty()) { // If a compaction is unscheduled, fall back to complete refresh of fs view since some log files could have been // moved. Its unsafe to incrementally sync in that case. - LOG.warn("Some pending compactions are no longer in new timeline (unscheduled ?). They are: {}", lostPendingCompactions); + log.warn("Some pending compactions are no longer in new timeline (unscheduled ?). They are: {}", lostPendingCompactions); return TimelineDiffResult.UNSAFE_SYNC_RESULT; } List finishedCompactionInstants = compactionInstants.stream() @@ -91,7 +92,7 @@ public static TimelineDiffResult getNewInstantsForIncrementalSync(HoodieTableMet return new TimelineDiffResult(newInstants, finishedCompactionInstants, finishedOrRemovedLogCompactionInstants, true); } else { // One or more timelines is empty - LOG.warn("One or more timelines is empty"); + log.warn("One or more timelines is empty"); return TimelineDiffResult.UNSAFE_SYNC_RESULT; } } @@ -125,40 +126,19 @@ private static List> getPendingActionTransiti /** * A diff result of timeline. */ + @AllArgsConstructor + @Getter public static class TimelineDiffResult { private final List newlySeenInstants; private final List finishedCompactionInstants; private final List finishedOrRemovedLogCompactionInstants; + @Accessors(fluent = true) private final boolean canSyncIncrementally; public static final TimelineDiffResult UNSAFE_SYNC_RESULT = new TimelineDiffResult(null, null, null, false); - public TimelineDiffResult(List newlySeenInstants, List finishedCompactionInstants, - List finishedOrRemovedLogCompactionInstants, boolean canSyncIncrementally) { - this.newlySeenInstants = newlySeenInstants; - this.finishedCompactionInstants = finishedCompactionInstants; - this.finishedOrRemovedLogCompactionInstants = finishedOrRemovedLogCompactionInstants; - this.canSyncIncrementally = canSyncIncrementally; - } - - public List getNewlySeenInstants() { - return newlySeenInstants; - } - - public List getFinishedCompactionInstants() { - return finishedCompactionInstants; - } - - public List getFinishedOrRemovedLogCompactionInstants() { - return finishedOrRemovedLogCompactionInstants; - } - - public boolean canSyncIncrementally() { - return canSyncIncrementally; - } - @Override public String toString() { return "TimelineDiffResult{" diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineLayout.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineLayout.java index d4ed4b53c916c..83ce1f20d2c25 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineLayout.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineLayout.java @@ -34,6 +34,8 @@ import org.apache.hudi.common.table.timeline.versioning.v2.TimelineV2Factory; import org.apache.hudi.common.util.collection.Pair; +import lombok.Getter; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -81,44 +83,20 @@ public static TimelineLayout fromVersion(TimelineLayoutVersion version) { /** * Table Layout where state transitions are managed by renaming files. */ + @Getter private static class TimelineLayoutV0 extends TimelineLayout { private final InstantGenerator instantGenerator = new InstantGeneratorV1(); private final InstantFileNameGenerator instantFileNameGenerator = new InstantFileNameGeneratorV1(); private final TimelineFactory timelineFactory = new TimelineV1Factory(this); private final InstantComparator instantComparator = new InstantComparatorV1(); - private final InstantFileNameParser fileNameParser = new InstantFileNameParserV2(); + private final InstantFileNameParser instantFileNameParser = new InstantFileNameParserV2(); @Override public Stream filterHoodieInstants(Stream instantStream) { return instantStream; } - @Override - public InstantGenerator getInstantGenerator() { - return instantGenerator; - } - - @Override - public InstantFileNameGenerator getInstantFileNameGenerator() { - return instantFileNameGenerator; - } - - @Override - public TimelineFactory getTimelineFactory() { - return timelineFactory; - } - - @Override - public InstantComparator getInstantComparator() { - return instantComparator; - } - - @Override - public InstantFileNameParser getInstantFileNameParser() { - return fileNameParser; - } - @Override public CommitMetadataSerDe getCommitMetadataSerDe() { return new CommitMetadataSerDeV1(); @@ -157,44 +135,20 @@ public Stream filterHoodieInstants(Stream instantS /** * Timeline corresponding to Hudi 1.x */ + @Getter private static class TimelineLayoutV2 extends TimelineLayout { private final InstantGenerator instantGenerator = new InstantGeneratorV2(); private final InstantFileNameGenerator instantFileNameGenerator = new InstantFileNameGeneratorV2(); private final TimelineFactory timelineFactory = new TimelineV2Factory(this); private final InstantComparator instantComparator = new InstantComparatorV2(); - private final InstantFileNameParser fileNameParser = new InstantFileNameParserV2(); + private final InstantFileNameParser instantFileNameParser = new InstantFileNameParserV2(); @Override public Stream filterHoodieInstants(Stream instantStream) { return TimelineLayout.filterHoodieInstantsByLatestState(instantStream, InstantComparatorV2::getComparableAction); } - @Override - public InstantGenerator getInstantGenerator() { - return instantGenerator; - } - - @Override - public InstantFileNameGenerator getInstantFileNameGenerator() { - return instantFileNameGenerator; - } - - @Override - public TimelineFactory getTimelineFactory() { - return timelineFactory; - } - - @Override - public InstantComparator getInstantComparator() { - return instantComparator; - } - - @Override - public InstantFileNameParser getInstantFileNameParser() { - return fileNameParser; - } - @Override public CommitMetadataSerDe getCommitMetadataSerDe() { return new CommitMetadataSerDeV2(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java index 4b2e6ee55fc62..6c27d1f8e2a67 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java @@ -40,8 +40,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; @@ -81,6 +80,7 @@ * 1) HiveSync - this can be used to query partitions that changed since previous sync. * 2) Incremental reads - InputFormats can use this API to query */ +@Slf4j public class TimelineUtils { public static final Set NOT_PARSABLE_TIMESTAMPS = new HashSet(3) { { @@ -89,7 +89,6 @@ public class TimelineUtils { add(HoodieTimeline.FULL_BOOTSTRAP_INSTANT_TS); } }; - private static final Logger LOG = LoggerFactory.getLogger(TimelineUtils.class); /** * Returns partitions that have new data strictly after commitTime. @@ -137,7 +136,7 @@ public static List getDroppedPartitions(HoodieTableMetaClient metaClient }); } catch (HoodieIOException e) { if (e.getCause() instanceof FileNotFoundException) { - LOG.warn("Instant {} not found in storage and has been archived", instant, e); + log.warn("Instant {} not found in storage and has been archived", instant, e); } else { throw e; } @@ -264,7 +263,7 @@ public static Map> getAllExtraMetadataForKey(HoodieTableM private static Option getMetadataValue(HoodieTableMetaClient metaClient, String extraMetadataKey, HoodieInstant instant) { try { - LOG.info("reading checkpoint info for:" + instant + " key: " + extraMetadataKey); + log.info("reading checkpoint info for:" + instant + " key: " + extraMetadataKey); byte[] contents = metaClient.getCommitsTimeline().getInstantDetails(instant).get(); if (instant.isCompleted()) { if (contents == null || contents.length == 0) { @@ -476,7 +475,7 @@ public static HoodieTimeline handleHollowCommitIfNeeded(HoodieTimeline completed "Found hollow commit: '%s'. Adjust config `%s` accordingly if to avoid throwing this exception.", hollowCommitTimestamp, INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT.key())); case BLOCK: - LOG.warn("Found hollow commit '{}'. Config `{}` was set to `{}`: no data will be returned beyond '{}' until it's completed.", + log.warn("Found hollow commit '{}'. Config `{}` was set to `{}`: no data will be returned beyond '{}' until it's completed.", hollowCommitTimestamp, INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT.key(), handlingMode, hollowCommitTimestamp); return completedCommitTimeline.findInstantsBefore(hollowCommitTimestamp); default: @@ -518,7 +517,7 @@ public static Option parseDateFromInstantTimeSafely(String timestamp) { if (NOT_PARSABLE_TIMESTAMPS.contains(timestamp)) { parsedDate = Option.of(new Date(Integer.parseInt(timestamp))); } else { - LOG.warn("Failed to parse timestamp {}: {}", timestamp, e.getMessage()); + log.warn("Failed to parse timestamp {}: {}", timestamp, e.getMessage()); parsedDate = Option.empty(); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/TimelineLayoutVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/TimelineLayoutVersion.java index feceb81e33b62..cb2cee9f30ac0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/TimelineLayoutVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/TimelineLayoutVersion.java @@ -20,12 +20,15 @@ import org.apache.hudi.common.util.ValidationUtils; +import lombok.Getter; + import java.io.Serializable; import java.util.Objects; /** * Metadata Layout Version. Add new version when timeline format changes */ +@Getter public class TimelineLayoutVersion implements Serializable, Comparable { public static final Integer VERSION_0 = 0; // pre 0.5.1 version format @@ -38,7 +41,6 @@ public class TimelineLayoutVersion implements Serializable, Comparable VALID_EXTENSIONS_IN_ACTIVE_TIMELINE = new HashSet<>(Arrays.asList( @@ -77,7 +80,6 @@ public class ActiveTimelineV1 extends BaseTimelineV1 implements HoodieActiveTime REQUESTED_INDEX_COMMIT_EXTENSION, INFLIGHT_INDEX_COMMIT_EXTENSION, INDEX_COMMIT_EXTENSION, REQUESTED_SAVE_SCHEMA_ACTION_EXTENSION, INFLIGHT_SAVE_SCHEMA_ACTION_EXTENSION, SAVE_SCHEMA_ACTION_EXTENSION)); - private static final Logger LOG = LoggerFactory.getLogger(ActiveTimelineV1.class); protected HoodieTableMetaClient metaClient; private final InstantFileNameGenerator instantFileNameGenerator = new InstantFileNameGeneratorV1(); @@ -89,7 +91,7 @@ protected ActiveTimelineV1(HoodieTableMetaClient metaClient, Set include this.metaClient = metaClient; // multiple casts will make this lambda serializable - // http://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.16 - LOG.debug("Loaded instants upto : " + lastInstant()); + log.debug("Loaded instants upto : {}", lastInstant()); } public ActiveTimelineV1(HoodieTableMetaClient metaClient) { @@ -100,15 +102,6 @@ public ActiveTimelineV1(HoodieTableMetaClient metaClient, boolean applyLayoutFil this(metaClient, Collections.unmodifiableSet(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE), applyLayoutFilter); } - /** - * For serialization and de-serialization only. - * - * @deprecated - */ - @Deprecated - public ActiveTimelineV1() { - } - /** * This method is only used when this object is deserialized in a spark executor. * @@ -126,13 +119,13 @@ public Set getValidExtensionsInActiveTimeline() { @Override public void createCompleteInstant(HoodieInstant instant) { - LOG.info("Creating a new complete instant {}", instant); + log.info("Creating a new complete instant {}", instant); createFileInMetaPath(instantFileNameGenerator.getFileName(instant), Option.empty(), false); } @Override public void createNewInstant(HoodieInstant instant) { - LOG.info("Creating a new instant {}", instant); + log.info("Creating a new instant {}", instant); // Create the in-flight file createFileInMetaPath(instantFileNameGenerator.getFileName(instant), Option.empty(), false); } @@ -140,7 +133,7 @@ public void createNewInstant(HoodieInstant instant) { @Override public HoodieInstant createRequestedCommitWithReplaceMetadata(String instantTime, String actionType) { HoodieInstant instant = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, actionType, instantTime); - LOG.info("Creating a new instant {}", instant); + log.info("Creating a new instant {}", instant); // Create the request replace file createFileInMetaPath(instantFileNameGenerator.getFileName(instant), Option.of(new HoodieRequestedReplaceMetadata()), false); return instant; @@ -148,12 +141,12 @@ public HoodieInstant createRequestedCommitWithReplaceMetadata(String instantTime @Override public HoodieInstant saveAsComplete(HoodieInstant instant, Option metadata) { - LOG.info("Marking instant complete " + instant); + log.info("Marking instant complete {}", instant); ValidationUtils.checkArgument(instant.isInflight(), "Could not mark an already completed instant as complete again " + instant); HoodieInstant completedInstant = instantGenerator.createNewInstant(HoodieInstant.State.COMPLETED, instant.getAction(), instant.requestedTime()); transitionState(instant, completedInstant, metadata); - LOG.info("Completed {}", instant); + log.info("Completed {}", instant); return completedInstant; } @@ -176,10 +169,10 @@ public HoodieInstant saveAsComplete(boolean shouldLock, HoodieInstant instan @Override public HoodieInstant revertToInflight(HoodieInstant instant) { - LOG.info("Reverting instant to inflight {}", instant); + log.info("Reverting instant to inflight {}", instant); HoodieInstant inflight = TimelineUtils.getInflightInstant(instant, metaClient); revertCompleteToInflight(instant, inflight); - LOG.info("Reverted {} to inflight {}", instant, inflight); + log.info("Reverted {} to inflight {}", instant, inflight); return inflight; } @@ -216,18 +209,18 @@ public void deleteCompactionRequested(HoodieInstant instant) { @Override public void deleteInstantFileIfExists(HoodieInstant instant) { - LOG.info("Deleting instant {}", instant); + log.info("Deleting instant {}", instant); StoragePath commitFilePath = getInstantFileNamePath(instantFileNameGenerator.getFileName(instant)); try { if (metaClient.getStorage().exists(commitFilePath)) { boolean result = metaClient.getStorage().deleteFile(commitFilePath); if (result) { - LOG.info("Removed instant {}", instant); + log.info("Removed instant {}", instant); } else { throw new HoodieIOException("Could not delete instant " + instant + " with path " + commitFilePath); } } else { - LOG.info("The commit {} to remove does not exist", commitFilePath); + log.info("The commit {} to remove does not exist", commitFilePath); } } catch (IOException e) { throw new HoodieIOException("Could not remove commit " + commitFilePath, e); @@ -235,12 +228,12 @@ public void deleteInstantFileIfExists(HoodieInstant instant) { } private void deleteInstantFile(HoodieInstant instant) { - LOG.info("Deleting instant {}", instant); + log.info("Deleting instant {}", instant); StoragePath inFlightCommitFilePath = getInstantFileNamePath(instantFileNameGenerator.getFileName(instant)); try { boolean result = metaClient.getStorage().deleteFile(inFlightCommitFilePath); if (result) { - LOG.info("Removed instant {}", instant); + log.info("Removed instant {}", instant); } else { throw new HoodieIOException("Could not delete instant " + instant + " with path " + inFlightCommitFilePath); } @@ -541,7 +534,7 @@ protected void transitionState(HoodieInstant fromInstant, HoodieInstant toIn } else { storage.createImmutableFileInPath(getInstantFileNamePath(instantFileNameGenerator.getFileName(toInstant)), getHoodieInstantWriterOption(this, metadata)); } - LOG.info("Create new file for toInstant ?{}", getInstantFileNamePath(instantFileNameGenerator.getFileName(toInstant))); + log.info("Create new file for toInstant? {}", getInstantFileNamePath(instantFileNameGenerator.getFileName(toInstant))); } } catch (IOException e) { throw new HoodieIOException("Could not complete " + fromInstant, e); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java index 8485dc7405d92..d95a8aab0c686 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java @@ -39,10 +39,9 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -61,13 +60,14 @@ import java.util.regex.Pattern; import java.util.stream.StreamSupport; +@Slf4j public class ArchivedTimelineLoaderV1 implements ArchivedTimelineLoader { + private static final String MERGE_ARCHIVE_PLAN_NAME = "mergeArchivePlan"; private static final Pattern ARCHIVE_FILE_PATTERN = Pattern.compile("^\\.commits_\\.archive\\.([0-9]+).*"); private static final String STATE_TRANSITION_TIME = "stateTransitionTime"; private static final String ACTION_TYPE_KEY = "actionType"; - private static final Logger LOG = LoggerFactory.getLogger(ArchivedTimelineLoaderV1.class); @Override public void loadInstants(HoodieTableMetaClient metaClient, @@ -172,7 +172,7 @@ public void loadInstants(HoodieTableMetaClient metaClient, HoodieMergeArchiveFilePlan plan = TimelineMetadataUtils.deserializeAvroMetadataLegacy(FileIOUtils.readDataFromPath(storage, planPath).get(), HoodieMergeArchiveFilePlan.class); String mergedArchiveFileName = plan.getMergedArchiveFileName(); if (!StringUtils.isNullOrEmpty(mergedArchiveFileName) && fs.getPath().getName().equalsIgnoreCase(mergedArchiveFileName)) { - LOG.debug("Catch exception because of reading uncompleted merging archive file {}. Ignore it here.", mergedArchiveFileName); + log.debug("Catch exception because of reading uncompleted merging archive file {}. Ignore it here.", mergedArchiveFileName); continue; } } @@ -207,7 +207,7 @@ private int getArchivedFileSuffix(StoragePathInfo f) { } } catch (NumberFormatException e) { // log and ignore any format warnings - LOG.warn("error getting suffix for archived file: {}", f.getPath()); + log.warn("error getting suffix for archived file: {}", f.getPath()); } // return default value in case of any errors diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineV1.java index 6fa23ec95c46f..aeea6e20c8686 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineV1.java @@ -28,6 +28,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.NoArgsConstructor; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.slf4j.Logger; @@ -53,6 +54,8 @@ import static org.apache.hudi.common.table.timeline.TimelineUtils.getInputStreamOptionLegacy; +// no-arg constructor is for serialization and de-serialization only +@NoArgsConstructor(onConstructor_ = @Deprecated) public class ArchivedTimelineV1 extends BaseTimelineV1 implements HoodieArchivedTimeline, HoodieInstantReader { private static final String HOODIE_COMMIT_ARCHIVE_LOG_FILE_PREFIX = "commits"; private static final String ACTION_TYPE_KEY = "actionType"; @@ -155,14 +158,6 @@ public ArchivedTimelineV1(HoodieTableMetaClient metaClient, Set logFiles this(metaClient, null, new LogFileFilter(logFiles), state); } - /** - * For serialization and de-serialization only. - * - * @deprecated - */ - public ArchivedTimelineV1() { - } - @Override public HoodieInstantReader getInstantReader() { return this; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CommitMetadataSerDeV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CommitMetadataSerDeV1.java index 24be7d83aedc3..52a24dcc9d0bc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CommitMetadataSerDeV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CommitMetadataSerDeV1.java @@ -26,9 +26,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.storage.HoodieInstantWriter; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.specific.SpecificRecordBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; @@ -37,8 +36,8 @@ import static org.apache.hudi.common.table.timeline.MetadataConversionUtils.removeNullKeyFromMapMembersForCommitMetadata; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata; +@Slf4j public class CommitMetadataSerDeV1 implements CommitMetadataSerDe { - private static final Logger LOG = LoggerFactory.getLogger(CommitMetadataSerDeV1.class); @Override public T deserialize(HoodieInstant instant, InputStream inputStream, BooleanSupplier isEmptyInstant, Class clazz) throws IOException { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CompletionTimeQueryViewV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CompletionTimeQueryViewV1.java index f996627801766..18aeae515ec3f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CompletionTimeQueryViewV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/CompletionTimeQueryViewV1.java @@ -28,6 +28,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.VisibleForTesting; +import lombok.Getter; + import java.io.Serializable; import java.time.Instant; import java.util.Date; @@ -40,6 +42,7 @@ import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN; public class CompletionTimeQueryViewV1 implements CompletionTimeQueryView, Serializable { + private static final long serialVersionUID = 1L; private static final long MILLI_SECONDS_IN_THREE_DAYS = 3 * 24 * 3600 * 1000; @@ -60,6 +63,7 @@ public class CompletionTimeQueryViewV1 implements CompletionTimeQueryView, Seria * a completion query for t5 would trigger lazy loading with this cursor instant updated to t5. * This sliding window model amortizes redundant loading from different queries. */ + @Getter private final String cursorInstant; /** @@ -229,11 +233,6 @@ private void setCompletionTime(String beginInstantTime, String completionTime) { this.beginToCompletionInstantTimeMap.putIfAbsent(beginInstantTime, completionTime); } - @Override - public String getCursorInstant() { - return cursorInstant; - } - @Override public boolean isEmptyTable() { return this.beginToCompletionInstantTimeMap.isEmpty(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ActiveTimelineV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ActiveTimelineV2.java index 69e36596e48da..7a45582bad603 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ActiveTimelineV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ActiveTimelineV2.java @@ -50,8 +50,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.InputStream; @@ -67,6 +67,9 @@ import static org.apache.hudi.common.table.timeline.TimelineUtils.getHoodieInstantWriterOption; +// no-arg constructor is for serialization and de-serialization only; @Deprecated marks it as such +@NoArgsConstructor(onConstructor_ = @Deprecated) +@Slf4j public class ActiveTimelineV2 extends BaseTimelineV2 implements HoodieActiveTimeline { public static final Set VALID_EXTENSIONS_IN_ACTIVE_TIMELINE = new HashSet<>(Arrays.asList( @@ -82,8 +85,6 @@ public class ActiveTimelineV2 extends BaseTimelineV2 implements HoodieActiveTime REQUESTED_INDEX_COMMIT_EXTENSION, INFLIGHT_INDEX_COMMIT_EXTENSION, INDEX_COMMIT_EXTENSION, REQUESTED_SAVE_SCHEMA_ACTION_EXTENSION, INFLIGHT_SAVE_SCHEMA_ACTION_EXTENSION, SAVE_SCHEMA_ACTION_EXTENSION, REQUESTED_CLUSTERING_COMMIT_EXTENSION, INFLIGHT_CLUSTERING_COMMIT_EXTENSION)); - - private static final Logger LOG = LoggerFactory.getLogger(ActiveTimelineV2.class); protected HoodieTableMetaClient metaClient; private final InstantFileNameGenerator instantFileNameGenerator = new InstantFileNameGeneratorV2(); @@ -95,7 +96,7 @@ private ActiveTimelineV2(HoodieTableMetaClient metaClient, Set includedE this.metaClient = metaClient; // multiple casts will make this lambda serializable - // http://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.16 - LOG.debug("Loaded instants upto : {}", lastInstant()); + log.debug("Loaded instants upto: {}", lastInstant()); } public ActiveTimelineV2(HoodieTableMetaClient metaClient) { @@ -106,15 +107,6 @@ public ActiveTimelineV2(HoodieTableMetaClient metaClient, boolean applyLayoutFil this(metaClient, Collections.unmodifiableSet(VALID_EXTENSIONS_IN_ACTIVE_TIMELINE), applyLayoutFilter); } - /** - * For serialization and de-serialization only. - * - * @deprecated - */ - @Deprecated - public ActiveTimelineV2() { - } - /** * This method is only used when this object is deserialized in a spark executor. * @@ -132,13 +124,13 @@ public Set getValidExtensionsInActiveTimeline() { @Override public void createCompleteInstant(HoodieInstant instant) { - LOG.info("Creating a new complete instant " + instant); + log.info("Creating a new complete instant {}", instant); createCompleteFileInMetaPath(true, instant, Option.empty()); } @Override public void createNewInstant(HoodieInstant instant) { - LOG.info("Creating a new instant " + instant); + log.info("Creating a new instant: {}", instant); ValidationUtils.checkArgument(!instant.isCompleted()); createFileInMetaPath(instantFileNameGenerator.getFileName(instant), Option.empty(), false); } @@ -146,7 +138,7 @@ public void createNewInstant(HoodieInstant instant) { @Override public HoodieInstant createRequestedCommitWithReplaceMetadata(String instantTime, String actionType) { HoodieInstant instant = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, actionType, instantTime); - LOG.info("Creating a new instant " + instant); + log.info("Creating a new instant: {}", instant); // Create the request replace file createFileInMetaPath(instantFileNameGenerator.getFileName(instant), Option.of(new HoodieRequestedReplaceMetadata()), false); return instant; @@ -164,12 +156,12 @@ public HoodieInstant saveAsComplete(boolean shouldLock, HoodieInstant instan @Override public HoodieInstant saveAsComplete(boolean shouldLock, HoodieInstant instant, Option metadata, Option completionTimeOpt) { - LOG.info("Marking instant complete {}", instant); + log.info("Marking instant complete {}", instant); ValidationUtils.checkArgument(instant.isInflight(), "Could not mark an already completed instant as complete again " + instant); HoodieInstant commitInstant = instantGenerator.createNewInstant(HoodieInstant.State.COMPLETED, instant.getAction(), instant.requestedTime()); HoodieInstant completedInstant = transitionStateToComplete(shouldLock, instant, commitInstant, metadata, completionTimeOpt); - LOG.info("Completed " + instant); + log.info("Completed {}", instant); return completedInstant; } @@ -182,10 +174,10 @@ public HoodieInstant saveAsComplete(boolean shouldLock, HoodieInstant instan @Override public HoodieInstant revertToInflight(HoodieInstant instant) { - LOG.info("Reverting instant to inflight {}", instant); + log.info("Reverting instant to inflight {}", instant); HoodieInstant inflight = TimelineUtils.getInflightInstant(instant, metaClient); revertCompleteToInflight(instant, inflight); - LOG.info("Reverted {} to inflight {}", instant, inflight); + log.info("Reverted {} to inflight {}", instant, inflight); return inflight; } @@ -223,18 +215,18 @@ public void deleteCompactionRequested(HoodieInstant instant) { @Override public void deleteInstantFileIfExists(HoodieInstant instant) { - LOG.info("Deleting instant {}", instant); + log.info("Deleting instant {}", instant); StoragePath commitFilePath = getInstantFileNamePath(instantFileNameGenerator.getFileName(instant)); try { if (metaClient.getStorage().exists(commitFilePath)) { boolean result = metaClient.getStorage().deleteFile(commitFilePath); if (result) { - LOG.info("Removed instant {}", instant); + log.info("Removed instant {}", instant); } else { throw new HoodieIOException("Could not delete instant " + instant + " with path " + commitFilePath); } } else { - LOG.info("The commit {} to remove does not exist", commitFilePath); + log.info("The commit {} to remove does not exist", commitFilePath); } } catch (IOException e) { throw new HoodieIOException("Could not remove commit " + commitFilePath, e); @@ -242,12 +234,12 @@ public void deleteInstantFileIfExists(HoodieInstant instant) { } protected void deleteInstantFile(HoodieInstant instant) { - LOG.info("Deleting instant {}", instant); + log.info("Deleting instant {}", instant); StoragePath filePath = getInstantFileNamePath(instantFileNameGenerator.getFileName(instant)); try { boolean result = metaClient.getStorage().deleteFile(filePath); if (result) { - LOG.info("Removed instant {}", instant); + log.info("Removed instant {}", instant); } else { throw new HoodieIOException("Could not delete instant " + instant + " with path " + filePath); } @@ -602,7 +594,7 @@ protected void transitionPendingState( } else { storage.createImmutableFileInPath(getInstantFileNamePath(toInstantFileName), getInstantWriter(metadata)); } - LOG.info("Create new file for toInstant ?{}", getInstantFileNamePath(toInstantFileName)); + log.info("Create new file for toInstant ?{}", getInstantFileNamePath(toInstantFileName)); } } catch (IOException e) { throw new HoodieIOException("Could not complete " + fromInstant, e); @@ -763,7 +755,7 @@ protected String createCompleteFileInMetaPath(boolean shouldLock, HoodieInst metaClient.getStorage().createImmutableFileInPath(fullPath, writerOption); } completionTimeRef.set(completionTime); - LOG.info("Created new file for toInstant: {}", fullPath); + log.info("Created new file for toInstant: {}", fullPath); }); return completionTimeRef.get(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java index e773c40692ee5..516b4ad4cedd8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.VisibleForTesting; +import lombok.Getter; import org.apache.avro.generic.GenericRecord; import java.io.Serializable; @@ -73,6 +74,7 @@ public class CompletionTimeQueryViewV2 implements CompletionTimeQueryView, Seria * a completion query for t5 would trigger lazy loading with this cursor instant updated to t5. * This sliding window model amortizes redundant loading from different queries. */ + @Getter private volatile String cursorInstant; /** @@ -313,11 +315,6 @@ private void setCompletionTime(String beginInstantTime, String completionTime) { this.instantTimeToCompletionTimeMap.putIfAbsent(beginInstantTime, completionTime); } - @Override - public String getCursorInstant() { - return cursorInstant; - } - @Override public boolean isEmptyTable() { return this.instantTimeToCompletionTimeMap.isEmpty(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index 9c4debe193659..c9468f152fbd6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -50,8 +50,8 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.FileNotFoundException; import java.io.IOException; @@ -93,9 +93,9 @@ * * The actual mechanism of fetching file slices from different view storages is delegated to sub-classes. */ +@Slf4j public abstract class AbstractTableFileSystemView implements SyncableFileSystemView, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(AbstractTableFileSystemView.class); protected final HoodieTableMetadata tableMetadata; protected HoodieTableMetaClient metaClient; @@ -104,13 +104,14 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV // This is the commits timeline that will be visible for all views extending this view // This is nothing but the write timeline, which contains both ingestion and compaction(major and minor) writers. + @Getter private HoodieTimeline visibleCommitsAndCompactionTimeline; // Used to concurrently load and populate partition views private final ConcurrentHashMap addedPartitions = new ConcurrentHashMap<>(4096); // Sampling logger for replaced file groups read logs (log at INFO once every 5 times) - private final SamplingLogger replacedFileGroupsReadSamplingLogger = new SamplingLogger(LOG, 5); + private final SamplingLogger replacedFileGroupsReadSamplingLogger = new SamplingLogger(log, 5); // Locks to control concurrency. Sync operations use write-lock blocking all fetch operations. // For the common-case, we allow concurrent read of single or multiple partitions @@ -199,7 +200,7 @@ public List addFilesToView(String partitionPath, List sourceFileMappings = reader.getSourceFileMappingForPartition(partition); addBootstrapBaseFileMapping(sourceFileMappings.stream() @@ -211,7 +212,7 @@ public List addFilesToView(String partitionPath, List partitionList) { try { // For metadata table, log at DEBUG. For data table, log at INFO. if (metaClient.isMetadataTable()) { - LOG.debug("Building file system view for {} partition(s)", partitionSet.size()); + log.debug("Building file system view for {} partition(s)", partitionSet.size()); } else { - LOG.info("Building file system view for {} partition(s)", partitionSet.size()); + log.info("Building file system view for {} partition(s)", partitionSet.size()); } // Pairs of relative partition path and absolute partition path @@ -419,20 +420,20 @@ private void ensurePartitionsLoadedCorrectly(List partitionList) { Map, List> pathInfoMap = tableMetadata.listPartitions(absolutePartitionPathList); long endLsTs = System.currentTimeMillis(); - LOG.debug("Time taken to list partitions {} ={}", partitionSet, (endLsTs - beginLsTs)); + log.debug("Time taken to list partitions {} ={}", partitionSet, (endLsTs - beginLsTs)); pathInfoMap.forEach((partitionPair, statuses) -> { String relativePartitionStr = partitionPair.getLeft(); List groups = addFilesToView(relativePartitionStr, statuses); if (groups.isEmpty()) { storePartitionView(relativePartitionStr, Collections.emptyList()); } - LOG.debug("#files found in partition ({}) ={}", relativePartitionStr, statuses.size()); + log.debug("#files found in partition ({}) ={}", relativePartitionStr, statuses.size()); }); } catch (IOException e) { throw new HoodieIOException("Failed to list base files in partitions " + partitionSet, e); } long endTs = System.currentTimeMillis(); - LOG.debug("Time to load partition {} ={}", partitionSet, (endTs - beginTs)); + log.debug("Time to load partition {} ={}", partitionSet, (endTs - beginTs)); } partitionSet.forEach(partition -> @@ -451,7 +452,7 @@ private List getAllFilesInPartition(String relativePartitionPat long beginLsTs = System.currentTimeMillis(); List pathInfoList = tableMetadata.getAllFilesInPartition(partitionPath); long endLsTs = System.currentTimeMillis(); - LOG.debug( + log.debug( "#files found in partition ({}}) = {}, Time taken ={}", relativePartitionPath, pathInfoList.size(), (endLsTs - beginLsTs)); return pathInfoList; } @@ -473,9 +474,9 @@ protected void ensurePartitionLoadedCorrectly(String partition) { try { // For metadata table, log at DEBUG. For data table, log at INFO. if (metaClient.isMetadataTable()) { - LOG.debug("Building file system view for partition ({})", partitionPathStr); + log.debug("Building file system view for partition ({})", partitionPathStr); } else { - LOG.info("Building file system view for partition ({})", partitionPathStr); + log.info("Building file system view for partition ({})", partitionPathStr); } List groups = addFilesToView(partitionPathStr, getAllFilesInPartition(partitionPathStr)); if (groups.isEmpty()) { @@ -485,10 +486,10 @@ protected void ensurePartitionLoadedCorrectly(String partition) { throw new HoodieIOException("Failed to list base files in partition " + partitionPathStr, e); } } else { - LOG.debug("View already built for Partition :{}", partitionPathStr); + log.debug("View already built for Partition :{}", partitionPathStr); } long endTs = System.currentTimeMillis(); - LOG.debug("Time to load partition ({}) ={}", partitionPathStr, (endTs - beginTs)); + log.debug("Time to load partition ({}) ={}", partitionPathStr, (endTs - beginTs)); return true; }); } @@ -580,7 +581,7 @@ private boolean isFileSliceAfterPendingCompaction(FileSlice fileSlice) { */ protected Stream filterBaseFileAfterPendingCompaction(FileSlice fileSlice, boolean includeEmptyFileSlice) { if (isFileSliceAfterPendingCompaction(fileSlice)) { - LOG.debug("File Slice ({}) is in pending compaction", fileSlice); + log.debug("File Slice ({}) is in pending compaction", fileSlice); // Base file is filtered out of the file-slice as the corresponding compaction // instant not completed yet. FileSlice transformed = new FileSlice(fileSlice.getPartitionPath(), fileSlice.getBaseInstantTime(), fileSlice.getFileId()); @@ -606,7 +607,7 @@ private Stream filterUncommittedFiles(FileSlice fileSlice, boolean in .collect(Collectors.toList()); if ((fileSlice.getBaseFile().isPresent() && !committedBaseFile.isPresent()) || committedLogFiles.size() != fileSlice.getLogFileCnt()) { - LOG.debug("File Slice ({}) has uncommitted files.", fileSlice); + log.debug("File Slice ({}) has uncommitted files.", fileSlice); // A file is filtered out of the file-slice if the corresponding // instant has not completed yet. FileSlice transformed = new FileSlice(fileSlice.getPartitionPath(), fileSlice.getBaseInstantTime(), fileSlice.getFileId()); @@ -630,7 +631,7 @@ private FileSlice filterUncommittedLogs(FileSlice fileSlice) { .filter(logFile -> completionTimeQueryView.isCompleted(logFile.getDeltaCommitTime())) .collect(Collectors.toList()); if (committedLogFiles.size() != fileSlice.getLogFileCnt()) { - LOG.debug("File Slice ({}) has uncommitted log files.", fileSlice); + log.debug("File Slice ({}) has uncommitted log files.", fileSlice); // A file is filtered out of the file-slice if the corresponding // instant has not completed yet. FileSlice transformed = new FileSlice(fileSlice.getPartitionPath(), fileSlice.getBaseInstantTime(), fileSlice.getFileId()); @@ -1206,7 +1207,7 @@ public final Stream getAllFileGroupsStateless(String partitionS private Map getBootstrapBaseFileMappings(String partition) { try (BootstrapIndex.IndexReader reader = bootstrapIndex.createReader()) { - LOG.info("Bootstrap Index available for partition {}", partition); + log.info("Bootstrap Index available for partition {}", partition); List sourceFileMappings = reader.getSourceFileMappingForPartition(partition); return sourceFileMappings.stream() @@ -1742,13 +1743,4 @@ public void sync() { writeLock.unlock(); } } - - /** - * Return Only Commits and Compaction timeline for building file-groups. - * - * @return {@code HoodieTimeline} - */ - public HoodieTimeline getVisibleCommitsAndCompactionTimeline() { - return visibleCommitsAndCompactionTimeline; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java index d1baad8495165..c77e3b99152a7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java @@ -32,8 +32,7 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StorageConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.concurrent.ConcurrentHashMap; @@ -58,8 +57,8 @@ * view. FileSystemViewManager uses a factory to construct specific implementation of file-system view and passes it to * clients for querying. */ +@Slf4j public class FileSystemViewManager { - private static final Logger LOG = LoggerFactory.getLogger(FileSystemViewManager.class); private static final String HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS = "org.apache.hudi.common.table.view.HoodieMetaserverFileSystemView"; @@ -142,7 +141,7 @@ public void close() { private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(HoodieEngineContext engineContext, FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient, boolean metadataTableEnabled, SerializableFunctionUnchecked metadataCreator) { - LOG.info("Creating RocksDB based view for basePath {}.", metaClient.getBasePath()); + log.info("Creating RocksDB based view for basePath {}.", metaClient.getBasePath()); HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator); return new RocksDbBasedFileSystemView(tableMetadata, metaClient, timeline, viewConf); @@ -159,7 +158,7 @@ private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystem HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig, boolean metadataTableEnabled, SerializableFunctionUnchecked metadataCreator) { - LOG.info("Creating SpillableMap based view for basePath {}.", metaClient.getBasePath()); + log.info("Creating SpillableMap based view for basePath {}.", metaClient.getBasePath()); HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator); return new SpillableMapBasedFileSystemView(tableMetadata, metaClient, timeline, viewConf, commonConfig); @@ -171,7 +170,7 @@ private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystem private static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieEngineContext engineContext, FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient, boolean metadataTableEnabled, SerializableFunctionUnchecked metadataCreator) { - LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath()); + log.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath()); HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator); if (metaClient.getMetaserverConfig().isMetaserverEnabled()) { @@ -204,7 +203,7 @@ public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline HoodieTableMetaClient metaClient, HoodieMetadataConfig metadataConfig, HoodieTimeline timeline) { - LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath()); + log.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath()); HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataConfig.isEnabled(), unused -> metaClient.getTableFormat().getMetadataFactory().create(engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString())); @@ -225,7 +224,7 @@ public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline */ private static RemoteHoodieTableFileSystemView createRemoteFileSystemView(FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) { - LOG.info("Creating remote view for basePath {}. Server={}:{}, Timeout={}", metaClient.getBasePath(), + log.info("Creating remote view for basePath {}. Server={}:{}, Timeout={}", metaClient.getBasePath(), viewConf.getRemoteViewServerHost(), viewConf.getRemoteViewServerPort(), viewConf.getRemoteTimelineClientTimeoutSecs()); return new RemoteHoodieTableFileSystemView(metaClient, viewConf); } @@ -254,27 +253,27 @@ public static FileSystemViewManager createViewManager(final HoodieEngineContext final FileSystemViewStorageConfig config, final HoodieCommonConfig commonConfig, final SerializableFunctionUnchecked metadataCreator) { - LOG.info("Creating View Manager with storage type {}.", config.getStorageType()); + log.info("Creating View Manager with storage type {}.", config.getStorageType()); boolean metadataTableEnabled = metadataConfig.isEnabled(); switch (config.getStorageType()) { case EMBEDDED_KV_STORE: - LOG.debug("Creating embedded rocks-db based Table View"); + log.debug("Creating embedded rocks-db based Table View"); return new FileSystemViewManager(context, config, (metaClient, viewConf) -> createRocksDBBasedFileSystemView(context, viewConf, metaClient, metadataTableEnabled, metadataCreator)); case SPILLABLE_DISK: - LOG.debug("Creating Spillable Disk based Table View"); + log.debug("Creating Spillable Disk based Table View"); return new FileSystemViewManager(context, config, (metaClient, viewConf) -> createSpillableMapBasedFileSystemView(context, viewConf, metaClient, commonConfig, metadataTableEnabled, metadataCreator)); case MEMORY: - LOG.debug("Creating in-memory based Table View"); + log.debug("Creating in-memory based Table View"); return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> createInMemoryFileSystemView(context, viewConfig, metaClient, metadataTableEnabled, metadataCreator)); case REMOTE_ONLY: - LOG.debug("Creating remote only table view"); + log.debug("Creating remote only table view"); return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> createRemoteFileSystemView(viewConfig, metaClient)); case REMOTE_FIRST: - LOG.debug("Creating remote first table view"); + log.debug("Creating remote first table view"); return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> { RemoteHoodieTableFileSystemView remoteFileSystemView = createRemoteFileSystemView(viewConfig, metaClient); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java index 10d8879f8233c..cb1aa5723a0af 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java @@ -33,8 +33,8 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayList; @@ -52,10 +52,9 @@ * @see TableFileSystemView * @since 0.3.0 */ +@Slf4j public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystemView { - private static final Logger LOG = LoggerFactory.getLogger(HoodieTableFileSystemView.class); - //TODO: [HUDI-6249] change the maps below to implement ConcurrentMap // mapping from partition paths to file groups contained within them @@ -89,6 +88,7 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem /** * Flag to determine if closed. */ + @Getter private boolean closed = false; HoodieTableFileSystemView(HoodieTableMetadata tableMetadata, boolean enableIncrementalTimelineSync) { @@ -402,7 +402,7 @@ protected boolean isPartitionAvailableInStore(String partitionPath) { @Override protected void storePartitionView(String partitionPath, List fileGroups) { - LOG.debug("Adding file-groups for partition :{}, #FileGroups={}", partitionPath, fileGroups.size()); + log.debug("Adding file-groups for partition :{}, #FileGroups={}", partitionPath, fileGroups.size()); List newList = new ArrayList<>(fileGroups); partitionToFileGroupsMap.put(partitionPath, newList); } @@ -453,9 +453,4 @@ protected void closeResources() throws Exception { public void close() { super.close(); } - - @Override - public boolean isClosed() { - return closed; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java index c56ccbe97eb6c..997fbdea224bb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java @@ -47,8 +47,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; @@ -59,10 +58,9 @@ /** * Adds the capability to incrementally sync the changes to file-system view as and when new instants gets completed. */ +@Slf4j public abstract class IncrementalTimelineSyncFileSystemView extends AbstractTableFileSystemView { - private static final Logger LOG = LoggerFactory.getLogger(IncrementalTimelineSyncFileSystemView.class); - // Allows incremental Timeline syncing private final boolean incrementalTimelineSyncEnabled; @@ -98,19 +96,19 @@ protected void maySyncIncrementally() { if (incrementalTimelineSyncEnabled) { TimelineDiffResult diffResult = TimelineDiffHelper.getNewInstantsForIncrementalSync(metaClient, oldTimeline, newTimeline); if (diffResult.canSyncIncrementally()) { - LOG.info("Doing incremental sync"); + log.info("Doing incremental sync"); // need to refresh the completion time query view // before amending existing file groups. refreshCompletionTimeQueryView(); runIncrementalSync(newTimeline, diffResult); - LOG.info("Finished incremental sync"); + log.info("Finished incremental sync"); // Reset timeline to latest refreshTimeline(newTimeline); return; } } } catch (Exception ioe) { - LOG.error("Got exception trying to perform incremental sync. Reverting to complete sync", ioe); + log.error("Got exception trying to perform incremental sync. Reverting to complete sync", ioe); } clear(); // Initialize with new Hoodie timeline. @@ -125,7 +123,7 @@ protected void maySyncIncrementally() { */ private void runIncrementalSync(HoodieTimeline timeline, TimelineDiffResult diffResult) { - LOG.info("Timeline Diff Result is :{}", diffResult); + log.info("Timeline Diff Result is :{}", diffResult); // First remove pending compaction instants which were completed diffResult.getFinishedCompactionInstants().stream().forEach(instant -> { @@ -180,7 +178,7 @@ private void runIncrementalSync(HoodieTimeline timeline, TimelineDiffResult diff * @param instant Compaction Instant to be removed */ private void removePendingCompactionInstant(HoodieInstant instant) throws IOException { - LOG.info("Removing completed compaction instant ({})", instant); + log.info("Removing completed compaction instant ({})", instant); HoodieCompactionPlan plan = CompactionUtils.getCompactionPlan(metaClient, instant.requestedTime()); removePendingCompactionOperations(CompactionUtils.getPendingCompactionOperations(instant, plan) .map(instantPair -> Pair.of(instantPair.getValue().getKey(), @@ -194,7 +192,7 @@ private void removePendingCompactionInstant(HoodieInstant instant) throws IOExce * @param instant Log Compaction Instant to be removed */ private void removePendingLogCompactionInstant(HoodieInstant instant) throws IOException { - LOG.info("Removing completed log compaction instant ({})", instant); + log.info("Removing completed log compaction instant ({})", instant); HoodieCompactionPlan plan = CompactionUtils.getLogCompactionPlan(metaClient, instant.requestedTime()); removePendingLogCompactionOperations(CompactionUtils.getPendingCompactionOperations(instant, plan) .map(instantPair -> Pair.of(instantPair.getValue().getKey(), @@ -208,7 +206,7 @@ private void removePendingLogCompactionInstant(HoodieInstant instant) throws IOE * @param instant Compaction Instant */ private void addPendingCompactionInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing pending compaction instant ({})", instant); + log.info("Syncing pending compaction instant ({})", instant); HoodieCompactionPlan compactionPlan = CompactionUtils.getCompactionPlan(metaClient, instant.requestedTime()); List> pendingOps = CompactionUtils.getPendingCompactionOperations(instant, compactionPlan) @@ -238,7 +236,7 @@ private void addPendingCompactionInstant(HoodieTimeline timeline, HoodieInstant * @param instant Compaction Instant */ private void addPendingLogCompactionInstant(HoodieInstant instant) throws IOException { - LOG.info("Syncing pending log compaction instant ({})", instant); + log.info("Syncing pending log compaction instant ({})", instant); HoodieCompactionPlan compactionPlan = CompactionUtils.getLogCompactionPlan(metaClient, instant.requestedTime()); List> pendingOps = CompactionUtils.getPendingCompactionOperations(instant, compactionPlan) @@ -257,10 +255,10 @@ private void addPendingLogCompactionInstant(HoodieInstant instant) throws IOExce * @param instant Instant */ private void addCommitInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing committed instant ({})", instant); + log.info("Syncing committed instant ({})", instant); HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(instant); updatePartitionWriteFileGroups(commitMetadata.getPartitionToWriteStats(), timeline, instant); - LOG.info("Done Syncing committed instant ({})", instant); + log.info("Done Syncing committed instant ({})", instant); } private void updatePartitionWriteFileGroups(Map> partitionToWriteStats, @@ -269,7 +267,7 @@ private void updatePartitionWriteFileGroups(Map> p partitionToWriteStats.entrySet().stream().forEach(entry -> { String partition = entry.getKey(); if (isPartitionAvailableInStore(partition)) { - LOG.info("Syncing partition ({}) of instant ({})", partition, instant); + log.info("Syncing partition ({}) of instant ({})", partition, instant); List pathInfoList = entry.getValue().stream() .map(p -> new StoragePathInfo( new StoragePath(String.format("%s/%s", metaClient.getBasePath(), p.getPath())), @@ -279,10 +277,10 @@ private void updatePartitionWriteFileGroups(Map> p buildFileGroups(partition, pathInfoList, timeline.filterCompletedAndCompactionInstants(), false); applyDeltaFileSlicesToPartitionView(partition, fileGroups, DeltaApplyMode.ADD); } else { - LOG.warn("Skipping partition ({}) when syncing instant ({}) as it is not loaded", partition, instant); + log.warn("Skipping partition ({}) when syncing instant ({}) as it is not loaded", partition, instant); } }); - LOG.info("Done Syncing committed instant ({})", instant); + log.info("Done Syncing committed instant ({})", instant); } /** @@ -292,7 +290,7 @@ private void updatePartitionWriteFileGroups(Map> p * @param instant Restore Instant */ private void addRestoreInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing restore instant ({})", instant); + log.info("Syncing restore instant ({})", instant); HoodieRestoreMetadata metadata = timeline.readRestoreMetadata(instant); Map>> partitionFiles = @@ -312,7 +310,7 @@ private void addRestoreInstant(HoodieTimeline timeline, HoodieInstant instant) t .map(HoodieInstantInfo::getCommitTime).collect(Collectors.toSet()); removeReplacedFileIdsAtInstants(rolledbackInstants); } - LOG.info("Done Syncing restore instant ({})", instant); + log.info("Done Syncing restore instant ({})", instant); } /** @@ -322,13 +320,13 @@ private void addRestoreInstant(HoodieTimeline timeline, HoodieInstant instant) t * @param instant Rollback Instant */ private void addRollbackInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing rollback instant ({})", instant); + log.info("Syncing rollback instant ({})", instant); HoodieRollbackMetadata metadata = timeline.readRollbackMetadata(instant); metadata.getPartitionMetadata().entrySet().stream().forEach(e -> { removeFileSlicesForPartition(timeline, instant, e.getKey(), e.getValue().getSuccessDeleteFiles()); }); - LOG.info("Done Syncing rollback instant ({})", instant); + log.info("Done Syncing rollback instant ({})", instant); } /** @@ -338,7 +336,7 @@ private void addRollbackInstant(HoodieTimeline timeline, HoodieInstant instant) * @param instant REPLACE Instant */ private void addReplaceInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing replace instant ({})", instant); + log.info("Syncing replace instant ({})", instant); HoodieReplaceCommitMetadata replaceMetadata = timeline.readReplaceCommitMetadata(instant); updatePartitionWriteFileGroups(replaceMetadata.getPartitionToWriteStats(), timeline, instant); replaceMetadata.getPartitionToReplaceFileIds().entrySet().stream().forEach(entry -> { @@ -346,10 +344,10 @@ private void addReplaceInstant(HoodieTimeline timeline, HoodieInstant instant) t Map replacedFileIds = entry.getValue().stream() .collect(Collectors.toMap(replaceStat -> new HoodieFileGroupId(partition, replaceStat), replaceStat -> instant)); - LOG.info("For partition ({}) of instant ({}), excluding {} file groups", partition, instant, replacedFileIds.size()); + log.info("For partition ({}) of instant ({}), excluding {} file groups", partition, instant, replacedFileIds.size()); addReplacedFileGroups(replacedFileIds); }); - LOG.info("Done Syncing REPLACE instant ({})", instant); + log.info("Done Syncing REPLACE instant ({})", instant); } /** @@ -360,7 +358,7 @@ private void addReplaceInstant(HoodieTimeline timeline, HoodieInstant instant) t * @param instant Clean instant */ private void addCleanInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException { - LOG.info("Syncing cleaner instant ({})", instant); + log.info("Syncing cleaner instant ({})", instant); HoodieCleanMetadata cleanMetadata = CleanerUtils.getCleanerMetadata(metaClient, instant); cleanMetadata.getPartitionMetadata().entrySet().stream().forEach(entry -> { final StoragePath basePath = metaClient.getBasePath(); @@ -371,13 +369,13 @@ private void addCleanInstant(HoodieTimeline timeline, HoodieInstant instant) thr .collect(Collectors.toList()); removeFileSlicesForPartition(timeline, instant, entry.getKey(), fullPathList); }); - LOG.info("Done Syncing cleaner instant ({})", instant); + log.info("Done Syncing cleaner instant ({})", instant); } private void removeFileSlicesForPartition(HoodieTimeline timeline, HoodieInstant instant, String partition, List paths) { if (isPartitionAvailableInStore(partition)) { - LOG.info("Removing file slices for partition ({}) for instant ({})", partition, instant); + log.info("Removing file slices for partition ({}) for instant ({})", partition, instant); List pathInfoList = paths.stream() .map(p -> new StoragePathInfo(new StoragePath(p), 0, false, (short) 0, 0, 0)) .collect(Collectors.toList()); @@ -385,7 +383,7 @@ private void removeFileSlicesForPartition(HoodieTimeline timeline, HoodieInstant buildFileGroups(partition, pathInfoList, timeline.filterCompletedAndCompactionInstants(), false); applyDeltaFileSlicesToPartitionView(partition, fileGroups, DeltaApplyMode.REMOVE); } else { - LOG.warn("Skipping partition ({}) when syncing instant ({}) as it is not loaded", partition, instant); + log.warn("Skipping partition ({}) when syncing instant ({}) as it is not loaded", partition, instant); } } @@ -408,7 +406,7 @@ enum DeltaApplyMode { protected void applyDeltaFileSlicesToPartitionView(String partition, List deltaFileGroups, DeltaApplyMode mode) { if (deltaFileGroups.isEmpty()) { - LOG.info("No delta file groups for partition :{}", partition); + log.info("No delta file groups for partition :{}", partition); return; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java index e6d60c13b548c..c54c0edcafc32 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java @@ -34,10 +34,11 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpStatus; import org.apache.http.client.HttpResponseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.List; @@ -48,11 +49,11 @@ * A file system view which proxies request to a preferred File System View implementation. In case of error, flip all * subsequent calls to a backup file-system view implementation. */ +@Slf4j public class PriorityBasedFileSystemView implements SyncableFileSystemView, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(PriorityBasedFileSystemView.class); - private final transient HoodieEngineContext engineContext; + @Getter(AccessLevel.PACKAGE) private final SyncableFileSystemView preferredView; private final SerializableFunctionUnchecked secondaryViewCreator; private SyncableFileSystemView secondaryView; @@ -69,7 +70,7 @@ public PriorityBasedFileSystemView(SyncableFileSystemView preferredView, Seriali private R execute(Function0 preferredFunction, Function0 secondaryFunction) { if (errorOnPreferredView) { - LOG.warn("Routing request to secondary file-system view"); + log.warn("Routing request to secondary file-system view"); return secondaryFunction.apply(); } else { try { @@ -84,7 +85,7 @@ private R execute(Function0 preferredFunction, Function0 secondaryFunc private R execute(T1 val, Function1 preferredFunction, Function1 secondaryFunction) { if (errorOnPreferredView) { - LOG.warn("Routing request to secondary file-system view"); + log.warn("Routing request to secondary file-system view"); return secondaryFunction.apply(val); } else { try { @@ -100,7 +101,7 @@ private R execute(T1 val, Function1 preferredFunction, Function1< private R execute(T1 val, T2 val2, Function2 preferredFunction, Function2 secondaryFunction) { if (errorOnPreferredView) { - LOG.warn("Routing request to secondary file-system view"); + log.warn("Routing request to secondary file-system view"); return secondaryFunction.apply(val, val2); } else { try { @@ -116,7 +117,7 @@ private R execute(T1 val, T2 val2, Function2 preferredFun private R execute(T1 val, T2 val2, T3 val3, Function3 preferredFunction, Function3 secondaryFunction) { if (errorOnPreferredView) { - LOG.warn("Routing request to secondary file-system view"); + log.warn("Routing request to secondary file-system view"); return secondaryFunction.apply(val, val2, val3); } else { try { @@ -131,9 +132,9 @@ private R execute(T1 val, T2 val2, T3 val3, Function3 getLatestFileSlice(String partitionPath, String fileId) return execute(partitionPath, fileId, preferredView::getLatestFileSlice, (path, fgId) -> getSecondaryView().getLatestFileSlice(path, fgId)); } - SyncableFileSystemView getPreferredView() { - return preferredView; - } - synchronized SyncableFileSystemView getSecondaryView() { if (secondaryView == null) { secondaryView = secondaryViewCreator.apply(engineContext); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java index 4cdcb09681227..431292e88aa22 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java @@ -46,8 +46,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.afterburner.AfterburnerModule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.Serializable; @@ -62,6 +61,7 @@ /** * A proxy for table file-system view which translates local View API calls to REST calls to remote timeline service. */ +@Slf4j public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, Serializable { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new AfterburnerModule()); @@ -127,8 +127,6 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, public static final String INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM = "includependingcompaction"; public static final String MULTI_VALUE_SEPARATOR = ","; - - private static final Logger LOG = LoggerFactory.getLogger(RemoteHoodieTableFileSystemView.class); private static final TypeReference> FILE_SLICE_DTOS_REFERENCE = new TypeReference>() {}; private static final TypeReference> FILE_GROUP_DTOS_REFERENCE = new TypeReference>() {}; private static final TypeReference BOOLEAN_TYPE_REFERENCE = new TypeReference() {}; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index 2620a29784fcd..fee37f33380bc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -38,8 +38,9 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.util.HashMap; @@ -65,16 +66,16 @@ * support view-state preservation across restarts, Hoodie timeline also needs to be stored inorder to detect changes to * timeline across restarts. */ +@Slf4j public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSystemView { - private static final Logger LOG = LoggerFactory.getLogger(RocksDbBasedFileSystemView.class); - private final FileSystemViewStorageConfig config; private final RocksDBSchemaHelper schemaHelper; private RocksDBDAO rocksDB; + @Getter(AccessLevel.PACKAGE) private boolean closed = false; public RocksDbBasedFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline, @@ -96,7 +97,7 @@ public RocksDbBasedFileSystemView(HoodieTableMetadata tableMetadata, HoodieTable protected void init(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) { schemaHelper.getAllColumnFamilies().forEach(rocksDB::addColumnFamily); super.init(metaClient, visibleActiveTimeline); - LOG.info("Created ROCKSDB based file-system view at {}", config.getRocksdbBasePath()); + log.info("Created ROCKSDB based file-system view at {}", config.getRocksdbBasePath()); } @Override @@ -111,7 +112,7 @@ protected void resetPendingCompactionOperations(Stream> fetchFileGroupsInPendingCl @Override void resetFileGroupsInPendingClustering(Map fgIdToInstantMap) { - LOG.info("Resetting file groups in pending clustering to ROCKSDB based file-system view at " + log.info("Resetting file groups in pending clustering to ROCKSDB based file-system view at " + config.getRocksdbBasePath() + ", Total file-groups=" + fgIdToInstantMap.size()); // Delete all replaced file groups rocksDB.prefixDelete(schemaHelper.getColFamilyForFileGroupsInPendingClustering(), "part="); // Now add new entries addFileGroupsInPendingClustering(fgIdToInstantMap.entrySet().stream().map(entry -> Pair.of(entry.getKey(), entry.getValue()))); - LOG.info("Resetting replacedFileGroups to ROCKSDB based file-system view complete"); + log.info("Resetting replacedFileGroups to ROCKSDB based file-system view complete"); } @Override @@ -246,7 +247,7 @@ void removeFileGroupsInPendingClustering(Stream fileGroups) { - LOG.info("Resetting and adding new partition ({}) to ROCKSDB based file-system view at {}, Total file-groups={}", + log.info("Resetting and adding new partition ({}) to ROCKSDB based file-system view at {}, Total file-groups={}", partitionPath, config.getRocksdbBasePath(), fileGroups.size()); String lookupKey = schemaHelper.getKeyForPartitionLookup(partitionPath); @@ -303,7 +304,7 @@ protected void storePartitionView(String partitionPath, List fi // record that partition is loaded. rocksDB.put(schemaHelper.getColFamilyForStoredPartitions(), lookupKey, Boolean.TRUE); - LOG.info("Finished adding new partition ({}}) to ROCKSDB based file-system view at {}, Total file-groups={}", + log.info("Finished adding new partition ({}}) to ROCKSDB based file-system view at {}, Total file-groups={}", partitionPath, config.getRocksdbBasePath(), fileGroups.size()); } @@ -322,7 +323,7 @@ protected void applyDeltaFileSlicesToPartitionView(String partition, List !logFiles.containsKey(e.getKey())) .forEach(p -> newLogFiles.put(p.getKey(), p.getValue())); newLogFiles.values().forEach(newFileSlice::addLogFile); - LOG.info("Adding back new File Slice after add FS={}", newFileSlice); + log.info("Adding back new File Slice after add FS={}", newFileSlice); return newFileSlice; } case REMOVE: { - LOG.info("Removing old File Slice ={}", fs); + log.info("Removing old File Slice ={}", fs); FileSlice newFileSlice = new FileSlice(oldSlice.getFileGroupId(), oldSlice.getBaseInstantTime()); fs.getBaseFile().orElseGet(() -> { oldSlice.getBaseFile().ifPresent(newFileSlice::setBaseFile); @@ -357,7 +358,7 @@ protected void applyDeltaFileSlicesToPartitionView(String partition, List 0)) { - LOG.info("Adding back new file-slice after remove FS={}", newFileSlice); + log.info("Adding back new file-slice after remove FS={}", newFileSlice); return newFileSlice; } return null; @@ -406,7 +407,7 @@ void resetBootstrapBaseFileMapping(Stream bootstrapBas rocksDB.putInBatch(batch, schemaHelper.getColFamilyForBootstrapBaseFile(), schemaHelper.getKeyForBootstrapBaseFile(externalBaseFile.getFileGroupId()), externalBaseFile); }); - LOG.info("Initializing external data file mapping. Count={}", batch.count()); + log.info("Initializing external data file mapping. Count={}", batch.count()); }); } @@ -516,14 +517,14 @@ Option fetchHoodieFileGroup(String partitionPath, String fileId @Override protected void resetReplacedFileGroups(final Map replacedFileGroups) { - LOG.info("Resetting replacedFileGroups to ROCKSDB based file-system view at " + log.info("Resetting replacedFileGroups to ROCKSDB based file-system view at " + config.getRocksdbBasePath() + ", Total file-groups=" + replacedFileGroups.size()); // Delete all replaced file groups rocksDB.prefixDelete(schemaHelper.getColFamilyForReplacedFileGroups(), "part="); // Now add new entries addReplacedFileGroups(replacedFileGroups); - LOG.info("Resetting replacedFileGroups to ROCKSDB based file-system view complete"); + log.info("Resetting replacedFileGroups to ROCKSDB based file-system view complete"); } @Override @@ -542,7 +543,7 @@ protected void addReplacedFileGroups(final Map }) ); - LOG.info("Finished adding replaced file groups to partition (" + partitionPath + ") to ROCKSDB based view at " + log.info("Finished adding replaced file groups to partition (" + partitionPath + ") to ROCKSDB based view at " + config.getRocksdbBasePath() + ", Total file-groups=" + partitionToReplacedFileGroupsEntry.getValue().size()); }); } @@ -599,20 +600,15 @@ private static boolean isFileSliceWithoutCompactionBarrier(FileSlice fileSlice) public void close() { try { writeLock.lock(); - LOG.info("Closing Rocksdb !!"); + log.info("Closing Rocksdb !!"); closed = true; closeResources(); rocksDB.close(); - LOG.info("Closed Rocksdb !!"); + log.info("Closed Rocksdb !!"); } catch (Exception e) { throw new HoodieException("Unable to close file system view", e); } finally { writeLock.unlock(); } } - - @Override - boolean isClosed() { - return closed; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java index f531907653b13..37bf6b4b05a20 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java @@ -35,8 +35,7 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.IOException; @@ -48,10 +47,9 @@ /** * Table FileSystemView implementation where view is stored in spillable disk using fixed memory. */ +@Slf4j public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView { - private static final Logger LOG = LoggerFactory.getLogger(SpillableMapBasedFileSystemView.class); - private final long maxMemoryForFileGroupMap; private final long maxMemoryForPendingCompaction; private final long maxMemoryForPendingLogCompaction; @@ -75,7 +73,7 @@ public SpillableMapBasedFileSystemView(HoodieTableMetadata tableMetadata, Hoodie new File(baseStoreDir).mkdirs(); diskMapType = commonConfig.getSpillableDiskMapType(); isBitCaskDiskMapCompressionEnabled = commonConfig.isBitCaskDiskMapCompressionEnabled(); - LOG.info("Initializing SpillableMapBasedFileSystemView with memory configs: " + log.info("Initializing SpillableMapBasedFileSystemView with memory configs: " + "maxMemoryForFileGroupMap={}, maxMemoryForPendingCompaction={}, maxMemoryForPendingLogCompaction={}, " + "maxMemoryForBootstrapBaseFile={}, maxMemoryForReplaceFileGroups={}, maxMemoryForClusteringFileGroups={}, baseStoreDir={}", maxMemoryForFileGroupMap, maxMemoryForPendingCompaction, maxMemoryForPendingLogCompaction, From c2681b229b32863d0e067a4b55421b25c7b995f8 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 16 Jun 2026 15:20:11 +0700 Subject: [PATCH 079/255] [MINOR] Wait for ZK connection in lock provider to de-flake direct-marker detection test (#19014) (cherry picked from commit f3ccf1b3c31b52e7ece2d596a7749369e2f7e0b0) --- .../lock/BaseZookeeperBasedLockProvider.java | 27 +++++++++++++++-- .../TestZookeeperBasedLockProvider.java | 21 ++++++++++++++ ...edDetectionStrategyWithZKLockProvider.java | 29 ++++++++++++++++--- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/BaseZookeeperBasedLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/BaseZookeeperBasedLockProvider.java index d5b04c15c005e..6cdee60e2d1b1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/BaseZookeeperBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/BaseZookeeperBasedLockProvider.java @@ -68,6 +68,7 @@ public BaseZookeeperBasedLockProvider(final LockConfiguration lockConfiguration, this.lockConfiguration = lockConfiguration; zkBasePath = getZkBasePath(lockConfiguration); lockKey = getLockKey(lockConfiguration); + int connectionTimeoutMs = ConfigUtils.getIntWithAltKeys(lockConfiguration.getConfig(), ZK_CONNECTION_TIMEOUT_MS); this.curatorFrameworkClient = CuratorFrameworkFactory.builder() .connectString(ConfigUtils.getStringWithAltKeys(lockConfiguration.getConfig(), ZK_CONNECT_URL)) .retryPolicy(new BoundedExponentialBackoffRetry( @@ -75,10 +76,32 @@ public BaseZookeeperBasedLockProvider(final LockConfiguration lockConfiguration, ConfigUtils.getIntWithAltKeys(lockConfiguration.getConfig(), LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS), ConfigUtils.getIntWithAltKeys(lockConfiguration.getConfig(), LOCK_ACQUIRE_NUM_RETRIES))) .sessionTimeoutMs(ConfigUtils.getIntWithAltKeys(lockConfiguration.getConfig(), ZK_SESSION_TIMEOUT_MS)) - .connectionTimeoutMs(ConfigUtils.getIntWithAltKeys(lockConfiguration.getConfig(), ZK_CONNECTION_TIMEOUT_MS)) + .connectionTimeoutMs(connectionTimeoutMs) .build(); this.curatorFrameworkClient.start(); - createPathIfNotExists(); + // Once started, the Curator client owns background threads. If anything below throws, the + // constructor never returns the instance, so the caller can never invoke close() - clean up here. + try { + if (!this.curatorFrameworkClient.blockUntilConnected(connectionTimeoutMs, TimeUnit.MILLISECONDS)) { + throw new HoodieLockException("Failed to connect to ZooKeeper within " + connectionTimeoutMs + " ms"); + } + createPathIfNotExists(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + closeQuietly(); + throw new HoodieLockException("Interrupted while waiting to connect to ZooKeeper", e); + } catch (RuntimeException e) { + closeQuietly(); + throw e; + } + } + + private void closeQuietly() { + try { + this.curatorFrameworkClient.close(); + } catch (Exception ex) { + log.warn("Failed to close ZooKeeper client after failed initialization", ex); + } } protected abstract String getZkBasePath(LockConfiguration lockConfiguration); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestZookeeperBasedLockProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestZookeeperBasedLockProvider.java index fab3dee8f8f6e..fa1a8329b65fc 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestZookeeperBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestZookeeperBasedLockProvider.java @@ -42,6 +42,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.time.Duration; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -194,4 +195,24 @@ public void testUnlockWithoutLock() { ZookeeperBasedLockProvider zookeeperBasedLockProvider = new ZookeeperBasedLockProvider(zkConfWithZkBasePathAndLockKeyLock, null); zookeeperBasedLockProvider.unlock(); } + + @Test + public void testFailFastWhenZkUnreachable() { + Properties properties = new Properties(); + // Nothing listens on 127.0.0.1:1, so the connect-wait must time out instead of hanging. + properties.setProperty(ZK_CONNECT_URL_PROP_KEY, "127.0.0.1:1"); + properties.setProperty(ZK_BASE_PATH_PROP_KEY, basePath); + properties.setProperty(ZK_LOCK_KEY_PROP_KEY, key); + properties.setProperty(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "100"); + properties.setProperty(LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY, "300"); + properties.setProperty(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY, "1"); + properties.setProperty(ZK_SESSION_TIMEOUT_MS_PROP_KEY, "1000"); + properties.setProperty(ZK_CONNECTION_TIMEOUT_MS_PROP_KEY, "1000"); + LockConfiguration unreachable = new LockConfiguration(properties); + // Construction must fail fast (seconds, bounded by the connection timeout) with a + // HoodieLockException instead of being amplified into a multi-minute retry hang. + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(15), () -> + Assertions.assertThrows(HoodieLockException.class, + () -> new ZookeeperBasedLockProvider(unreachable, null))); + } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java index deb551e9d2458..a2a5a92f69eb4 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java @@ -51,9 +51,16 @@ import java.util.List; import java.util.Properties; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY; import static org.apache.hudi.common.config.LockConfiguration.ZK_BASE_PATH_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.ZK_CONNECTION_TIMEOUT_MS_PROP_KEY; import static org.apache.hudi.common.config.LockConfiguration.ZK_CONNECT_URL_PROP_KEY; import static org.apache.hudi.common.config.LockConfiguration.ZK_LOCK_KEY_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.ZK_SESSION_TIMEOUT_MS_PROP_KEY; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -82,6 +89,15 @@ private void setUp(boolean partitioned) throws Exception { properties.setProperty(ZK_CONNECT_URL_PROP_KEY, server.getConnectString()); properties.setProperty(ZK_BASE_PATH_PROP_KEY, server.getTempDirectory().getAbsolutePath()); properties.setProperty(ZK_LOCK_KEY_PROP_KEY, "key"); + // Bound lock retries and ZK timeouts so a transient connection failure fails fast in seconds + // instead of being amplified by the production-default retry layers into a multi-minute hang. + properties.setProperty(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "1000"); + properties.setProperty(LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000"); + properties.setProperty(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "3"); + properties.setProperty(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY, "3"); + properties.setProperty(ZK_SESSION_TIMEOUT_MS_PROP_KEY, "10000"); + properties.setProperty(ZK_CONNECTION_TIMEOUT_MS_PROP_KEY, "10000"); + properties.setProperty(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "1000"); config = getConfigBuilder() .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder() @@ -104,10 +120,15 @@ private void setUp(boolean partitioned) throws Exception { @AfterEach public void clean() throws IOException { - cleanupResources(); - FileIOUtils.deleteDirectory(new File(basePath)); - if (server != null) { - server.close(); + try { + cleanupResources(); + FileIOUtils.deleteDirectory(new File(basePath)); + } finally { + // Always stop the embedded ZooKeeper server, even if resource cleanup or directory + // deletion above throws, so the server is not leaked across parameterized runs. + if (server != null) { + server.close(); + } } } From 969a7e8ef050167385de52d00007ccb202b322a1 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Tue, 16 Jun 2026 17:01:20 +0800 Subject: [PATCH 080/255] fix(flink): fix the mor small file record size estimation (#18991) (cherry picked from commit e853440bac04f1ca9869a749d7f69d409d647ab3) --- .../profile/DeltaWriteProfile.java | 36 +++++- .../partitioner/profile/WriteProfile.java | 60 +++++----- .../sink/partitioner/TestBucketAssigner.java | 105 +++++++++++++++++- .../table/ITTestDynamicBucketStreamWrite.java | 7 +- 4 files changed, 179 insertions(+), 29 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java index f73adb37d3379..a79c78e6d3639 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java @@ -22,12 +22,15 @@ import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.SyncableFileSystemView; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.action.commit.SmallFile; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -38,7 +41,9 @@ * *

    Note: assumes the index can always index log files for Flink write. */ +@Slf4j public class DeltaWriteProfile extends WriteProfile { + public DeltaWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) { super(config, context); } @@ -86,12 +91,34 @@ protected List smallFilesProfile(String partitionPath) { return smallFileLocations; } + @Override + protected long averageBytesPerRecord() { + long avgSize = config.getCopyOnWriteRecordSizeEstimate(); + HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants(); + if (!commitTimeline.empty()) { + long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D); + if (sizeFromCommitMetadata > 0) { + avgSize = sizeFromCommitMetadata; + } + } else { + HoodieTimeline deltaCommitTimeline = metaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants(); + if (!deltaCommitTimeline.empty()) { + long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(deltaCommitTimeline, logFileToParquetCompressionRatio()); + if (sizeFromCommitMetadata > 0) { + avgSize = sizeFromCommitMetadata; + } + } + } + log.info("Refresh average bytes per record => " + avgSize); + return avgSize; + } + protected SyncableFileSystemView getFileSystemView() { return (SyncableFileSystemView) getTable().getSliceView(); } private long getTotalFileSize(FileSlice fileSlice) { - return fileSlice.getTotalFileSizeAsParquetFormat(config.getLogFileToParquetCompressionRatio()); + return fileSlice.getTotalFileSizeAsParquetFormat(logFileToParquetCompressionRatio()); } private boolean isSmallFile(FileSlice fileSlice) { @@ -99,4 +126,11 @@ private boolean isSmallFile(FileSlice fileSlice) { return totalSize < config.getParquetMaxFileSize(); } + private double logFileToParquetCompressionRatio() { + if (config.getLogDataBlockFormat().isPresent() + && config.getLogDataBlockFormat().get() == HoodieLogBlock.HoodieLogBlockType.PARQUET_DATA_BLOCK) { + return 1D; + } + return config.getLogFileToParquetCompressionRatio(); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java index cbcc71cedb6ec..c3819bf081166 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java @@ -117,7 +117,6 @@ public WriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) this.context = context; this.basePath = new Path(config.getBasePath()); this.smallFilesMap = new HashMap<>(); - this.recordsPerBucket = config.getCopyOnWriteInsertSplitSize(); this.metaClient = StreamerUtil.createMetaClient( config.getBasePath(), context.getStorageConf().unwrapAs(Configuration.class)); this.metadataCache = new HashMap<>(); @@ -134,35 +133,46 @@ public WriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) * Obtains the average record size based on records written during previous commits. Used for estimating how many * records pack into one file. */ - private long averageBytesPerRecord() { + protected long averageBytesPerRecord() { long avgSize = config.getCopyOnWriteRecordSizeEstimate(); - long fileSizeThreshold = (long) (config.getRecordSizeEstimationThreshold() * config.getParquetSmallFileLimit()); - HoodieTimeline commitTimeline = metaClient.getCommitsTimeline().filterCompletedInstants(); + HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants(); if (!commitTimeline.empty()) { - // Go over the reverse ordered commits to get a more recent estimate of average record size. - Iterator instants = commitTimeline.getReverseOrderedInstants().iterator(); - while (instants.hasNext()) { - HoodieInstant instant = instants.next(); - final HoodieCommitMetadata commitMetadata = - this.metadataCache.computeIfAbsent( - instant.requestedTime(), - k -> WriteProfiles.getCommitMetadataSafely(config.getTableName(), basePath, instant, commitTimeline) - .orElse(null)); - if (commitMetadata == null) { - continue; - } - long totalBytesWritten = commitMetadata.fetchTotalBytesWritten(); - long totalRecordsWritten = commitMetadata.fetchTotalRecordsWritten(); - if (totalBytesWritten > fileSizeThreshold && totalRecordsWritten > 0) { - avgSize = (long) Math.ceil((1.0 * totalBytesWritten) / totalRecordsWritten); - break; - } + long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D); + if (sizeFromCommitMetadata > 0) { + avgSize = sizeFromCommitMetadata; } } log.info("Refresh average bytes per record => " + avgSize); return avgSize; } + protected long calculateRecordSizeThroughCommitMetadata(HoodieTimeline commitTimeline, double fileSizeCalibrationRatio) { + long fileSizeThreshold = recordSizeEstimationFileSizeThreshold(); + // Go over the reverse ordered commits to get a more recent estimate of average record size. + Iterator instants = commitTimeline.getReverseOrderedInstants().iterator(); + while (instants.hasNext()) { + HoodieInstant instant = instants.next(); + final HoodieCommitMetadata commitMetadata = + this.metadataCache.computeIfAbsent( + instant.requestedTime(), + k -> WriteProfiles.getCommitMetadataSafely(config.getTableName(), basePath, instant, commitTimeline) + .orElse(null)); + if (commitMetadata == null) { + continue; + } + long totalBytesWritten = commitMetadata.fetchTotalBytesWritten(); + long totalRecordsWritten = commitMetadata.fetchTotalRecordsWritten(); + if (totalBytesWritten > fileSizeThreshold && totalRecordsWritten > 0) { + return (long) Math.ceil((fileSizeCalibrationRatio * totalBytesWritten) / totalRecordsWritten); + } + } + return -1L; + } + + private long recordSizeEstimationFileSizeThreshold() { + return (long) (0.1D * config.getParquetSmallFileLimit()); + } + /** * Returns a list of small files in the given partition path. * @@ -228,10 +238,8 @@ private void cleanMetadataCache(Stream instants) { private void recordProfile() { this.avgSize = averageBytesPerRecord(); - if (config.shouldAllowMultiWriteOnSameInstant()) { - this.recordsPerBucket = config.getParquetMaxFileSize() / avgSize; - log.info("Refresh insert records per bucket => " + recordsPerBucket); - } + this.recordsPerBucket = config.getParquetMaxFileSize() / avgSize; + log.info("Refresh insert records per bucket => " + recordsPerBucket); } /** diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java index 4f682084050bc..8281ebbd86e4f 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java @@ -19,11 +19,16 @@ package org.apache.hudi.sink.partitioner; import org.apache.hudi.client.common.HoodieFlinkEngineContext; +import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.configuration.HadoopConfigurations; import org.apache.hudi.hadoop.fs.HadoopFSUtils; +import org.apache.hudi.sink.partitioner.profile.DeltaWriteProfile; import org.apache.hudi.sink.partitioner.profile.WriteProfile; import org.apache.hudi.table.action.commit.BucketInfo; import org.apache.hudi.table.action.commit.BucketType; @@ -151,7 +156,8 @@ public void testAddInsert() { @Test public void testInsertOverBucketAssigned() { - conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(), "2"); + conf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(512 * 1024)); writeConfig = FlinkWriteClients.getHoodieClientConfig(conf); MockBucketAssigner mockBucketAssigner = new MockBucketAssigner(context, writeConfig); @@ -402,6 +408,103 @@ public void testWriteProfileMetadataCache() throws Exception { writeProfile.getMetadataCache().size(), is(3)); } + @Test + public void testWriteProfileRecordsPerBucketUsesProfiledRecordSize() { + conf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(), "2"); + conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024"); + writeConfig = FlinkWriteClients.getHoodieClientConfig(conf); + + WriteProfile writeProfile = new WriteProfile(writeConfig, context); + + assertThat("Average record size should use the configured estimate for an empty table", + writeProfile.getAvgSize(), is(1024L)); + assertThat("Records per bucket should be derived from the max parquet file size", + writeProfile.getRecordsPerBucket(), is(1024L)); + } + + @Test + public void testWriteProfileRecordsPerBucketUsesProfiledRecordSizeWithSmallEstimationThreshold() throws Exception { + conf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(1024 * 1024)); + conf.setString(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT.key(), "1"); + TestData.writeData(TestData.DATA_SET_INSERT, conf); + + writeConfig = FlinkWriteClients.getHoodieClientConfig(conf); + WriteProfile writeProfile = new WriteProfile(writeConfig, context); + String latestInstant = getLastCompleteInstant(writeProfile); + HoodieCommitMetadata commitMetadata = writeProfile.getMetadataCache().get(latestInstant); + assertNotNull(commitMetadata); + long expectedAvgSize = (long) Math.ceil( + 1.0 * commitMetadata.fetchTotalBytesWritten() / commitMetadata.fetchTotalRecordsWritten()); + + assertThat("Average record size should use commit metadata when it is large enough relative to small file limit", + writeProfile.getAvgSize(), is(expectedAvgSize)); + assertThat("Records per bucket should use the profiled record size", + writeProfile.getRecordsPerBucket(), is(writeConfig.getParquetMaxFileSize() / expectedAvgSize)); + } + + @Test + public void testDeltaWriteProfileRecordsPerBucketUsesCompressionRatio() throws Exception { + File morPath = new File(tempFile, "mor"); + Configuration morConf = TestConfigurations.getDefaultConf(morPath.getAbsolutePath()); + morConf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); + morConf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + morConf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024"); + morConf.setString(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT.key(), "1"); + morConf.setString(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(), "0.5"); + StreamerUtil.initTableIfNotExists(morConf); + TestData.writeData(TestData.DATA_SET_INSERT, morConf); + + HoodieWriteConfig morWriteConfig = FlinkWriteClients.getHoodieClientConfig(morConf); + HoodieFlinkEngineContext morContext = new HoodieFlinkEngineContext( + HadoopFSUtils.getStorageConf(HadoopConfigurations.getHadoopConf(morConf)), + new FlinkTaskContextSupplier(null)); + + DeltaWriteProfile writeProfile = new DeltaWriteProfile(morWriteConfig, morContext); + String latestInstant = getLastCompleteInstant(writeProfile); + HoodieCommitMetadata commitMetadata = writeProfile.getMetadataCache().get(latestInstant); + assertNotNull(commitMetadata); + long expectedAvgSize = (long) Math.ceil( + 0.5 * commitMetadata.fetchTotalBytesWritten() / commitMetadata.fetchTotalRecordsWritten()); + + assertThat("Average record size from commit metadata should be corrected for MOR log-to-parquet compression", + writeProfile.getAvgSize(), is(expectedAvgSize)); + assertThat("Records per bucket should use the corrected MOR average record size", + writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / expectedAvgSize)); + } + + @Test + public void testDeltaWriteProfileRecordsPerBucketSkipsCompressionRatioForParquetLogBlocks() throws Exception { + File morPath = new File(tempFile, "mor_parquet_logs"); + Configuration morConf = TestConfigurations.getDefaultConf(morPath.getAbsolutePath()); + morConf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); + morConf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + morConf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024"); + morConf.setString(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT.key(), "1"); + morConf.setString(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(), "0.5"); + morConf.setString(HoodieStorageConfig.LOGFILE_DATA_BLOCK_FORMAT.key(), "parquet"); + StreamerUtil.initTableIfNotExists(morConf); + TestData.writeData(TestData.DATA_SET_INSERT, morConf); + + HoodieWriteConfig morWriteConfig = FlinkWriteClients.getHoodieClientConfig(morConf); + HoodieFlinkEngineContext morContext = new HoodieFlinkEngineContext( + HadoopFSUtils.getStorageConf(HadoopConfigurations.getHadoopConf(morConf)), + new FlinkTaskContextSupplier(null)); + + DeltaWriteProfile writeProfile = new DeltaWriteProfile(morWriteConfig, morContext); + String latestInstant = getLastCompleteInstant(writeProfile); + HoodieCommitMetadata commitMetadata = writeProfile.getMetadataCache().get(latestInstant); + assertNotNull(commitMetadata); + long expectedAvgSize = (long) Math.ceil( + 1.0 * commitMetadata.fetchTotalBytesWritten() / commitMetadata.fetchTotalRecordsWritten()); + + assertThat("Average record size from parquet log blocks should not be corrected again", + writeProfile.getAvgSize(), is(expectedAvgSize)); + assertThat("Records per bucket should use the uncorrected parquet log block average record size", + writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / expectedAvgSize)); + } + private static String getLastCompleteInstant(WriteProfile profile) { return StreamerUtil.getLastCompletedInstant(profile.getMetaClient()); } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestDynamicBucketStreamWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestDynamicBucketStreamWrite.java index 83b5b7f73cdfe..6fb0dceb6fa93 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestDynamicBucketStreamWrite.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestDynamicBucketStreamWrite.java @@ -166,8 +166,13 @@ void testInsertOverwrite(HoodieTableType tableType) { @ParameterizedTest @EnumSource(value = HoodieTableType.class) void testBucketScalesUpWithContinuousWrites(HoodieTableType tableType) { + Map smallBucketOptions = Map.of( + HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(), "1", + FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE.key(), "1", + HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT.key(), "1", + HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(1024 * 1024)); streamTableEnv.executeSql(getTableDDL( - "t1", tableType, Collections.singletonMap(HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(), "1"), true)); + "t1", tableType, smallBucketOptions, true)); execInsertSql(streamTableEnv, "insert into t1 values\n" + "('id1','Danny',23,TIMESTAMP '1970-01-01 00:00:01','par_scale'),\n" From a6bf0f6088080a914538d806b8afb2beba75a8e6 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 16 Jun 2026 17:12:36 +0700 Subject: [PATCH 081/255] perf(kafka-connect): reuse AvroConvertor across records in the connect writer (#19015) (cherry picked from commit b5879f95ad1447c41bd75ee284ba509268eaf1f1) --- .../hudi/connect/writers/AbstractConnectWriter.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java index 70fefe4316032..149ace0403cd5 100644 --- a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java +++ b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java @@ -53,6 +53,10 @@ public abstract class AbstractConnectWriter implements ConnectWriter avroRecord; - switch (connectConfigs.getKafkaValueConverter()) { + switch (kafkaValueConverter) { case KAFKA_AVRO_CONVERTER: avroRecord = Option.of((GenericRecord) record.value()); break; case KAFKA_STRING_CONVERTER: + if (convertor == null) { + convertor = new AvroConvertor(schemaProvider.getSourceHoodieSchema()); + } avroRecord = Option.of(convertor.fromJson((String) record.value())); break; case KAFKA_JSON_CONVERTER: throw new UnsupportedEncodingException("Currently JSON objects are not supported"); default: - throw new IOException("Unsupported Kafka Format type (" + connectConfigs.getKafkaValueConverter() + ")"); + throw new IOException("Unsupported Kafka Format type (" + kafkaValueConverter + ")"); } // Tag records with a file ID based on kafka partition and hudi partition. From d53340c37759d8096a9fad33cdefe88873d569b2 Mon Sep 17 00:00:00 2001 From: Aditya Goenka <63430370+ad1happy2go@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:59:00 +0530 Subject: [PATCH 082/255] fix(spark): Propagate merge configs to file group reader during clustering (#19007) (cherry picked from commit a4f0e261b9501aed3221dde3ef7b09323ea22918) --- .../strategy/ClusteringExecutionStrategy.java | 6 +- .../TestClusteringWithCustomMerger.scala | 133 ++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringWithCustomMerger.scala diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java index cc3c4a02eb227..9610f728f8c77 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java @@ -107,7 +107,11 @@ protected ClosableIterator> getRecordIterator(ReaderContextFacto protected TypedProperties getReaderProperties(long maxMemory) { HoodieWriteConfig config = getWriteConfig(); - TypedProperties props = new TypedProperties(); + // Seed from the full write config so that merge-related properties (e.g. the custom record + // merger impl classes, merge mode and strategy id) are propagated to the file group reader. + // Without this, reading source file groups with a CUSTOM merge mode fails to resolve the + // configured merger (HUDI-18980). This mirrors the compaction read path. + TypedProperties props = TypedProperties.copy(config.getProps()); props.setProperty(SPILLABLE_MAP_BASE_PATH.key(), config.getSpillableMapBasePath()); props.setProperty(SPILLABLE_DISK_MAP_TYPE.key(), config.getCommonConfig().getSpillableDiskMapType().toString()); props.setProperty(DISK_MAP_BITCASK_COMPRESSION_ENABLED.key(), Boolean.toString(config.getCommonConfig().isBitCaskDiskMapCompressionEnabled())); diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringWithCustomMerger.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringWithCustomMerger.scala new file mode 100644 index 0000000000000..7319bba9768cf --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringWithCustomMerger.scala @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +import org.apache.hudi.{DefaultSparkRecordMerger, HoodieDataSourceHelpers} +import org.apache.hudi.common.config.HoodieReaderConfig +import org.apache.hudi.common.model.HoodieRecordMerger +import org.apache.hudi.common.table.timeline.HoodieTimeline + +import org.apache.hadoop.fs.Path + +import scala.collection.JavaConverters._ + +/** + * Regression test for HUDI issue #18980: + * clustering on a table configured with {@code hoodie.write.record.merge.mode=CUSTOM} and a custom + * Spark merger failed with + * "No valid spark merger implementation set for `hoodie.write.record.merge.custom.implementation.classes`". + * + *

    The merge mode and strategy id are persisted as table config, but the custom merger impl classes + * are a write-side config that is not persisted. Before the fix, + * {@code ClusteringExecutionStrategy.getReaderProperties} built a fresh property set containing only + * the spill/memory keys, dropping the impl classes, so the file group reader could not resolve the + * configured merger. The fix seeds the reader properties from the full write config. + * + *

    Both clustering execution paths call {@code getReaderProperties} and reproduce the bug: + * the row-writer path ({@code MultipleSparkJobExecutionStrategy#readRecordsForGroupAsRow}, used when + * {@code hoodie.datasource.write.row.writer.enable} is true — the default the reporter hit) and the + * RDD path ({@code MultipleSparkJobExecutionStrategy#readRecordsForGroup} when it is false). The test + * is parameterized over both. + * + *

    This lives in {@code hudi-spark} (not {@code hudi-spark-client}): a CUSTOM/SPARK-typed merger + * forces the InternalRow write path, which needs a concrete {@code SparkXXXAdapter} and real Spark + * records — neither is available to {@code hudi-spark-client}'s test classpath / Avro test-data path. + */ +class TestClusteringWithCustomMerger extends HoodieSparkProcedureTestBase { + + test("Test clustering with CUSTOM record merge mode and a custom Spark merger") { + // hoodie.datasource.write.row.writer.enable: true exercises the row-writer clustering path + // (the default the reporter hit), false exercises the RDD path. Both call getReaderProperties. + Seq("true", "false").foreach { rowWriterEnabled => + withTempDir { tmp => + val tableName = generateTableName + val basePath = s"${tmp.getCanonicalPath}/$tableName" + val mergerClass = classOf[CustomSparkRecordMergerForClustering].getName + + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long, + | part long + |) using hudi + | tblproperties ( + | primaryKey = 'id', + | type = 'mor', + | orderingFields = 'ts', + | "hoodie.write.record.merge.mode" = "CUSTOM", + | "hoodie.write.record.merge.strategy.id" = "${HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID}", + | "${HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY}" = "$mergerClass", + | "hoodie.parquet.small.file.limit" = "0" + | ) + | partitioned by (part) + | location '$basePath' + """.stripMargin) + + withSQLConf( + "hoodie.compact.inline" -> "false", + "hoodie.compact.schedule.inline" -> "false") { + // Several commits into the same partition create multiple file groups for clustering to + // combine (small.file.limit=0 keeps every insert in a new base file). + spark.sql(s"insert into $tableName values (1, 'a1', 10.0, 1000, 100)") + spark.sql(s"insert into $tableName values (2, 'a2', 20.0, 1001, 100)") + spark.sql(s"insert into $tableName values (3, 'a3', 30.0, 1002, 100)") + // An update lands in a log file, so clustering must merge base + log records through the + // file group reader using the CUSTOM Spark merger. + spark.sql(s"update $tableName set price = 99.0, ts = 1003 where id = 1") + + // Pin the row-writer setting and propagate the custom merger impl classes (write-side, + // non-persisted) to the clustering write client. Before HUDI-18980 the reader properties + // dropped these and clustering failed with "No valid spark merger implementation set". + val clusteringOptions = + s"hoodie.datasource.write.row.writer.enable=$rowWriterEnabled," + + s"${HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY}=$mergerClass" + spark.sql(s"call run_clustering(table => '$tableName', options => '$clusteringOptions')").show() + + // Clustering must have produced exactly one completed replace commit. + val fs = new Path(basePath).getFileSystem(spark.sessionState.newHadoopConf()) + val replaceCommits = HoodieDataSourceHelpers.allCompletedCommitsCompactions(fs, basePath) + .getInstants.iterator().asScala + .filter(_.getAction == HoodieTimeline.REPLACE_COMMIT_ACTION) + .toSeq + assertResult(1)(replaceCommits.size) + + // All records remain readable after clustering, with the update applied. + checkAnswer(s"select id, name, price, ts, part from $tableName order by id")( + Seq(1, "a1", 99.0, 1003, 100), + Seq(2, "a2", 20.0, 1001, 100), + Seq(3, "a3", 30.0, 1002, 100) + ) + } + } + } + } +} + +/** + * A custom Spark record merger whose only purpose is to advertise the CUSTOM merge strategy id, so the + * table is configured with {@code RecordMergeMode.CUSTOM} and a custom merger impl class. It otherwise + * reuses the default Spark merge behavior. + */ +class CustomSparkRecordMergerForClustering extends DefaultSparkRecordMerger { + override def getMergingStrategy: String = HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID +} From 4f4229f7de859ef8ebd66b220767598190167f34 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 16 Jun 2026 20:34:19 +0700 Subject: [PATCH 083/255] perf(kafka-connect): memoize file id per partition path in the connect writer (#19016) (cherry picked from commit 8389464e161367c917612859a330164e2f7fd406) --- .../hudi/connect/writers/AbstractConnectWriter.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java index 149ace0403cd5..628a51d16852c 100644 --- a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java +++ b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/AbstractConnectWriter.java @@ -35,7 +35,9 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Base Hudi Writer that manages reading the raw Kafka records and @@ -57,6 +59,9 @@ public abstract class AbstractConnectWriter implements ConnectWriter fileIdByPartitionPath = new HashMap<>(); public AbstractConnectWriter(KafkaConnectConfigs connectConfigs, KeyGenerator keyGenerator, @@ -89,7 +94,12 @@ public void writeRecord(SinkRecord record) throws IOException { // Tag records with a file ID based on kafka partition and hudi partition. HoodieRecord hoodieRecord = new HoodieAvroRecord<>(keyGenerator.getKey(avroRecord.get()), new HoodieAvroPayload(avroRecord)); - String fileId = KafkaConnectUtils.hashDigest(String.format("%s-%s", record.kafkaPartition(), hoodieRecord.getPartitionPath())); + String partitionPath = hoodieRecord.getPartitionPath(); + String fileId = fileIdByPartitionPath.get(partitionPath); + if (fileId == null) { + fileId = KafkaConnectUtils.hashDigest(record.kafkaPartition() + "-" + partitionPath); + fileIdByPartitionPath.put(partitionPath, fileId); + } hoodieRecord.unseal(); hoodieRecord.setCurrentLocation(new HoodieRecordLocation(instantTime, fileId)); hoodieRecord.setNewLocation(new HoodieRecordLocation(instantTime, fileId)); From 98ad512fe59ee616d92afe8e6c20f331c8c84e96 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 16 Jun 2026 20:36:27 +0700 Subject: [PATCH 084/255] perf(kafka-connect): use a pre-sized ArrayList when flushing buffered records (#19017) (cherry picked from commit 8c2eacffb160c8e45d9940c738f41ea7bd94c831) --- .../apache/hudi/connect/writers/BufferedConnectWriter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java index 2893839acd75e..09b61ae608fae 100644 --- a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java +++ b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; /** @@ -109,11 +108,11 @@ public List flushRecords() { if (!bufferedRecords.isEmpty()) { if (isMorTable) { writeStatuses = writeClient.upsertPreppedRecords( - new LinkedList<>(bufferedRecords.values()), + new ArrayList<>(bufferedRecords.values()), instantTime); } else { writeStatuses = writeClient.bulkInsertPreppedRecords( - new LinkedList<>(bufferedRecords.values()), + new ArrayList<>(bufferedRecords.values()), instantTime, Option.empty()); } } From 71cef99d6e29639ec98c3a447ad384992c1d5759 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 17 Jun 2026 00:56:55 +0800 Subject: [PATCH 085/255] chore(test): document macOS-specific consistency-guard slowness in cleaner test (#17714) (#19003) (cherry picked from commit abb9b349dbe7b4034fd2e032f696a4fe3dff00f4) --- .../action/clean/TestCleanerInsertAndCleanByVersions.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java index 583135830d5ba..aa26921c174f8 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java @@ -133,6 +133,10 @@ private void testInsertAndCleanByVersions( .withBulkInsertParallelism(PARALLELISM) .withFinalizeWriteParallelism(PARALLELISM) .withDeleteParallelism(PARALLELISM) + // #17714: enabling the consistency check makes this test slow on macOS local file://. The cleaner's + // getFileStatus calls the guard's waitTillFileAppears, which misses the translated path and burns the + // full ~25.2s backoff per deleted file. Does NOT reproduce on CI (Linux), where the path resolves and + // it stays fast, so this gotcha is only visible locally. .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build()) .build(); try (final SparkRDDWriteClient client = getHoodieWriteClient(cfg)) { From 5d81ce44b2697ace910947ddfe8dc06ebfb36084 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 17 Jun 2026 02:37:44 +0800 Subject: [PATCH 086/255] perf(metadata): Resolve column-stats field schemas once per collection instead of per record (#19000) * perf(metadata): Resolve column-stats field schemas once per collection instead of per record collectColumnRangeMetadata iterated the target columns inside the per-record loop and, for every record and every target field, recomputed values that depend only on the fixed target field list: - field.schema().getNonNullType() rebuilds the union-member wrappers for nullable fields (a fresh HoodieSchema per call), and - because that non-null schema was a fresh instance per record, its toAvroSchema() (used for min/max compares) never memoized. Resolve the non-null HoodieSchema once per target field before the loop and iterate that precomputed list; holding a stable instance also lets toAvroSchema() memoize across records. Per-record work (type-support check, value extraction, min/max/null/value counts) is unchanged, so results are identical. Covered by the existing column-stats functional tests. Closes #18999 * review(metadata): stream per-field schema resolution and drop toAvroSchema memoize note (cherry picked from commit fb08a156b6f62324864759669d1c88c0e3faf282) --- .../hudi/metadata/HoodieTableMetadataUtil.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index 4823f041bae6d..cfdeb45b84ed6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -276,15 +276,19 @@ public static Map> collectColumnRa final Properties properties = new Properties(); properties.setProperty(HoodieStorageConfig.WRITE_UTC_TIMEZONE.key(), storageConfig.getString(HoodieStorageConfig.WRITE_UTC_TIMEZONE.key(), HoodieStorageConfig.WRITE_UTC_TIMEZONE.defaultValue().toString())); + // getNonNullType() rebuilds the union-member wrappers for nullable fields and depends only on the + // (fixed) target fields, so resolve it once per field instead of once per record per field. + List> nonNullFieldSchemas = targetFields.stream() + .map(p -> Pair.of(p.getKey(), p.getValue().schema().getNonNullType())) + .collect(Collectors.toList()); // Collect stats for all columns by iterating through records while accounting // corresponding stats records.forEachRemaining((record) -> { // For each column (field) we have to index update corresponding column stats // with the values from this record - targetFields.forEach(fieldNameFieldPair -> { - String fieldName = fieldNameFieldPair.getKey(); - HoodieSchemaField field = fieldNameFieldPair.getValue(); - HoodieSchema fieldSchema = field.schema().getNonNullType(); + nonNullFieldSchemas.forEach(fieldNameSchemaPair -> { + String fieldName = fieldNameSchemaPair.getKey(); + HoodieSchema fieldSchema = fieldNameSchemaPair.getValue(); if (!isColumnTypeSupported(fieldSchema, Option.of(record.getRecordType()), indexVersion)) { return; } From 8e08db36331702a56ecbcf98998499ba8ada90ce Mon Sep 17 00:00:00 2001 From: oglego <110243312+oglego@users.noreply.github.com> Date: Tue, 16 Jun 2026 21:20:44 -0500 Subject: [PATCH 087/255] test(hadoop-mr): enable rollback case in HoodieRealtimeRecordReader.testReader (#18693) (cherry picked from commit 973aeba167780db953badffb85888473bd5bf4f9) --- .../realtime/TestHoodieRealtimeRecordReader.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java index 3616b9909b773..0383f9f7e668c 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java @@ -19,6 +19,7 @@ package org.apache.hudi.hadoop.realtime; import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieRollbackPlan; import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.HoodieMemoryConfig; import org.apache.hudi.common.config.HoodieReaderConfig; @@ -213,8 +214,8 @@ private void testReaderInternal(ExternalSpillableMap.DiskMapType diskMapType, List> logVersionsWithAction = new ArrayList<>(); logVersionsWithAction.add(Pair.of(HoodieTimeline.DELTA_COMMIT_ACTION, 1)); logVersionsWithAction.add(Pair.of(HoodieTimeline.DELTA_COMMIT_ACTION, 2)); - // TODO: HUDI-154 Once Hive 2.x PR (PR-674) is merged, enable this change - // logVersionsWithAction.add(Pair.of(HoodieTimeline.ROLLBACK_ACTION, 3)); + logVersionsWithAction.add(Pair.of(HoodieTimeline.ROLLBACK_ACTION, 3)); + FileSlice fileSlice = new FileSlice(partitioned ? HadoopFSUtils.getRelativePartitionPath(new Path(basePath.toString()), new Path(partitionDir.getAbsolutePath())) : "default", baseInstant, "fileid0"); @@ -231,7 +232,7 @@ private void testReaderInternal(ExternalSpillableMap.DiskMapType diskMapType, HoodieLogFormat.Writer writer; if (action.equals(HoodieTimeline.ROLLBACK_ACTION)) { - writer = InputFormatTestUtil.writeRollback(partitionDir, storage, "fileid0", baseInstant, + writer = InputFormatTestUtil.writeRollback(partitionDir, storage, "fileid0", instantTime, instantTime, String.valueOf(baseInstantTs + logVersion - 1), logVersion); } else { @@ -243,7 +244,11 @@ private void testReaderInternal(ExternalSpillableMap.DiskMapType diskMapType, long size = writer.getCurrentSize(); writer.close(); assertTrue(size > 0, "block - size should be > 0"); - FileCreateUtilsLegacy.createDeltaCommit(COMMIT_METADATA_SER_DE, basePath.toString(), instantTime, commitMetadata); + if (action.equals(HoodieTimeline.ROLLBACK_ACTION)) { + FileCreateUtilsLegacy.createRequestedRollbackFile(basePath.toString(), instantTime, new HoodieRollbackPlan()); + } else { + FileCreateUtilsLegacy.createDeltaCommit(COMMIT_METADATA_SER_DE, basePath.toString(), instantTime, commitMetadata); + } // create a split with baseFile (parquet file written earlier) and new log file(s) fileSlice.addLogFile(writer.getLogFile()); From 9b701ca1e27e649ada053352713c5eff6934446f Mon Sep 17 00:00:00 2001 From: Krishen <22875197+kbuci@users.noreply.github.com> Date: Tue, 16 Jun 2026 19:29:07 -0700 Subject: [PATCH 088/255] feat(flink): Support writing out-of-line BLOB columns (#18958) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(flink): Support writing out-of-line BLOB columns --------- Co-authored-by: Krishen Bhan <“bkrishen@uber.com”> Co-authored-by: Cursor (cherry picked from commit 3ada03d744fd3f0f8d2c5b88c2589ebcdac0b3ce) --- .../hudi/util/HoodieSchemaConverter.java | 44 ++-- .../hudi/util/RowDataToAvroConverters.java | 8 + .../hudi/util/TestHoodieSchemaConverter.java | 17 ++ .../apache/hudi/table/ITTestBlobWrite.java | 226 ++++++++++++++++++ .../utils/TestRowDataToAvroConverters.java | 120 ++++++++++ 5 files changed, 394 insertions(+), 21 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestBlobWrite.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java index c413101dde461..c859675488ad7 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/HoodieSchemaConverter.java @@ -272,6 +272,16 @@ private static boolean isFamily(LogicalType logicalType, LogicalTypeFamily famil /** * Detects if a Flink RowType represents a BLOB structure by validating it matches the schema defined in {@link HoodieSchema.Blob}. + * + *

    Detection intentionally keys off the stable structural signals (field names and base type + * roots) and does not assert nested-field nullability. Flink SQL {@code CREATE TABLE} does + * not reliably preserve {@code NOT NULL} constraints on nested {@code ROW} fields, so requiring an + * exact nullability match would silently demote a user's BLOB column to a generic record when the + * column is declared through DDL. The canonical nullability is restored by {@link HoodieSchema#createBlob()}. + * + *

    TODO: This heuristic is a workaround for the lack of a native Flink/Parquet BLOB logical + * type. See apache/hudi#18711 for + * the tracked work to remove this structural inference. */ private static boolean isBlobStructure(RowType rowType) { // Validate: 3 fields with exact names @@ -286,24 +296,20 @@ private static boolean isBlobStructure(RowType rowType) { return false; } - // Validate 'type' field: non-null STRING - LogicalType typeField = rowType.getTypeAt(0); - if (!isFamily(typeField, LogicalTypeFamily.CHARACTER_STRING) || typeField.isNullable()) { + // Validate 'type' field: STRING + if (!isFamily(rowType.getTypeAt(0), LogicalTypeFamily.CHARACTER_STRING)) { return false; } - // Validate 'data' field: nullable BYTES/VARBINARY + // Validate 'data' field: BYTES/VARBINARY LogicalType dataField = rowType.getTypeAt(1); if (dataField.getTypeRoot() != LogicalTypeRoot.BINARY && dataField.getTypeRoot() != LogicalTypeRoot.VARBINARY) { return false; } - if (!dataField.isNullable()) { - return false; - } - // Validate 'reference' field: nullable ROW + // Validate 'reference' field: ROW LogicalType referenceField = rowType.getTypeAt(2); - if (!referenceField.isNullable() || referenceField.getTypeRoot() != LogicalTypeRoot.ROW) { + if (referenceField.getTypeRoot() != LogicalTypeRoot.ROW) { return false; } @@ -322,24 +328,20 @@ private static boolean isBlobStructure(RowType rowType) { } // Validate reference sub-field types - // external_path: non-null STRING - if (!isFamily(referenceRow.getTypeAt(0), LogicalTypeFamily.CHARACTER_STRING) - || referenceRow.getTypeAt(0).isNullable()) { + // external_path: STRING + if (!isFamily(referenceRow.getTypeAt(0), LogicalTypeFamily.CHARACTER_STRING)) { return false; } - // offset: nullable BIGINT - if (referenceRow.getTypeAt(1).getTypeRoot() != LogicalTypeRoot.BIGINT - || !referenceRow.getTypeAt(1).isNullable()) { + // offset: BIGINT + if (referenceRow.getTypeAt(1).getTypeRoot() != LogicalTypeRoot.BIGINT) { return false; } - // length: nullable BIGINT - if (referenceRow.getTypeAt(2).getTypeRoot() != LogicalTypeRoot.BIGINT - || !referenceRow.getTypeAt(2).isNullable()) { + // length: BIGINT + if (referenceRow.getTypeAt(2).getTypeRoot() != LogicalTypeRoot.BIGINT) { return false; } - // managed: non-null BOOLEAN - if (referenceRow.getTypeAt(3).getTypeRoot() != LogicalTypeRoot.BOOLEAN - || referenceRow.getTypeAt(3).isNullable()) { + // managed: BOOLEAN + if (referenceRow.getTypeAt(3).getTypeRoot() != LogicalTypeRoot.BOOLEAN) { return false; } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java index 6bf94b5270747..050ad483de104 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java @@ -148,6 +148,14 @@ public Object convert(HoodieSchema schema, Object object) { @Override public Object convert(HoodieSchema schema, Object object) { + // The BLOB `type` discriminator is a STRING in Flink but an ENUM in Avro. + // Detect that at call time from the HoodieSchema so the converter stays + // reusable across any row shape — not hard-wired by Flink row structure alone. + HoodieSchema nonNullSchema = schema.getNonNullType(); + if (nonNullSchema.getType() == HoodieSchemaType.ENUM) { + return new GenericData.EnumSymbol( + nonNullSchema.toAvroSchema(), object.toString()); + } return new Utf8(((BinaryStringData) object).toBytes()); } }; diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java index 9f6ad8f53bf73..b14aad4877de0 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestHoodieSchemaConverter.java @@ -771,6 +771,23 @@ public void testBlobStructureValidation() { HoodieSchema convertedSchema = HoodieSchemaConverter.convertToSchema(blobLikeRowType); assertEquals(HoodieSchemaType.BLOB, convertedSchema.getType()); + // Positive case: same structure but every nested field nullable. Flink SQL CREATE TABLE does not + // preserve NOT NULL on nested ROW fields, so detection must not depend on nested nullability. + DataType allNullableBlobRow = DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.TYPE, DataTypes.STRING().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.INLINE_DATA_FIELD, DataTypes.BYTES().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE, DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_PATH, DataTypes.STRING().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_OFFSET, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_LENGTH, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_IS_MANAGED, DataTypes.BOOLEAN().nullable()) + ).nullable()) + ).notNull(); + + RowType allNullableBlobRowType = (RowType) allNullableBlobRow.getLogicalType(); + HoodieSchema allNullableConverted = HoodieSchemaConverter.convertToSchema(allNullableBlobRowType); + assertEquals(HoodieSchemaType.BLOB, allNullableConverted.getType()); + // Negative case 1: Different field names DataType differentNames = DataTypes.ROW( DataTypes.FIELD("wrong_name", DataTypes.STRING().notNull()), diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestBlobWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestBlobWrite.java new file mode 100644 index 0000000000000..a66c9f0347658 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestBlobWrite.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table; + +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; +import org.apache.hudi.util.HoodieSchemaConverter; +import org.apache.hudi.util.StreamerUtil; +import org.apache.hudi.utils.FlinkMiniCluster; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.Row; +import org.apache.flink.util.CollectionUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.File; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * IT case for writing out-of-line (OOL) BLOB columns through the Flink writer and reading them back. + * + *

    Verifies that the Hudi {@link HoodieSchema.Blob} structure round-trips through the Flink + * write/read pipeline (both COW and MOR), that updates land through the MOR Avro log path, and that + * the stored table schema keeps the BLOB logical type instead of degrading the column to a generic + * Flink/Avro record. + */ +@ExtendWith(FlinkMiniCluster.class) +public class ITTestBlobWrite { + + @TempDir + File tempFile; + + private static final String BLOB_COLUMN = "blob_col"; + + private static final String BLOB_COLUMN_DDL = + " " + BLOB_COLUMN + " ROW<\n" + + " `type` STRING NOT NULL,\n" + + " `data` BYTES,\n" + + " `reference` ROW<\n" + + " external_path STRING NOT NULL,\n" + + " `offset` BIGINT,\n" + + " `length` BIGINT,\n" + + " managed BOOLEAN NOT NULL\n" + + " >\n" + + " >,\n"; + + /** + * Regression for {@code HoodieSchemaConverter#isBlobStructure}: Flink SQL {@code CREATE TABLE} + * may declare nested {@code ROW} fields as {@code NOT NULL}, but + * {@link ResolvedSchema#toPhysicalRowDataType()} does not always preserve those constraints in + * the {@link RowType}. Schema inference must still recognize the BLOB shape (same path as + * {@code HoodieTableFactory#inferAvroSchema}), otherwise {@code blob_col} would degrade to a + * generic RECORD in the committed Hoodie schema. + */ + @Test + public void testFlinkSqlDdlPhysicalRowTypeStillMapsToHoodieBlob() { + TableEnvironment tableEnv = batchEnv(); + final String probeTable = "flink_blob_ddl_probe"; + String createProbeDdl = + "CREATE TABLE " + + probeTable + + " (\n" + + " id BIGINT,\n" + + " name STRING,\n" + + BLOB_COLUMN_DDL + + " ts BIGINT\n" + + ") WITH ('connector'='blackhole')"; + tableEnv.executeSql(createProbeDdl); + + ResolvedSchema resolved = tableEnv.from(probeTable).getResolvedSchema(); + RowType physical = (RowType) resolved.toPhysicalRowDataType().getLogicalType(); + // DDL uses NOT NULL on nested BLOB ROW fields where the canonical Hoodie BLOB shape is + // stricter. Flink's physical RowType from ResolvedSchema#toPhysicalRowDataType() may or may + // not preserve those flags across versions (see Flink table config / release notes linked in + // the PR). We do not assert which fields widen — only that Hudi still recognizes the column as + // a BLOB (regression for HoodieSchemaConverter#isBlobStructure). + + HoodieSchema recordSchema = + HoodieSchemaConverter.convertToSchema( + physical, HoodieSchemaUtils.getRecordQualifiedName(probeTable)); + HoodieSchemaField blobField = + recordSchema + .getField(BLOB_COLUMN) + .orElseThrow(() -> new AssertionError("blob_col missing from converted HoodieSchema")); + assertTrue( + blobField.schema().isBlobField(), + "Physical RowType from Flink SQL DDL must still map to Hoodie BLOB, got: " + + blobField.schema()); + + tableEnv.executeSql("DROP TABLE " + probeTable); + } + + private void createTable(TableEnvironment tableEnv, String tablePath, HoodieTableType tableType) { + String createTableDdl = String.format( + "CREATE TABLE blob_table (\n" + + " id BIGINT,\n" + + " name STRING,\n" + + BLOB_COLUMN_DDL + + " ts BIGINT,\n" + + " PRIMARY KEY (id) NOT ENFORCED\n" + + ") WITH (\n" + + " 'connector' = 'hudi',\n" + + " 'path' = '%s',\n" + + " 'table.type' = '%s',\n" + + " 'ordering.fields' = 'ts'\n" + + ");", + tablePath, tableType.name()); + tableEnv.executeSql(createTableDdl); + } + + @ParameterizedTest + @EnumSource(value = HoodieTableType.class) + public void testWriteAndReadOutOfLineBlob(HoodieTableType tableType) throws Exception { + TableEnvironment tableEnv = batchEnv(); + String tablePath = new File(tempFile, "blob_table").getAbsolutePath(); + createTable(tableEnv, tablePath, tableType); + + // First batch: insert two OOL blob references. + execInsert(tableEnv, + "INSERT INTO blob_table VALUES\n" + + "(1, 'doc-1', ROW('OUT_OF_LINE', CAST(NULL AS BYTES), " + + "ROW('file1.bin', CAST(0 AS BIGINT), CAST(100 AS BIGINT), false)), 1000),\n" + + "(2, 'doc-2', ROW('OUT_OF_LINE', CAST(NULL AS BYTES), " + + "ROW('file1.bin', CAST(100 AS BIGINT), CAST(200 AS BIGINT), false)), 2000)"); + + List rows = readOrdered(tableEnv); + assertEquals(2, rows.size()); + assertOutOfLineRow(rows.get(0), 1L, "doc-1", "file1.bin", 0L, 100L, 1000L); + assertOutOfLineRow(rows.get(1), 2L, "doc-2", "file1.bin", 100L, 200L, 2000L); + + // Second batch: upsert the same keys with new references. For MOR this exercises the Avro + // log write path (RowData -> Avro), including the BLOB enum `type` field. + execInsert(tableEnv, + "INSERT INTO blob_table VALUES\n" + + "(1, 'doc-1', ROW('OUT_OF_LINE', CAST(NULL AS BYTES), " + + "ROW('file2.bin', CAST(500 AS BIGINT), CAST(300 AS BIGINT), false)), 3000),\n" + + "(2, 'doc-2', ROW('OUT_OF_LINE', CAST(NULL AS BYTES), " + + "ROW('file2.bin', CAST(800 AS BIGINT), CAST(400 AS BIGINT), false)), 4000)"); + + List updated = readOrdered(tableEnv); + assertEquals(2, updated.size()); + assertOutOfLineRow(updated.get(0), 1L, "doc-1", "file2.bin", 500L, 300L, 3000L); + assertOutOfLineRow(updated.get(1), 2L, "doc-2", "file2.bin", 800L, 400L, 4000L); + + // The stored table schema must keep the BLOB logical type, not a generic record. + assertBlobTypePreserved(tablePath); + } + + private static List readOrdered(TableEnvironment tableEnv) { + return CollectionUtil.iteratorToList( + tableEnv.executeSql("select id, name, blob_col, ts from blob_table order by id").collect()); + } + + private static void assertOutOfLineRow(Row row, long id, String name, String path, + long offset, long length, long ts) { + assertEquals(id, row.getField(0)); + assertEquals(name, row.getField(1)); + Row blob = (Row) row.getField(2); + assertNotNull(blob, "blob struct must be populated"); + assertEquals("OUT_OF_LINE", blob.getField(0)); + assertNull(blob.getField(1), "inline data must be null for OUT_OF_LINE blob"); + Row reference = (Row) blob.getField(2); + assertNotNull(reference, "reference must be populated for OUT_OF_LINE blob"); + assertEquals(path, reference.getField(0)); + assertEquals(offset, reference.getField(1)); + assertEquals(length, reference.getField(2)); + assertEquals(false, reference.getField(3)); + assertEquals(ts, row.getField(3)); + } + + private static void assertBlobTypePreserved(String tablePath) throws Exception { + HoodieTableMetaClient metaClient = + StreamerUtil.createMetaClient(tablePath, new org.apache.hadoop.conf.Configuration()); + HoodieSchema tableSchema = new TableSchemaResolver(metaClient).getTableSchema(); + HoodieSchemaField blobField = tableSchema.getField(BLOB_COLUMN) + .orElseThrow(() -> new AssertionError("blob_col field missing from table schema")); + assertTrue(blobField.schema().isBlobField(), + "blob_col must keep the BLOB logical type, found: " + blobField.schema()); + } + + private static TableEnvironment batchEnv() { + TableEnvironment tableEnv = org.apache.hudi.utils.TestTableEnvs.getBatchTableEnv(); + tableEnv.getConfig().getConfiguration() + .set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1); + return tableEnv; + } + + private static void execInsert(TableEnvironment tableEnv, String insertSql) throws Exception { + TableResult result = tableEnv.executeSql(insertSql); + result.getJobClient().get().getJobExecutionResult().get(120, TimeUnit.SECONDS); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java index 185c5830616c3..d7c68b7e318d4 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java @@ -18,15 +18,22 @@ package org.apache.hudi.utils; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.util.HoodieSchemaConverter; import org.apache.hudi.util.RowDataToAvroConverters; +import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; import org.apache.flink.formats.common.TimestampFormat; import org.apache.flink.formats.json.JsonToRowDataConverters; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; import org.junit.jupiter.api.Assertions; @@ -36,6 +43,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import static org.apache.flink.table.api.DataTypes.FIELD; import static org.apache.flink.table.api.DataTypes.ROW; @@ -81,4 +89,116 @@ void testRowDataToAvroStringToRowDataWithUtcTimezone() throws JsonProcessingExce Assertions.assertEquals("2021-03-30 08:44:29", formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli((Long) avroRecord.get(0)), ZoneId.of("UTC+1")))); Assertions.assertEquals("2021-03-30 15:44:29", formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli((Long) avroRecord.get(0)), ZoneId.of("Asia/Shanghai")))); } + + @Test + void testRowDataToAvroBlobTypeFieldWritesEnumSymbol() { + // Flink models the BLOB `type` discriminator as STRING, but its Avro encoding is an ENUM + // (blob_storage_type). The converter must emit a GenericData.EnumSymbol, not a plain Utf8, + // otherwise Avro log-block writes (MOR) fail with "value OUT_OF_LINE (a Utf8) is not a + // blob_storage_type". + DataType blobRow = DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.TYPE, DataTypes.STRING().notNull()), + DataTypes.FIELD(HoodieSchema.Blob.INLINE_DATA_FIELD, DataTypes.BYTES().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE, DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_PATH, DataTypes.STRING().notNull()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_OFFSET, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_LENGTH, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_IS_MANAGED, DataTypes.BOOLEAN().notNull()) + ).nullable())); + RowType rowType = (RowType) DataTypes.ROW(DataTypes.FIELD("blob_col", blobRow)).getLogicalType(); + + GenericRowData reference = new GenericRowData(4); + reference.setField(0, StringData.fromString("file1.bin")); + reference.setField(1, 0L); + reference.setField(2, 100L); + reference.setField(3, false); + + GenericRowData blob = new GenericRowData(3); + blob.setField(0, StringData.fromString(HoodieSchema.Blob.OUT_OF_LINE)); + blob.setField(1, null); + blob.setField(2, reference); + + GenericRowData top = new GenericRowData(1); + top.setField(0, blob); + + RowDataToAvroConverters.RowDataToAvroConverter converter = + RowDataToAvroConverters.createConverter(rowType); + GenericRecord avroRecord = + (GenericRecord) converter.convert(HoodieSchemaConverter.convertToSchema(rowType), top); + + GenericRecord blobRecord = (GenericRecord) avroRecord.get(0); + Object typeValue = blobRecord.get(HoodieSchema.Blob.TYPE); + Assertions.assertInstanceOf(GenericData.EnumSymbol.class, typeValue, + "BLOB `type` must be written as an Avro EnumSymbol, found: " + + (typeValue == null ? "null" : typeValue.getClass().getName())); + Assertions.assertEquals(HoodieSchema.Blob.OUT_OF_LINE, typeValue.toString()); + } + + /** + * A ROW whose field names match the BLOB structure but whose {@link HoodieSchema} carries a + * plain {@code STRING} (not {@code ENUM}) for the {@code type} field must write a plain + * {@link Utf8}, not a {@link GenericData.EnumSymbol}. + */ + @Test + void testBlobShapedRowWithPlainStringSchemaWritesUtf8() { + DataType blobShapedRow = DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.TYPE, DataTypes.STRING().notNull()), + DataTypes.FIELD(HoodieSchema.Blob.INLINE_DATA_FIELD, DataTypes.BYTES().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE, DataTypes.ROW( + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_PATH, DataTypes.STRING().notNull()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_OFFSET, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_LENGTH, DataTypes.BIGINT().nullable()), + DataTypes.FIELD(HoodieSchema.Blob.EXTERNAL_REFERENCE_IS_MANAGED, DataTypes.BOOLEAN().notNull()) + ).nullable())); + RowType outerRowType = (RowType) DataTypes.ROW( + DataTypes.FIELD("blob_col", blobShapedRow)).getLogicalType(); + + // Plain RECORD schema: field[0] is STRING (not ENUM) — mimics a non-BLOB record whose + // shape happens to match the BLOB structure. + HoodieSchema refSchema = HoodieSchema.createRecord("reference", null, null, Arrays.asList( + HoodieSchemaField.of(HoodieSchema.Blob.EXTERNAL_REFERENCE_PATH, + HoodieSchema.create(HoodieSchemaType.STRING)), + HoodieSchemaField.of(HoodieSchema.Blob.EXTERNAL_REFERENCE_OFFSET, + HoodieSchema.createNullable(HoodieSchemaType.LONG)), + HoodieSchemaField.of(HoodieSchema.Blob.EXTERNAL_REFERENCE_LENGTH, + HoodieSchema.createNullable(HoodieSchemaType.LONG)), + HoodieSchemaField.of(HoodieSchema.Blob.EXTERNAL_REFERENCE_IS_MANAGED, + HoodieSchema.create(HoodieSchemaType.BOOLEAN)) + )); + HoodieSchema plainBlobShapedSchema = HoodieSchema.createRecord("blob_col", null, null, Arrays.asList( + HoodieSchemaField.of(HoodieSchema.Blob.TYPE, HoodieSchema.create(HoodieSchemaType.STRING)), + HoodieSchemaField.of(HoodieSchema.Blob.INLINE_DATA_FIELD, + HoodieSchema.createNullable(HoodieSchemaType.BYTES)), + HoodieSchemaField.of(HoodieSchema.Blob.EXTERNAL_REFERENCE, + HoodieSchema.createNullable(refSchema)) + )); + HoodieSchema outerSchema = HoodieSchema.createRecord("outer", null, null, Arrays.asList( + HoodieSchemaField.of("blob_col", plainBlobShapedSchema) + )); + + GenericRowData reference = new GenericRowData(4); + reference.setField(0, StringData.fromString("file1.bin")); + reference.setField(1, 0L); + reference.setField(2, 100L); + reference.setField(3, false); + + GenericRowData blobRow = new GenericRowData(3); + blobRow.setField(0, StringData.fromString("OUT_OF_LINE")); + blobRow.setField(1, null); + blobRow.setField(2, reference); + + GenericRowData top = new GenericRowData(1); + top.setField(0, blobRow); + + RowDataToAvroConverters.RowDataToAvroConverter converter = + RowDataToAvroConverters.createConverter(outerRowType); + GenericRecord avroRecord = (GenericRecord) converter.convert(outerSchema, top); + + GenericRecord blobRecord = (GenericRecord) avroRecord.get(0); + Object typeValue = blobRecord.get(HoodieSchema.Blob.TYPE); + Assertions.assertInstanceOf(Utf8.class, typeValue, + "STRING field must write as Utf8 (not EnumSymbol) when HoodieSchema is not ENUM; found: " + + (typeValue == null ? "null" : typeValue.getClass().getName())); + Assertions.assertEquals("OUT_OF_LINE", typeValue.toString()); + } } \ No newline at end of file From a64cefc09293a0fb92e37455f9c484c05f816fcb Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Wed, 17 Jun 2026 11:25:41 +0800 Subject: [PATCH 089/255] fix(flink): reuse the preceeding avg size if there is no eligible estimation (#19022) * fix(flink): reuse the preceeding avg size if there is no eligible estimation (cherry picked from commit 5579715ef7d926fc68c9b0dc1918ff835c798e89) --- .../profile/DeltaWriteProfile.java | 6 +- .../partitioner/profile/WriteProfile.java | 8 +- .../sink/partitioner/TestBucketAssigner.java | 85 +++++++++++++++++++ 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java index a79c78e6d3639..2cdee3453a467 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java @@ -29,8 +29,6 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.action.commit.SmallFile; -import lombok.extern.slf4j.Slf4j; - import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -41,7 +39,6 @@ * *

    Note: assumes the index can always index log files for Flink write. */ -@Slf4j public class DeltaWriteProfile extends WriteProfile { public DeltaWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) { @@ -93,7 +90,7 @@ protected List smallFilesProfile(String partitionPath) { @Override protected long averageBytesPerRecord() { - long avgSize = config.getCopyOnWriteRecordSizeEstimate(); + long avgSize = this.avgSize > 0 ? this.avgSize : config.getCopyOnWriteRecordSizeEstimate(); HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants(); if (!commitTimeline.empty()) { long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D); @@ -109,7 +106,6 @@ protected long averageBytesPerRecord() { } } } - log.info("Refresh average bytes per record => " + avgSize); return avgSize; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java index c3819bf081166..304d8f5d7452e 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java @@ -78,7 +78,7 @@ public class WriteProfile { * The average record size. */ @Getter - private long avgSize = -1L; + protected long avgSize = -1L; /** * Total records to write for each bucket based on @@ -134,7 +134,7 @@ public WriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) * records pack into one file. */ protected long averageBytesPerRecord() { - long avgSize = config.getCopyOnWriteRecordSizeEstimate(); + long avgSize = this.avgSize > 0 ? this.avgSize : config.getCopyOnWriteRecordSizeEstimate(); HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants(); if (!commitTimeline.empty()) { long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D); @@ -142,7 +142,6 @@ protected long averageBytesPerRecord() { avgSize = sizeFromCommitMetadata; } } - log.info("Refresh average bytes per record => " + avgSize); return avgSize; } @@ -238,8 +237,9 @@ private void cleanMetadataCache(Stream instants) { private void recordProfile() { this.avgSize = averageBytesPerRecord(); + log.info("Refresh average bytes per record => {}", avgSize); this.recordsPerBucket = config.getParquetMaxFileSize() / avgSize; - log.info("Refresh insert records per bucket => " + recordsPerBucket); + log.info("Refresh insert records per bucket => {}", recordsPerBucket); } /** diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java index 8281ebbd86e4f..3b5c265f7c916 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.FlinkOptions; @@ -46,11 +47,13 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -444,6 +447,26 @@ public void testWriteProfileRecordsPerBucketUsesProfiledRecordSizeWithSmallEstim writeProfile.getRecordsPerBucket(), is(writeConfig.getParquetMaxFileSize() / expectedAvgSize)); } + @Test + public void testWriteProfileReusesPreviousAvgSizeWhenNoEligibleCommitOnReload() throws Exception { + conf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024"); + TestData.writeData(TestData.DATA_SET_INSERT, conf); + + writeConfig = FlinkWriteClients.getHoodieClientConfig(conf); + setScriptedRecordSizes(512L, -1L); + WriteProfile writeProfile = new ScriptedRecordSizeWriteProfile(writeConfig, context); + assertThat("Average record size should use the profiled commit metadata", + writeProfile.getAvgSize(), is(512L)); + + writeProfile.reload(1); + + assertThat("Average record size should reuse the previous estimate when no eligible commit metadata is found", + writeProfile.getAvgSize(), is(512L)); + assertThat("Records per bucket should continue to use the previous estimate", + writeProfile.getRecordsPerBucket(), is(writeConfig.getParquetMaxFileSize() / 512L)); + } + @Test public void testDeltaWriteProfileRecordsPerBucketUsesCompressionRatio() throws Exception { File morPath = new File(tempFile, "mor"); @@ -505,6 +528,34 @@ public void testDeltaWriteProfileRecordsPerBucketSkipsCompressionRatioForParquet writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / expectedAvgSize)); } + @Test + public void testDeltaWriteProfileReusesPreviousAvgSizeWhenNoEligibleDeltaCommitOnReload() throws Exception { + File morPath = new File(tempFile, "mor_reuse_previous_avg"); + Configuration morConf = TestConfigurations.getDefaultConf(morPath.getAbsolutePath()); + morConf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); + morConf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1); + morConf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024"); + StreamerUtil.initTableIfNotExists(morConf); + TestData.writeData(TestData.DATA_SET_INSERT, morConf); + + HoodieWriteConfig morWriteConfig = FlinkWriteClients.getHoodieClientConfig(morConf); + HoodieFlinkEngineContext morContext = new HoodieFlinkEngineContext( + HadoopFSUtils.getStorageConf(HadoopConfigurations.getHadoopConf(morConf)), + new FlinkTaskContextSupplier(null)); + + setScriptedRecordSizes(256L, -1L); + DeltaWriteProfile writeProfile = new ScriptedRecordSizeDeltaWriteProfile(morWriteConfig, morContext); + assertThat("Average record size should use the profiled delta commit metadata", + writeProfile.getAvgSize(), is(256L)); + + writeProfile.reload(1); + + assertThat("Average record size should reuse the previous estimate when no eligible delta commit metadata is found", + writeProfile.getAvgSize(), is(256L)); + assertThat("Records per bucket should continue to use the previous estimate", + writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / 256L)); + } + private static String getLastCompleteInstant(WriteProfile profile) { return StreamerUtil.getLastCompletedInstant(profile.getMetaClient()); } @@ -526,6 +577,40 @@ private void assertBucketEquals( assertThat(bucketInfo.getBucketType(), is(bucketType)); } + private static Queue scriptedRecordSizes = new ArrayDeque<>(); + + private static void setScriptedRecordSizes(Long... recordSizes) { + scriptedRecordSizes = new ArrayDeque<>(Arrays.asList(recordSizes)); + } + + /** + * WriteProfile with scripted record size estimates. + */ + static class ScriptedRecordSizeWriteProfile extends WriteProfile { + ScriptedRecordSizeWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) { + super(config, context); + } + + @Override + protected long calculateRecordSizeThroughCommitMetadata(HoodieTimeline commitTimeline, double fileSizeCalibrationRatio) { + return scriptedRecordSizes.remove(); + } + } + + /** + * DeltaWriteProfile with scripted record size estimates. + */ + static class ScriptedRecordSizeDeltaWriteProfile extends DeltaWriteProfile { + ScriptedRecordSizeDeltaWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) { + super(config, context); + } + + @Override + protected long calculateRecordSizeThroughCommitMetadata(HoodieTimeline commitTimeline, double fileSizeCalibrationRatio) { + return scriptedRecordSizes.remove(); + } + } + /** * Mock BucketAssigner that can specify small files explicitly. */ From 15dd0838f1a8ecd56deaf1d990caa0c191bef43b Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 17 Jun 2026 10:42:08 +0700 Subject: [PATCH 090/255] test(flink): de-flake testStreamReadMorTableWithCompactionFromEarliest (#19019) (cherry picked from commit d671c276261a7ac8fdda88f6169df571624637c9) --- .../hudi/table/ITTestHoodieDataSource.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 32b5b9476f143..301450a2602f4 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -3892,6 +3892,15 @@ private List fetchResultWithExpectedNum(TableEnvironment tEnv, TableResult // fail the job on stream-closed-mid-read (the right behavior for real I/O // failures), so this tolerance is scoped to the SuccessException-based test // pattern below and is NOT mirrored in production code. + // 3. NullPointerException from ParquetColumnarRowSplitReader#readNextRowGroup: the + // same benign teardown race as (2), observed with different timing. When the + // SplitFetcher's close() fully completes first, ParquetColumnarRowSplitReader#close + // nulls out its `reader` field, so the in-flight row-group read on the task thread + // surfaces as a NullPointerException (reader.readNextRowGroup() on a null reader) + // instead of an IOException("Stream is closed!"). Same functional outcome - the + // sink has already collected the expected rows - only the error symptom differs. + // Tolerated narrowly (an NPE originating from that exact frame) for the same + // reason as (2), and likewise NOT mirrored in production code. if (!isAcceptableTerminalFailure(e)) { throw new AssertionError("Unexpected job failure", e); } @@ -3917,8 +3926,41 @@ private static boolean isAcceptableTerminalFailure(Throwable e) { if (msg != null && msg.contains("Stream is closed")) { return true; } + // The NPE twin of the "Stream is closed!" teardown race (cause #3 at the call site): + // a NullPointerException whose own stack trace originates from + // ParquetColumnarRowSplitReader#readNextRowGroup, i.e. reader.readNextRowGroup() ran on a + // null `reader` that ParquetColumnarRowSplitReader#close had just nulled out. Scoped to + // that exact frame so genuine NPEs - and the legitimate IOException("expecting more + // rows...") thrown from the same method - still fail the test. + if (isNullPointerException(cur) && containsReadNextRowGroupFrame(cur)) { + return true; + } cur = cur.getCause(); } return false; } + + /** + * True for a real {@link NullPointerException} as well as one wrapped in Flink's + * {@code SerializedThrowable} when the failure is propagated back from the cluster (its + * {@code toString()} preserves the original {@code java.lang.NullPointerException} prefix). + */ + private static boolean isNullPointerException(Throwable t) { + return t instanceof NullPointerException + || t.toString().startsWith(NullPointerException.class.getName()); + } + + /** + * Whether {@code t}'s stack trace (preserved even through {@code SerializedThrowable}) + * contains a {@code ParquetColumnarRowSplitReader#readNextRowGroup} frame. + */ + private static boolean containsReadNextRowGroupFrame(Throwable t) { + for (StackTraceElement frame : t.getStackTrace()) { + if (frame.getClassName().endsWith("ParquetColumnarRowSplitReader") + && "readNextRowGroup".equals(frame.getMethodName())) { + return true; + } + } + return false; + } } From 783a9935c80f703c1334fe3f7cd215c699d542cc Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Thu, 18 Jun 2026 09:33:12 +0700 Subject: [PATCH 091/255] test(flink): retry short CollectSink reads to de-flake stream-read ITs (#19030) * test(flink): retry short CollectSink reads to de-flake stream-read ITs ITTestHoodieDataSource streaming reads are collected with CollectTableSink and terminated by a forced SuccessException at the expected row count. A tolerated teardown race (Stream-is-closed / readNextRowGroup NPE, broadened in #19019) can fire before the read completes, so the job ends with fewer rows than expected and the test asserts on an incomplete result. Wrap submit-and-collect in submitAndFetchWithRetry, which re-reads (up to 3 times) when the collected count is below the expectation; re-reading the already committed table is idempotent. Log the swallowed non-SuccessException cause for diagnosability. Test-only change, no production code touched. * addressed review comments: fix stale comment reference and shorten terminal-cause log --------- Co-authored-by: Vova Kolmakov (cherry picked from commit c4426265e1f4f349dca53671a0826e7e68cb6513) --- .../hudi/table/ITTestHoodieDataSource.java | 72 +++++++++++++++++-- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 301450a2602f4..399829f94189e 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -79,6 +79,8 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -126,6 +128,15 @@ */ @ExtendWith(FlinkMiniCluster.class) public class ITTestHoodieDataSource { + private static final Logger LOG = LoggerFactory.getLogger(ITTestHoodieDataSource.class); + + // A streaming read collected via CollectTableSink is terminated by a forced SuccessException once it + // reaches its expected row count. A benign teardown race (see isAcceptableTerminalFailure) can instead + // close the source stream mid-read and terminate the job before all rows are emitted, leaving an + // incomplete result. Re-reading from the same (already committed) table is idempotent, so retry a few + // times before giving up. See submitAndFetchWithRetry. + private static final int MAX_STREAM_READ_ATTEMPTS = 3; + private TableEnvironment streamTableEnv; private TableEnvironment batchTableEnv; @@ -560,7 +571,7 @@ void testStreamReadWithDeletes() throws Exception { + " 'connector' = '" + CollectSinkTableFactory.FACTORY_ID + "',\n" + " 'sink-expected-row-num' = '2'" + ")"; - List result = execSelectSqlWithExpectedNum(streamTableEnv, "select name, sum(age) from t1 group by name", sinkDDL); + List result = submitAndFetchWithRetry(streamTableEnv, "select name, sum(age) from t1 group by name", sinkDDL, 2); final String expected = "[+I(+I[Danny, 24]), +I(+I[Stephen, 34])]"; assertRowsEquals(result, expected, true); } @@ -3839,15 +3850,32 @@ private List execSelectSqlWithExpectedNum(TableEnvironment tEnv, String sel } else { sinkDDL = TestConfigurations.getCollectSinkDDLWithExpectedNum("sink", expectedNum); } - return execSelectSqlWithExpectedNum(tEnv, select, sinkDDL); + return submitAndFetchWithRetry(tEnv, select, sinkDDL, expectedNum); } /** - * Use CollectTableSink to collect results with expected row number. + * Submits a streaming select that collects into the {@link CollectSinkTableFactory} sink and returns + * the collected rows. + * + *

    The streaming job is terminated by a forced {@link CollectSinkTableFactory.SuccessException} once + * {@code expectedNum} rows are collected. A benign teardown race (see {@link #isAcceptableTerminalFailure}) + * can instead end the job before all rows are emitted, leaving a short result. Re-reading the already + * committed table is idempotent, so retry up to {@link #MAX_STREAM_READ_ATTEMPTS} times when the result + * is short; this keeps the race from surfacing as a confusing row-count assertion failure. */ - private List execSelectSqlWithExpectedNum(TableEnvironment tEnv, String select, String sinkDDL) { - TableResult tableResult = submitSelectSql(tEnv, select, sinkDDL); - return fetchResultWithExpectedNum(tEnv, tableResult); + private List submitAndFetchWithRetry(TableEnvironment tEnv, String select, String sinkDDL, int expectedNum) { + List rows = Collections.emptyList(); + for (int attempt = 1; attempt <= MAX_STREAM_READ_ATTEMPTS; attempt++) { + TableResult tableResult = submitSelectSql(tEnv, select, sinkDDL); + rows = fetchResultWithExpectedNum(tEnv, tableResult); + if (expectedNum <= 0 || rows.size() >= expectedNum) { + return rows; + } + LOG.warn("Streaming read collected {} of {} expected rows on attempt {}/{}; a tolerated teardown " + + "race ended the job before the read completed. Retrying. select=[{}]", + rows.size(), expectedNum, attempt, MAX_STREAM_READ_ATTEMPTS, select); + } + return rows; } private TableResult submitSelectSql(TableEnvironment tEnv, String select, String sinkDDL) { @@ -3904,6 +3932,14 @@ private List fetchResultWithExpectedNum(TableEnvironment tEnv, TableResult if (!isAcceptableTerminalFailure(e)) { throw new AssertionError("Unexpected job failure", e); } + // The races (2)/(3) usually fire after the sink has collected its expected rows, but can also fire + // before - ending the read with a short result. Log the tolerated cause so an incomplete read is + // diagnosable; submitAndFetchWithRetry re-reads when the collected count is below the expectation. + if (!isSuccessException(e)) { + LOG.warn("Streaming read terminated by a tolerated teardown race ({}); collected {} rows so far.", + describeTerminalCause(e), + CollectSinkTableFactory.RESULT.values().stream().mapToInt(List::size).sum()); + } } tEnv.executeSql("DROP TABLE IF EXISTS sink"); return CollectSinkTableFactory.RESULT.values().stream() @@ -3963,4 +3999,28 @@ private static boolean containsReadNextRowGroupFrame(Throwable t) { } return false; } + + /** + * Whether {@code e} (or any of its causes) is the normal {@link CollectSinkTableFactory.SuccessException} + * terminator (the happy path), as opposed to one of the tolerated teardown-race symptoms. + */ + private static boolean isSuccessException(Throwable e) { + for (Throwable cur = e; cur != null; cur = cur.getCause()) { + if (cur instanceof CollectSinkTableFactory.SuccessException) { + return true; + } + } + return false; + } + + /** + * Short description of {@code e}'s root cause, for logging which tolerated terminal failure fired. + */ + private static String describeTerminalCause(Throwable e) { + Throwable root = e; + while (root.getCause() != null) { + root = root.getCause(); + } + return root.getClass().getSimpleName() + ": " + root.getMessage(); + } } From 38f524b48bdc6713a77a957a97ea858923ce7f87 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Thu, 18 Jun 2026 14:41:52 +0700 Subject: [PATCH 092/255] test(trino): de-flake TestHudi*FileOperations by asserting only synchronous reads (#19004) (cherry picked from commit 4bafe5a6075ef27691ef126cb412759d83e4a611) --- .../TestHudiAlluxioCacheFileOperations.java | 79 ++++++++++++------- .../TestHudiMemoryCacheFileOperations.java | 26 +++--- .../hudi/TestHudiNoCacheFileOperations.java | 26 +++--- 3 files changed, 70 insertions(+), 61 deletions(-) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java index 89507721afa59..911c5d4c3e7d4 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultiset; import com.google.common.collect.Multiset; +import io.airlift.units.Duration; import io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer; import io.trino.plugin.hudi.util.FileOperationUtils.FileOperation; import io.trino.testing.AbstractTestQueryFramework; @@ -36,7 +37,6 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.getCacheOperationSpans; import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_FG_PT_V8_MOR; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.DATA; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.INDEX_DEFINITION; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.LOG; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE; @@ -44,7 +44,9 @@ import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TABLE_PROPERTIES; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TIMELINE; import static io.trino.testing.MultisetAssertions.assertMultisetsEqual; +import static io.trino.testing.assertions.Assert.assertEventually; import static java.util.stream.Collectors.toCollection; +import static org.assertj.core.api.Assertions.assertThat; @ResourceLock("HUDI_CACHE_SYSTEM") @Execution(ExecutionMode.SAME_THREAD) @@ -66,10 +68,11 @@ protected DistributedQueryRunner createQueryRunner() .put("fs.cache.directories", cacheDirectory.toAbsolutePath().toString()) .put("fs.cache.max-sizes", "100MB") .put("hudi.metadata.cache.enabled", "false") - // Disable async table-statistics refresh: it reads the metadata table on a - // background executor whose spans can outlive the query and leak into the next - // test's measurement (the symmetric off-by-N flake). Disabling it makes the - // file-operation counts deterministic right after the query returns. + // Disable the async table-statistics refresh: on the first query it reads the index + // definitions and table-property files (and the metadata table) on a background + // executor. Those non-metadata-table reads land in the asserted set and their timing + // is non-deterministic, so we turn the refresh off and assert only the synchronous + // planning-path reads. .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); @@ -87,12 +90,6 @@ public void testSelectWithFilter() assertFileSystemAccesses( query, ImmutableMultiset.builder() - .addCopies(new FileOperation("Alluxio.readCached", DATA), 2) - .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 27) - .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 4) - .addCopies(new FileOperation("Alluxio.readCached", LOG), 15) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4) - .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 10) .addCopies(new FileOperation("InputFile.length", TIMELINE), 2) .addCopies(new FileOperation("InputFile.length", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) @@ -103,12 +100,6 @@ public void testSelectWithFilter() assertFileSystemAccesses( query, ImmutableMultiset.builder() - .addCopies(new FileOperation("Alluxio.readCached", DATA), 2) - .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 27) - .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 4) - .addCopies(new FileOperation("Alluxio.readCached", LOG), 15) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4) - .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 10) .addCopies(new FileOperation("InputFile.length", TIMELINE), 2) .addCopies(new FileOperation("InputFile.length", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) @@ -127,12 +118,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() - .addCopies(new FileOperation("Alluxio.readCached", DATA), 6) - .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 215) - .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 8) - .addCopies(new FileOperation("Alluxio.readCached", LOG), 30) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) - .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 69) .addCopies(new FileOperation("InputFile.length", TIMELINE), 4) .addCopies(new FileOperation("InputFile.length", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) @@ -142,12 +127,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() - .addCopies(new FileOperation("Alluxio.readCached", DATA), 6) - .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 215) - .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 8) - .addCopies(new FileOperation("Alluxio.readCached", LOG), 30) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) - .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 69) .addCopies(new FileOperation("InputFile.length", TIMELINE), 4) .addCopies(new FileOperation("InputFile.length", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) @@ -156,6 +135,40 @@ public void testJoin() .build()); } + @Test + public void testReadsServedFromAlluxioCache() + { + // The tests above intentionally do not assert exact Alluxio cache hit/miss counts: the cache + // write is asynchronous, so the per-query counts flake (a write from one query can still be in + // flight when the next query runs). This test instead gives count-independent coverage that the + // Alluxio cache is actually engaged: once the cache is warmed, at least one read is served from + // it (an "Alluxio.readCached" span). assertEventually re-runs the query until the asynchronous + // cache write has landed and a genuine hit is observed, and fails loudly at the deadline if the + // cache never serves a read. + @Language("SQL") String query = "SELECT * FROM " + HUDI_MULTI_FG_PT_V8_MOR; + DistributedQueryRunner queryRunner = getDistributedQueryRunner(); + + // Warm the cache; the page write into Alluxio happens on a background thread. + queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); + + assertEventually( + Duration.valueOf("30s"), + Duration.valueOf("500ms"), + () -> { + queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); + assertThat(countCachedReads(queryRunner)) + .as("Alluxio.readCached spans (cache hits)") + .isGreaterThanOrEqualTo(1); + }); + } + + private static long countCachedReads(QueryRunner queryRunner) + { + return getCacheOperationSpans(queryRunner).stream() + .filter(span -> span.getName().equals("Alluxio.readCached")) + .count(); + } + private void assertFileSystemAccesses(@Language("SQL") String query, Multiset expectedCacheAccesses) { DistributedQueryRunner queryRunner = getDistributedQueryRunner(); @@ -169,6 +182,14 @@ public static Multiset getFileOperations(QueryRunner queryRunner) .stream() .filter(span -> !span.getName().startsWith("InputFile.exists")) .map(FileOperation::create) + // Metadata-table reads are issued from Hudi background pools (split loading, partition + // listing, table-statistics refresh) whose spans can outlive the synchronous query and + // land in the next query's measurement window, so their per-query counts are not + // deterministic. Alluxio cache hits/misses (Alluxio.*) depend on whether an earlier + // asynchronous cache write had already completed, so their counts are not deterministic + // either. Both are excluded; only synchronous foreground reads are asserted. + .filter(operation -> operation.fileType() != METADATA_TABLE) + .filter(operation -> !operation.operationType().startsWith("Alluxio.")) .collect(toCollection(HashMultiset::create)); } } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java index 61867b1cde7dd..8e1b2fe819119 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java @@ -56,10 +56,11 @@ protected DistributedQueryRunner createQueryRunner() .put("hudi.metadata-enabled", "true") .put("hudi.metadata.cache.enabled", "true") .put("fs.cache.enabled", "false") - // Disable async table-statistics refresh: it reads the metadata table on a - // background executor whose spans can outlive the query and leak into the next - // test's measurement (the symmetric off-by-N flake). Disabling it makes the - // file-operation counts deterministic right after the query returns. + // Disable the async table-statistics refresh: on the first query it reads the index + // definitions and table-property files (and the metadata table) on a background + // executor. Those non-metadata-table reads land in the asserted set and their timing + // is non-deterministic, so we turn the refresh off and assert only the synchronous + // planning-path reads. .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); @@ -78,11 +79,8 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2) - .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 4) - .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 6) .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2) .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -92,11 +90,8 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2) - .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 4) - .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 6) .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2) .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -114,11 +109,8 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6) - .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 29) - .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 40) .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4) .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) @@ -127,11 +119,8 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6) - .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 29) - .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 40) .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4) .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2) - .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) @@ -153,6 +142,11 @@ private static Multiset getFileOperations(QueryRunner queryRunner .filter(span -> !span.getName().startsWith("InputFile.exists")) .filter(span -> !isTrinoSchemaOrPermissions(getFileLocation(span))) .map(FileOperation::create) + // Metadata-table reads are issued from Hudi background pools (split loading, partition + // listing, table-statistics refresh) whose spans can outlive the synchronous query and + // land in the next query's measurement window. Their per-query counts are therefore + // non-deterministic, so they are excluded; only synchronous foreground reads are asserted. + .filter(operation -> operation.fileType() != METADATA_TABLE) .collect(toCollection(HashMultiset::create)); } } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java index 71518b9fc67b6..55071241fff7a 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java @@ -56,10 +56,11 @@ protected DistributedQueryRunner createQueryRunner() .put("hudi.metadata-enabled", "true") .put("hudi.metadata.cache.enabled", "false") .put("fs.cache.enabled", "false") - // Disable async table-statistics refresh: it reads the metadata table on a - // background executor whose spans can outlive the query and leak into the next - // test's measurement (the symmetric off-by-N flake). Disabling it makes the - // file-operation counts deterministic right after the query returns. + // Disable the async table-statistics refresh: on the first query it reads the index + // definitions and table-property files (and the metadata table) on a background + // executor. Those non-metadata-table reads land in the asserted set and their timing + // is non-deterministic, so we turn the refresh off and assert only the synchronous + // planning-path reads. .put("hudi.table-statistics-enabled", "false") .buildOrThrow(); @@ -78,9 +79,6 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 6) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1) @@ -92,10 +90,7 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 6) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1) .add(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) @@ -114,10 +109,7 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 6) - .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 29) - .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 29) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 40) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4) @@ -127,10 +119,7 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 6) - .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 29) - .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 29) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 40) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4) @@ -153,6 +142,11 @@ private static Multiset getFileOperations(Quer .filter(span -> !span.getName().startsWith("InputFile.exists")) .filter(span -> !isTrinoSchemaOrPermissions(getFileLocation(span))) .map(FileOperationUtils.FileOperation::create) + // Metadata-table reads are issued from Hudi background pools (split loading, partition + // listing, table-statistics refresh) whose spans can outlive the synchronous query and + // land in the next query's measurement window. Their per-query counts are therefore + // non-deterministic, so they are excluded; only synchronous foreground reads are asserted. + .filter(operation -> operation.fileType() != METADATA_TABLE) .collect(toCollection(HashMultiset::create)); } } From 5223271b194d2fea8250020ad3c23ac622a548ca Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Mon, 22 Jun 2026 01:48:47 -0700 Subject: [PATCH 093/255] fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats (#18904) * fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats HoodieHeartbeatClient could permanently stop generating heartbeats for an instant, causing later commits to abort with "Heartbeat for instant ... has expired" even though the writer was still alive: - The heartbeat file is written synchronously on the Timer thread. Because the timer uses scheduleAtFixedRate, a slow or hung storage write blocks the thread and freezes all subsequent heartbeats for that instant. - When a heartbeat refresh is delayed past the tolerable interval, updateHeartbeat() called Thread.currentThread().interrupt(), permanently killing the timer thread and turning a transient delay into a permanent blackout. Fix: - Perform the heartbeat file write on a bounded daemon executor (Future.get with a per-interval timeout) so a slow or hung storage call cannot block the timer thread; a timed-out write is retried on the next tick. - Remove the self-interrupt; log a warning and continue. The commit-time check HeartbeatUtils.abortIfHeartbeatExpired() remains the sole enforcement point. Add TestHoodieHeartbeatClient.testTimerSurvivesHungHeartbeatWrite. * fix: address review comments on heartbeat resilience - isHeartbeatExpired: handle a null last-heartbeat-time (the very first write can time out, leaving it unset) by falling back to the DFS read, avoiding an NPE on the unboxing comparison. - On a detected lapse, stop refreshing the heartbeat (cancel the timer) and do not advance the last-heartbeat time, so a lapsed writer still aborts at commit via HeartbeatUtils.abortIfHeartbeatExpired() and cannot resurrect a heartbeat that a concurrent cleaner (LAZY failed-writes policy) may already have acted on. The timer is cancelled cleanly rather than via Thread.interrupt(). - close(): make idempotent via a closed flag and guard executor creation after close. - Use boxed Long for heartbeatWriteTimeoutMs to match the sibling duration fields. - Raise the default hoodie.client.heartbeat.tolerable.misses from 2 to 10 so transient driver pauses (e.g. GC) or storage-latency spikes do not abort a still-healthy writer. * fix: keep heartbeat client reusable after close A write client reuses its HoodieHeartbeatClient across operations: after close(), startCommit() and acquireRollbackHeartbeatIfMultiWriter() call heartbeatClient.start() again. The previously added "already closed" guard in the executor accessor turned this valid reuse into a failure (observed in TestJavaHoodieBackedMetadata and TestHoodieJavaClientOnCopyOnWriteStorage). Remove the guard and the closed flag; close() remains idempotent via synchronization plus the null-check on the executor, so repeated/concurrent close() is still safe while reuse-after-close works (the executor is lazily re-created on the next heartbeat). * switch timer into a executor service --------- Co-authored-by: danny0405 (cherry picked from commit c6f38c6b6e84212f6f9aec5c88ae271ff424e741) --- .../heartbeat/HoodieHeartbeatClient.java | 150 ++++++++++++++---- .../apache/hudi/config/HoodieWriteConfig.java | 6 +- .../heartbeat/TestHoodieHeartbeatClient.java | 102 ++++++++++++ 3 files changed, 224 insertions(+), 34 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java index a043f73e632c5..b8f2f15fdf0a4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java @@ -19,6 +19,7 @@ package org.apache.hudi.client.heartbeat; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.CustomizedThreadFactory; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieHeartbeatException; @@ -35,9 +36,15 @@ import java.io.OutputStream; import java.io.Serializable; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.apache.hudi.common.heartbeat.HoodieHeartbeatUtils.getLastHeartbeatTime; @@ -58,7 +65,16 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable { // heartbeat interval in millis private final Long heartbeatIntervalInMs; private final Long maxAllowableHeartbeatIntervalInMs; + // Maximum time the scheduler thread will wait for a single heartbeat file write to complete before + // abandoning it and letting the next tick retry. Bounded to one interval so that a slow/hung + // storage write cannot block the scheduler thread (and thus freeze all subsequent heartbeats). + private final Long heartbeatWriteTimeoutMs; private final Map instantToHeartbeatMap; + // Daemon executor used to perform the (potentially slow) storage write off the scheduler thread so the + // write can be time-bounded. A cached pool is intentional: if one write hangs, that thread is left + // parked while the next tick proceeds on a fresh thread. Lazily created and marked transient since + // this client is Serializable with a transient storage handle. + private transient ExecutorService heartbeatWriteExecutor; public HoodieHeartbeatClient(HoodieStorage storage, String basePath, Long heartbeatIntervalInMs, Integer numTolerableHeartbeatMisses) { @@ -68,9 +84,18 @@ public HoodieHeartbeatClient(HoodieStorage storage, String basePath, Long heartb this.heartbeatFolderPath = HoodieTableMetaClient.getHeartbeatFolderPath(basePath); this.heartbeatIntervalInMs = heartbeatIntervalInMs; this.maxAllowableHeartbeatIntervalInMs = this.heartbeatIntervalInMs * numTolerableHeartbeatMisses; + this.heartbeatWriteTimeoutMs = this.heartbeatIntervalInMs; this.instantToHeartbeatMap = new ConcurrentHashMap<>(); } + private synchronized ExecutorService getHeartbeatWriteExecutor() { + if (heartbeatWriteExecutor == null) { + heartbeatWriteExecutor = + Executors.newCachedThreadPool(new CustomizedThreadFactory("heartbeat_write", true)); + } + return heartbeatWriteExecutor; + } + @Data static class Heartbeat { @@ -79,10 +104,12 @@ static class Heartbeat { private boolean isHeartbeatStopped = false; private Long lastHeartbeatTime; private Integer numHeartbeats = 0; - private Timer timer = new Timer(true); + private ScheduledExecutorService heartbeatScheduler = + Executors.newSingleThreadScheduledExecutor(new CustomizedThreadFactory("heartbeat_scheduler", true)); + private ScheduledFuture scheduledFuture; } - class HeartbeatTask extends TimerTask { + class HeartbeatTask implements Runnable { private final String instantTime; @@ -92,7 +119,11 @@ class HeartbeatTask extends TimerTask { @Override public void run() { - updateHeartbeat(instantTime); + try { + updateHeartbeat(instantTime); + } catch (Exception e) { + log.error("Failed to update heartbeat for instant {}; will retry on next tick", instantTime, e); + } } } @@ -114,11 +145,11 @@ public void start(String instantTime) { newHeartbeat.setHeartbeatStarted(true); instantToHeartbeatMap.put(instantTime, newHeartbeat); // Ensure heartbeat is generated for the first time with this blocking call. - // Since timer submits the task to a thread, no guarantee when that thread will get CPU + // Since scheduler submits the task to a thread, no guarantee when that thread will get CPU // cycles to generate the first heartbeat. updateHeartbeat(instantTime); - newHeartbeat.getTimer().scheduleAtFixedRate(new HeartbeatTask(instantTime), this.heartbeatIntervalInMs, - this.heartbeatIntervalInMs); + newHeartbeat.setScheduledFuture(newHeartbeat.getHeartbeatScheduler().scheduleAtFixedRate( + new HeartbeatTask(instantTime), this.heartbeatIntervalInMs, this.heartbeatIntervalInMs, TimeUnit.MILLISECONDS)); } /** @@ -130,7 +161,7 @@ public void start(String instantTime) { public Heartbeat stop(String instantTime) throws HoodieException { Heartbeat heartbeat = instantToHeartbeatMap.remove(instantTime); if (isHeartbeatStarted(heartbeat)) { - stopHeartbeatTimer(heartbeat); + stopHeartbeatScheduler(heartbeat); HeartbeatUtils.deleteHeartbeatFile(storage, basePath, instantTime); log.info("Deleted heartbeat file for instant {}", instantTime); } @@ -138,12 +169,12 @@ public Heartbeat stop(String instantTime) throws HoodieException { } /** - * Stops all timers of heartbeats started via this instance of the client. + * Stops all heartbeat schedulers started via this instance of the client. * * @throws HoodieException */ public void stopHeartbeatTimers() throws HoodieException { - instantToHeartbeatMap.values().stream().filter(this::isHeartbeatStarted).forEach(this::stopHeartbeatTimer); + instantToHeartbeatMap.values().stream().filter(this::isHeartbeatStarted).forEach(this::stopHeartbeatScheduler); } /** @@ -158,17 +189,24 @@ private boolean isHeartbeatStarted(Heartbeat heartbeat) { } /** - * Stops the timer of the given heartbeat. + * Stops the scheduler of the given heartbeat. * * @param heartbeat The heartbeat to stop. */ - private void stopHeartbeatTimer(Heartbeat heartbeat) { + private void stopHeartbeatScheduler(Heartbeat heartbeat) { log.info("Stopping heartbeat for instant {}", heartbeat.getInstantTime()); - heartbeat.getTimer().cancel(); + shutdownHeartbeatScheduler(heartbeat); heartbeat.setHeartbeatStopped(true); log.info("Stopped heartbeat for instant {}", heartbeat.getInstantTime()); } + private void shutdownHeartbeatScheduler(Heartbeat heartbeat) { + if (heartbeat.getScheduledFuture() != null) { + heartbeat.getScheduledFuture().cancel(false); + } + heartbeat.getHeartbeatScheduler().shutdownNow(); + } + public static Boolean heartbeatExists(HoodieStorage storage, String basePath, String instantTime) throws IOException { StoragePath heartbeatFilePath = new StoragePath( HoodieTableMetaClient.getHeartbeatFolderPath(basePath), instantTime); @@ -178,17 +216,18 @@ public static Boolean heartbeatExists(HoodieStorage storage, String basePath, St public boolean isHeartbeatExpired(String instantTime) throws IOException { Long currentTime = System.currentTimeMillis(); Heartbeat lastHeartbeatForWriter = instantToHeartbeatMap.get(instantTime); - if (lastHeartbeatForWriter == null) { - log.info("Heartbeat not found in internal map, falling back to reading from DFS"); - long lastHeartbeatForWriterTime = getLastHeartbeatTime(this.storage, basePath, instantTime); - lastHeartbeatForWriter = new Heartbeat(); - lastHeartbeatForWriter.setLastHeartbeatTime(lastHeartbeatForWriterTime); - lastHeartbeatForWriter.setInstantTime(instantTime); - lastHeartbeatForWriter.getTimer().cancel(); + Long lastHeartbeatTime = lastHeartbeatForWriter == null ? null : lastHeartbeatForWriter.getLastHeartbeatTime(); + // lastHeartbeatTime can be null when the heartbeat is not in the internal map, or when it is in the + // map but no heartbeat has been generated yet (e.g. the first write timed out). In both cases fall + // back to reading the last heartbeat time from DFS (returns 0 if no heartbeat file exists, which is + // correctly treated as expired). + if (lastHeartbeatTime == null) { + log.info("Heartbeat time not available in internal map, falling back to reading from DFS"); + lastHeartbeatTime = getLastHeartbeatTime(this.storage, basePath, instantTime); } - if (currentTime - lastHeartbeatForWriter.getLastHeartbeatTime() > this.maxAllowableHeartbeatIntervalInMs) { + if (currentTime - lastHeartbeatTime > this.maxAllowableHeartbeatIntervalInMs) { log.warn("Heartbeat expired, currentTime = {}, last heartbeat = {}, heartbeat interval = {}", currentTime, - lastHeartbeatForWriter, this.heartbeatIntervalInMs); + lastHeartbeatTime, this.heartbeatIntervalInMs); return true; } return false; @@ -197,20 +236,31 @@ public boolean isHeartbeatExpired(String instantTime) throws IOException { private void updateHeartbeat(String instantTime) throws HoodieHeartbeatException { try { Long newHeartbeatTime = System.currentTimeMillis(); - OutputStream outputStream = - this.storage.create( - new StoragePath(heartbeatFolderPath, instantTime), true); - outputStream.close(); + writeHeartbeatFile(instantTime); Heartbeat heartbeat = instantToHeartbeatMap.get(instantTime); if (heartbeat.getLastHeartbeatTime() != null && isHeartbeatExpired(instantTime)) { - log.error("Aborting, missed generating heartbeat within allowable interval {} ms", this.maxAllowableHeartbeatIntervalInMs); - // Since TimerTask allows only java.lang.Runnable, cannot throw an exception and bubble to the caller thread, hence - // explicitly interrupting the timer thread. - Thread.currentThread().interrupt(); + // A previous refresh was delayed past the tolerable interval. Stop refreshing this heartbeat + // (cancel the scheduler) and do NOT advance the last heartbeat time, so the heartbeat stays expired + // and the writer aborts at commit time via HeartbeatUtils.abortIfHeartbeatExpired(). We must not + // keep refreshing here: a concurrent process (e.g. an async cleaner under LAZY failed-writes + // policy) may already have started rolling back this instant once it observed the expiry, and + // resurrecting the heartbeat could let this writer commit on top of rolled-back files. + // The scheduler is cancelled cleanly rather than via Thread.interrupt(), which would permanently + // kill the scheduler thread (turning a transient delay into a permanent blackout on the first miss). + log.error("Missed generating heartbeat for instant {} within allowable interval {} ms; stopping heartbeat refresh", + instantTime, this.maxAllowableHeartbeatIntervalInMs); + shutdownHeartbeatScheduler(heartbeat); + return; } heartbeat.setInstantTime(instantTime); heartbeat.setLastHeartbeatTime(newHeartbeatTime); heartbeat.setNumHeartbeats(heartbeat.getNumHeartbeats() + 1); + } catch (TimeoutException te) { + // The storage write did not complete within the bounded window. Do not advance the last heartbeat + // time (the write is unconfirmed); the next scheduled tick will retry on a fresh executor thread. + // Crucially, the scheduler thread is freed instead of being blocked by a hung storage call. + log.warn("Heartbeat file write for instant {} did not complete within {} ms; will retry on next tick", + instantTime, this.heartbeatWriteTimeoutMs); } catch (IOException io) { boolean isHeartbeatStopped = instantToHeartbeatMap.get(instantTime).isHeartbeatStopped(); if (isHeartbeatStopped) { @@ -221,13 +271,49 @@ private void updateHeartbeat(String instantTime) throws HoodieHeartbeatException } } + /** + * Writes the heartbeat file for the given instant on a dedicated daemon executor, bounded by + * {@link #heartbeatWriteTimeoutMs}. Performing the storage write off the scheduler thread (and with a + * timeout) ensures that a slow or hung storage call cannot block the scheduler thread and freeze all + * subsequent heartbeats for this instant. + */ + private void writeHeartbeatFile(String instantTime) throws IOException, TimeoutException { + Future future = getHeartbeatWriteExecutor().submit(() -> { + try (OutputStream outputStream = + this.storage.create(new StoragePath(heartbeatFolderPath, instantTime), true)) { + // create + close confirms the heartbeat file write landed on storage. + } + return null; + }); + try { + future.get(heartbeatWriteTimeoutMs, TimeUnit.MILLISECONDS); + } catch (TimeoutException te) { + future.cancel(true); + throw te; + } catch (InterruptedException ie) { + future.cancel(true); + Thread.currentThread().interrupt(); + throw new HoodieHeartbeatException("Interrupted while writing heartbeat for instant " + instantTime, ie); + } catch (ExecutionException ee) { + Throwable cause = ee.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } + throw new HoodieHeartbeatException("Failed to write heartbeat for instant " + instantTime, cause); + } + } + public Heartbeat getHeartbeat(String instantTime) { return this.instantToHeartbeatMap.get(instantTime); } @Override - public void close() { + public synchronized void close() { this.stopHeartbeatTimers(); this.instantToHeartbeatMap.clear(); + if (heartbeatWriteExecutor != null) { + heartbeatWriteExecutor.shutdownNow(); + heartbeatWriteExecutor = null; + } } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index a663c63dba780..1ba577faabf12 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -680,9 +680,11 @@ public class HoodieWriteConfig extends HoodieConfig { public static final ConfigProperty CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES = ConfigProperty .key("hoodie.client.heartbeat.tolerable.misses") - .defaultValue(2) + .defaultValue(10) .markAdvanced() - .withDocumentation("Number of heartbeat misses, before a writer is deemed not alive and all pending writes are aborted."); + .withDocumentation("Number of heartbeat misses, before a writer is deemed not alive and all pending writes are aborted. " + + "A higher value tolerates transient driver pauses (e.g. GC) or storage-latency spikes that would otherwise " + + "delay a heartbeat and cause a still-healthy writer's commit to be aborted."); public static final ConfigProperty CLUSTERING_BLOCK_FOR_PENDING_INGESTION = ConfigProperty .key("hoodie.clustering.fail.on.pending.ingestion.during.conflict.resolution") diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java index c7ea5fa87bbd6..5feb79ee83bc0 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java @@ -21,12 +21,18 @@ import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import org.apache.hudi.storage.hadoop.HoodieHadoopStorage; +import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.io.OutputStream; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; @@ -113,4 +119,100 @@ public void testStopHeartbeatTimers() throws IOException { assertFalse(hoodieHeartbeatClient.isHeartbeatExpired(instantTime1)); assertTrue(hoodieHeartbeatClient.getHeartbeat(instantTime1).isHeartbeatStopped()); } + + /** + * Regression test for the heartbeat-expiry incident: a single slow/hung storage write must not + * block (freeze) the heartbeat scheduler thread. The first heartbeat write blocks (simulating a hung + * cloud-storage call); we assert the scheduler keeps producing heartbeats on fresh threads once that + * write times out, proving the scheduler thread was not blocked by the synchronous storage call (#1). + * A high tolerable-misses is used so that recovery after the blocked write does not itself trip the + * expiry path (which intentionally stops refresh on a genuine lapse). + */ + @Test + public void testSlowHeartbeatWriteDoesNotBlockScheduler() { + CountDownLatch releaseFirstWrite = new CountDownLatch(1); + SlowCreateStorage slowStorage = + new SlowCreateStorage((FileSystem) metaClient.getStorage().getFileSystem(), releaseFirstWrite); + // interval 1s, write timeout = 1s; high tolerable-misses so the ~1s recovery gap stays well within + // the allowable window and the scheduler keeps beating rather than treating it as a lapse. + HoodieHeartbeatClient hoodieHeartbeatClient = + new HoodieHeartbeatClient(slowStorage, metaClient.getBasePath().toString(), + heartBeatInterval, 10); + try { + hoodieHeartbeatClient.start(instantTime1); + // Despite the first write hanging, the scheduler must keep generating heartbeats on fresh threads. + await().atMost(15, SECONDS) + .until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() >= 2); + } finally { + releaseFirstWrite.countDown(); + hoodieHeartbeatClient.close(); + } + } + + @Test + public void testScheduledHeartbeatRetriesAfterWriteFailure() { + FailOnceAfterInitialCreateStorage storage = + new FailOnceAfterInitialCreateStorage((FileSystem) metaClient.getStorage().getFileSystem()); + HoodieHeartbeatClient hoodieHeartbeatClient = + new HoodieHeartbeatClient(storage, metaClient.getBasePath().toString(), heartBeatInterval, 10); + try { + hoodieHeartbeatClient.start(instantTime1); + await().atMost(10, SECONDS).until(storage::hasInjectedFailure); + await().atMost(10, SECONDS) + .until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() >= 2); + } finally { + hoodieHeartbeatClient.close(); + } + } + + /** + * A storage wrapper whose first {@code create()} call blocks until released, simulating a hung + * storage write. All subsequent calls delegate normally. + */ + private static class SlowCreateStorage extends HoodieHadoopStorage { + + private final AtomicBoolean firstCall = new AtomicBoolean(true); + private final CountDownLatch releaseFirstWrite; + + SlowCreateStorage(FileSystem fs, CountDownLatch releaseFirstWrite) { + super(fs); + this.releaseFirstWrite = releaseFirstWrite; + } + + @Override + public OutputStream create(StoragePath path, boolean overwrite) throws IOException { + if (firstCall.getAndSet(false)) { + try { + releaseFirstWrite.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while simulating a hung heartbeat write", e); + } + } + return super.create(path, overwrite); + } + } + + private static class FailOnceAfterInitialCreateStorage extends HoodieHadoopStorage { + + private final AtomicInteger createCalls = new AtomicInteger(0); + private final AtomicBoolean injectedFailure = new AtomicBoolean(false); + + FailOnceAfterInitialCreateStorage(FileSystem fs) { + super(fs); + } + + @Override + public OutputStream create(StoragePath path, boolean overwrite) throws IOException { + int currentCall = createCalls.incrementAndGet(); + if (currentCall == 2 && injectedFailure.compareAndSet(false, true)) { + throw new IOException("Injected scheduled heartbeat write failure"); + } + return super.create(path, overwrite); + } + + private boolean hasInjectedFailure() { + return injectedFailure.get(); + } + } } From d42b416c78fdc589c8a72e24c31c9ccef2b8ed97 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Wed, 24 Jun 2026 12:19:40 +0800 Subject: [PATCH 094/255] =?UTF-8?q?perf:=20Add=20dedicated=20batch=20size?= =?UTF-8?q?=20config=20for=20LSM=20timeline=20migration=20on=20u=E2=80=A6?= =?UTF-8?q?=20(#19052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: Add dedicated batch size config for LSM timeline migration on upgrade (cherry picked from commit cf314f1c3c75faf0863489f63bc2fc803080d4b2) --- .../hudi/config/HoodieArchivalConfig.java | 14 ++++ .../apache/hudi/config/HoodieWriteConfig.java | 4 + .../upgrade/SevenToEightUpgradeHandler.java | 13 ++-- .../TestSevenToEightUpgradeHandler.java | 74 +++++++++++++++++++ .../hudi/utils/TestFlinkWriteClients.java | 13 ++++ 5 files changed, 113 insertions(+), 5 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java index 8854c87edeaba..e97e268fa9f9f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java @@ -88,6 +88,15 @@ public class HoodieArchivalConfig extends HoodieConfig { .withDocumentation("Archiving of instants is batched in best-effort manner, to pack more instants into a single" + " archive log. This config controls such archival batch size."); + public static final ConfigProperty MIGRATION_COMMITS_ARCHIVAL_BATCH_SIZE = ConfigProperty + .key("hoodie.timeline.migration.commits.archival.batch") + .defaultValue(500) + .markAdvanced() + .withDocumentation("Batch size used when migrating the legacy archived timeline to the LSM timeline during a" + + " table version upgrade. A larger batch size minimizes the number of parquet files (and the associated" + + " remote storage operations like exists check, parquet write and manifest update) created during the" + + " one-time migration, which significantly reduces the total migration time."); + public static final ConfigProperty TIMELINE_COMPACTION_BATCH_SIZE = ConfigProperty .key("hoodie.timeline.compaction.batch.size") .defaultValue(10) @@ -211,6 +220,11 @@ public HoodieArchivalConfig.Builder withCommitsArchivalBatchSize(int batchSize) return this; } + public HoodieArchivalConfig.Builder withMigrationCommitsArchivalBatchSize(int batchSize) { + archivalConfig.setValue(MIGRATION_COMMITS_ARCHIVAL_BATCH_SIZE, String.valueOf(batchSize)); + return this; + } + public Builder withArchiveBeyondSavepoint(boolean archiveBeyondSavepoint) { archivalConfig.setValue(ARCHIVE_BEYOND_SAVEPOINT, String.valueOf(archiveBeyondSavepoint)); return this; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 1ba577faabf12..f349f744a6ccc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -2021,6 +2021,10 @@ public int getCommitArchivalBatchSize() { return getInt(HoodieArchivalConfig.COMMITS_ARCHIVAL_BATCH_SIZE); } + public int getMigrationCommitArchivalBatchSize() { + return getInt(HoodieArchivalConfig.MIGRATION_COMMITS_ARCHIVAL_BATCH_SIZE); + } + public boolean shouldBlockArchivalOnCleanECTR() { return getBoolean(HoodieArchivalConfig.BLOCK_ARCHIVAL_ON_LATEST_CLEAN_ECTR); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java index 7f7338e3da30b..533e443539929 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java @@ -137,7 +137,7 @@ && isComplexKeyGeneratorWithSingleRecordKeyField(tableConfig)) { }, instants.size()); } - upgradeToLSMTimeline(table, context, config); + upgradeToLSMTimeline(table, config); return new UpgradeDowngrade.TableConfigChangeSet(tablePropsToAdd, Collections.emptySet()); } @@ -249,7 +249,7 @@ static void upgradeKeyGeneratorType(HoodieTableConfig tableConfig, Map ValidationUtils.checkState(TimelineLayoutVersion.LAYOUT_VERSION_1.equals(timelineLayoutVersion), "Upgrade to LSM timeline is only supported for layout version 1. Given version: " + timelineLayoutVersion)); @@ -257,7 +257,12 @@ static void upgradeToLSMTimeline(HoodieTable table, HoodieEngineContext engineCo LegacyArchivedMetaEntryReader reader = new LegacyArchivedMetaEntryReader(table.getMetaClient()); StoragePath archivePath = new StoragePath(table.getMetaClient().getMetaPath(), "timeline/history"); LSMTimelineWriter lsmTimelineWriter = LSMTimelineWriter.getInstance(config, table, Option.of(archivePath)); - int batchSize = config.getCommitArchivalBatchSize(); + // Use a dedicated, larger batch size for the one-time migration to minimize the number of parquet + // files created on remote storage. Each write() call involves multiple remote storage operations + // (exists check, parquet write, manifest update); the regular archival batch size is much smaller + // than what migration needs, so with hundreds of actions it creates excessive I/O that + // significantly increases the migration time. + int batchSize = config.getMigrationCommitArchivalBatchSize(); List activeActionsBatch = new ArrayList<>(batchSize); try (ClosableIterator iterator = reader.getActiveActionsIterator()) { while (iterator.hasNext()) { @@ -265,7 +270,6 @@ static void upgradeToLSMTimeline(HoodieTable table, HoodieEngineContext engineCo // If the batch is full, write it to the LSM timeline if (activeActionsBatch.size() == batchSize) { lsmTimelineWriter.write(new ArrayList<>(activeActionsBatch), Option.empty(), Option.empty()); - lsmTimelineWriter.compactAndClean(engineContext); activeActionsBatch.clear(); } } @@ -273,7 +277,6 @@ static void upgradeToLSMTimeline(HoodieTable table, HoodieEngineContext engineCo // Write any remaining actions in the final batch if (!activeActionsBatch.isEmpty()) { lsmTimelineWriter.write(new ArrayList<>(activeActionsBatch), Option.empty(), Option.empty()); - lsmTimelineWriter.compactAndClean(engineContext); } } } catch (Exception e) { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestSevenToEightUpgradeHandler.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestSevenToEightUpgradeHandler.java index 66d03811b76fc..400f37ba768cc 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestSevenToEightUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestSevenToEightUpgradeHandler.java @@ -19,14 +19,22 @@ package org.apache.hudi.table.upgrade; +import org.apache.hudi.client.timeline.versioning.v2.LSMTimelineWriter; +import org.apache.hudi.client.utils.LegacyArchivedMetaEntryReader; import org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.ActiveAction; +import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.table.HoodieTable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -34,10 +42,14 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.apache.hudi.common.table.HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME; @@ -52,8 +64,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.isA; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -153,6 +172,61 @@ void testUpgradeMergeMode(String payloadClass, String preCombineField, String ex } } + @Test + void testUpgradeToLSMTimelineSingleBatch() throws Exception { + // A single batch large enough to hold all actions should result in exactly one write() call, + // proving the migration batch size config (not the regular archival batch size) drives batching. + LSMTimelineWriter writer = runMigration(500, 4); + verify(writer, times(1)).write(any(), any(), any()); + verify(writer, never()).compactAndClean(any()); + } + + @Test + void testUpgradeToLSMTimelineBatchesByMigrationBatchSize() throws Exception { + // With more actions than the migration batch size, the in-loop batching branch must fire: + // 4 actions with a batch size of 2 -> [2, 2] -> 2 write() calls. This pins that the configured + // migration batch size (not just "all actions in one batch") actually governs batching. + LSMTimelineWriter writer = runMigration(2, 4); + verify(writer, times(2)).write(any(), any(), any()); + verify(writer, never()).compactAndClean(any()); + } + + /** + * Runs {@link SevenToEightUpgradeHandler#upgradeToLSMTimeline} with the given migration batch size + * over the given number of archived actions, and returns the (mocked) LSM timeline writer for verification. + */ + private LSMTimelineWriter runMigration(int migrationBatchSize, int totalActions) { + HoodieTable table = mock(HoodieTable.class); + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + + when(table.getMetaClient()).thenReturn(metaClient); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(tableConfig.getTimelineLayoutVersion()).thenReturn(Option.of(TimelineLayoutVersion.LAYOUT_VERSION_1)); + when(metaClient.getMetaPath()).thenReturn(new StoragePath("/tmp/.hoodie")); + when(config.getMigrationCommitArchivalBatchSize()).thenReturn(migrationBatchSize); + // The regular archival batch size must not be consulted during migration. + lenient().when(config.getCommitArchivalBatchSize()).thenReturn(1); + + List actions = new ArrayList<>(); + for (int i = 0; i < totalActions; i++) { + actions.add(mock(ActiveAction.class)); + } + + LSMTimelineWriter writer = mock(LSMTimelineWriter.class); + try (MockedStatic mockedWriterStatic = mockStatic(LSMTimelineWriter.class); + MockedConstruction mockedReader = Mockito.mockConstruction( + LegacyArchivedMetaEntryReader.class, + (readerMock, ctx) -> when(readerMock.getActiveActionsIterator()) + .thenReturn(ClosableIterator.wrap(actions.iterator())))) { + mockedWriterStatic.when(() -> LSMTimelineWriter.getInstance( + any(HoodieWriteConfig.class), any(HoodieTable.class), any(Option.class))).thenReturn(writer); + + SevenToEightUpgradeHandler.upgradeToLSMTimeline(table, config); + } + return writer; + } + private static Map createMap(Object... keyValues) { Map map = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java index 0762bd66838d0..4bc9db0850de7 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.io.FileGroupReaderBasedMergeHandle; @@ -120,6 +121,18 @@ void testUserConfiguredGlobalRecordIndexMinFileGroupCountIsNotOverridden() { assertEquals(12, writeConfig.getGlobalRecordLevelIndexMinFileGroupCount()); } + @Test + void testUserConfiguredMigrationCommitArchivalBatchSizeIsPropagated() { + // A raw hoodie.* property set on the Flink configuration must be propagated to the write config + // (and therefore reach the upgrade handler that reads it during the v7 -> v8 LSM timeline migration). + conf.setString(HoodieArchivalConfig.MIGRATION_COMMITS_ARCHIVAL_BATCH_SIZE.key(), "123"); + HoodieWriteConfig writeConfig = FlinkWriteClients.getHoodieClientConfig(conf, false, false); + assertEquals(123, writeConfig.getMigrationCommitArchivalBatchSize()); + // The regular archival batch size must stay independent at its own default. + assertEquals(Integer.parseInt(HoodieArchivalConfig.COMMITS_ARCHIVAL_BATCH_SIZE.defaultValue()), + writeConfig.getCommitArchivalBatchSize()); + } + @ParameterizedTest @ValueSource(strings = {"", "DIRECT", "TIMELINE_SERVER_BASED"}) void testMarkerType(String markerType) throws Exception { From ff1a69c47ddf10b7dddd4644129219c9da858cb2 Mon Sep 17 00:00:00 2001 From: Surya Prasanna Date: Wed, 24 Jun 2026 08:13:58 -0700 Subject: [PATCH 095/255] test(clean): Cover tests executing pending clean before scheduling a new one (#19051) Enhance cleaner tests Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: sivabalan Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 8c45288dadc4e48e7de89191e7c529b1d6379cd3) --- .../fs/TestHoodieSerializableFileStatus.java | 8 +-- .../TestHoodieClientOnCopyOnWriteStorage.java | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/fs/TestHoodieSerializableFileStatus.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/fs/TestHoodieSerializableFileStatus.java index 61857cb9837c4..7759ee5e2fb25 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/fs/TestHoodieSerializableFileStatus.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/common/fs/TestHoodieSerializableFileStatus.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.fs; -import org.apache.hudi.client.common.HoodieSparkEngineContext; -import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.hadoop.fs.HoodieSerializableFileStatus; import org.apache.hudi.testutils.HoodieSparkClientTestHarness; @@ -45,7 +43,6 @@ @TestInstance(Lifecycle.PER_CLASS) public class TestHoodieSerializableFileStatus extends HoodieSparkClientTestHarness { - HoodieEngineContext engineContext; List testPaths; @BeforeAll @@ -55,7 +52,6 @@ public void setUp() throws IOException { for (int i = 0; i < 5; i++) { testPaths.add(new Path("s3://table-bucket/")); } - engineContext = new HoodieSparkEngineContext(jsc); } @AfterAll @@ -67,7 +63,7 @@ public void tearDown() { public void testNonSerializableFileStatus() { Exception e = Assertions.assertThrows(SparkException.class, () -> { - List statuses = engineContext.flatMap(testPaths, path -> { + List statuses = context.flatMap(testPaths, path -> { FileSystem fileSystem = new NonSerializableFileSystem(); return Arrays.stream(fileSystem.listStatus(path)); }, 5); @@ -78,7 +74,7 @@ public void testNonSerializableFileStatus() { @Test public void testHoodieFileStatusSerialization() { - Assertions.assertDoesNotThrow(() -> engineContext.flatMap(testPaths, path -> { + Assertions.assertDoesNotThrow(() -> context.flatMap(testPaths, path -> { FileSystem fileSystem = new NonSerializableFileSystem(); return Arrays.stream(HoodieSerializableFileStatus.fromFileStatuses(fileSystem.listStatus(path))); }, 5)); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index ffb829165c7b1..0d00c08bda29e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -53,6 +53,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -133,6 +134,7 @@ import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER; import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT; import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED; +import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLUSTERING_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION; @@ -2152,6 +2154,61 @@ public void testRollingMetadataPreservedInCleanCommits() throws Exception { "Schema should be rolled over into new commit even with lookback=1"); } + @Test + public void testExecutingPendingCleanInstantsBeforeSchedulingNewCleanInstant() throws Exception { + Properties props = new Properties(); + props.setProperty("hoodie.clean.automatic", "false"); + HoodieWriteConfig cfg = getConfigBuilder().withProperties(props).build(); + SparkRDDWriteClient client = getHoodieWriteClient(cfg); + + // Bulk insert + String firstCommit = WriteClientTestUtils.createNewInstantTime(); + insertFirstBatch(cfg, client, firstCommit, "000", 100, SparkRDDWriteClient::bulkInsert, + false, false, 100, INSTANT_GENERATOR); + + // First upsert + String secondCommit = WriteClientTestUtils.createNewInstantTime(); + updateBatch(cfg, client, secondCommit, firstCommit, Option.empty(), "000", 100, + SparkRDDWriteClient::upsert, false, true, 100, 100, 2, INSTANT_GENERATOR); + + // Second upsert + String thirdCommit = WriteClientTestUtils.createNewInstantTime(); + updateBatch(cfg, client, thirdCommit, secondCommit, Option.empty(), "000", 100, + SparkRDDWriteClient::upsert, false, true, 100, 100, 3, INSTANT_GENERATOR); + + // Schedule clean operation + Properties cleanProps = new Properties(); + cleanProps.setProperty("hoodie.clean.automatic", "true"); + cleanProps.setProperty("hoodie.clean.commits.retained", "1"); + cleanProps.setProperty("hoodie.clean.multiple.enabled", "false"); + HoodieWriteConfig cleanCfg = getConfigBuilder().withProperties(cleanProps).build(); + SparkRDDWriteClient cleanClient = new SparkRDDWriteClient(context, cleanCfg); + cleanClient.scheduleTableService(Option.empty(), TableServiceType.CLEAN); + + // Verify whether clean operation is scheduled. + Option firstCleanInstant = metaClient.reloadActiveTimeline().lastInstant(); + assertTrue(firstCleanInstant.isPresent()); + assertEquals(CLEAN_ACTION, firstCleanInstant.get().getAction()); + + // Third upsert + String fourthCommit = WriteClientTestUtils.createNewInstantTime(); + updateBatch(cfg, client, fourthCommit, thirdCommit, Option.empty(), "000", 100, + SparkRDDWriteClient::upsert, false, true, 100, 100, 4, INSTANT_GENERATOR); + + // Execute clean operation. This should complete pending clean operation. + cleanClient.clean(); + + // Verify timeline + HoodieTimeline timeline = metaClient.reloadActiveTimeline(); + assertEquals(5, timeline.countInstants()); + assertEquals(0, timeline.filterInflights().countInstants()); + HoodieTimeline cleanTimeline = timeline.filter(instant -> instant.getAction().equals(CLEAN_ACTION)); + assertEquals(1, cleanTimeline.countInstants()); + HoodieInstant cleanInstant = cleanTimeline.getInstants().get(0); + assertTrue(cleanInstant.isCompleted()); + assertEquals(firstCleanInstant.get().requestedTime(), cleanInstant.requestedTime()); + } + /** * Disabling row writer here as clustering tests will throw the error below if it is used. * java.util.concurrent.CompletionException: java.lang.ClassNotFoundException From af0fbfea603a8eecc6b72388c8d5fb160f175d42 Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 25 Jun 2026 19:50:36 +0800 Subject: [PATCH 096/255] test(spark): de-flake TestHoodieClientMultiWriter early-conflict detection (#19069) The early-conflict-detection test's commit-004 path waited heartBeatIntervalForCommit4 * 2 (6s) for the failed commit 003's heartbeat to expire, which assumed the default hoodie.client.heartbeat.tolerable.misses of 2 (expiry window = misses * interval = 2 * 3000 = 6000ms). #18904 raised that default to 10, growing the window to 30000ms, so commit 004 saw 003 as still alive and hit a false early conflict, failing assertDoesNotThrow. Because that failure threw before the un-finally'd TestingServer was closed, and surefire runs the shard with reuseForks=true, the leaked embedded ZooKeeper kept holding its admin server's fixed port 8080, so every later rerun failed with "Unable to instantiate ZookeeperBasedLockProvider" (BindException: Address already in use). Fixes: - Pin heartbeat tolerable misses (constant EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES) in buildWriteConfigForEarlyConflictDetect so the wait no longer depends on the global default, and derive the wait as interval * (misses + 1) for a clear margin. - Disable ZooKeeper's unused embedded admin server (zookeeper.admin.enableServer=false) to remove the fixed-port-8080 collision. - Move the TestingServer into a field closed in @AfterEach so a failing assertion cannot leak it across the reused fork. Closes #19068 (cherry picked from commit 08ba3f7e4e80fa8c55d8fd79e04c9944c6df2d27) --- .../client/TestHoodieClientMultiWriter.java | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index f07414f18af13..bebdf9d88e60d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -137,8 +137,21 @@ @Tag("functional") public class TestHoodieClientMultiWriter extends HoodieClientTestBase { + // Pin the tolerable heartbeat misses used by the early-conflict-detection test so its + // heartbeat-expiry wait does not depend on the global default + // (hoodie.client.heartbeat.tolerable.misses), which changed from 2 to 10 in #18904. + private static final int EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES = 2; + + static { + // ZooKeeper's embedded admin server binds the fixed default port 8080 (Curator only + // randomizes the client port), so a concurrent or leaked TestingServer in a reused + // fork collides with "Address already in use". The admin server is unused here. + System.setProperty("zookeeper.admin.enableServer", "false"); + } + private Properties lockProperties = null; + private TestingServer zkTestingServer = null; /** * super is not thread safe!! @@ -175,6 +188,10 @@ public void setUpMORTestTable() throws IOException { @AfterEach public void clean() throws IOException { + if (zkTestingServer != null) { + zkTestingServer.close(); + zkTestingServer = null; + } cleanupResources(); } @@ -462,14 +479,14 @@ private void testHoodieClientBasicMultiWriterWithEarlyConflictDetection(String t int heartBeatIntervalForCommit4 = 3 * 1000; HoodieWriteConfig writeConfig; - TestingServer server = null; if (earlyConflictDetectionStrategy.equalsIgnoreCase(SimpleTransactionDirectMarkerBasedDetectionStrategy.class.getName())) { // need to setup zk related env there. Bcz SimpleTransactionDirectMarkerBasedDetectionStrategy is only support zk lock for now. - server = new TestingServer(); + // zkTestingServer is closed in @AfterEach so a failing assertion cannot leak it (and its port-8080 admin server). + zkTestingServer = new TestingServer(); Properties properties = new Properties(); properties.setProperty(ZK_BASE_PATH_PROP_KEY, basePath); - properties.setProperty(ZK_CONNECT_URL_PROP_KEY, server.getConnectString()); - properties.setProperty(ZK_BASE_PATH_PROP_KEY, server.getTempDirectory().getAbsolutePath()); + properties.setProperty(ZK_CONNECT_URL_PROP_KEY, zkTestingServer.getConnectString()); + properties.setProperty(ZK_BASE_PATH_PROP_KEY, zkTestingServer.getTempDirectory().getAbsolutePath()); properties.setProperty(ZK_SESSION_TIMEOUT_MS_PROP_KEY, "10000"); properties.setProperty(ZK_CONNECTION_TIMEOUT_MS_PROP_KEY, "10000"); properties.setProperty(ZK_LOCK_KEY_PROP_KEY, "key"); @@ -521,8 +538,11 @@ private void testHoodieClientBasicMultiWriterWithEarlyConflictDetection(String t storage.create(heartbeatFilePath, true); // Wait for heart beat expired for failed commitTime3 "003" - // Otherwise commit4 still can see conflict between failed write 003. - Thread.sleep(heartBeatIntervalForCommit4 * 2); + // Otherwise commit4 still can see conflict between failed write 003. The early-conflict + // check treats 003 as alive until its heartbeat is older than + // (tolerable misses * heartbeat interval); tolerable misses is pinned in + // buildWriteConfigForEarlyConflictDetect, so wait one interval past that window. + Thread.sleep(heartBeatIntervalForCommit4 * (EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES + 1)); final String nextCommitTime4 = "004"; assertDoesNotThrow(() -> { @@ -541,9 +561,6 @@ private void testHoodieClientBasicMultiWriterWithEarlyConflictDetection(String t assertTrue(completedInstant.contains(nextCommitTime4)); FileIOUtils.deleteDirectory(new File(basePath)); - if (server != null) { - server.close(); - } client1.close(); client2.close(); client3.close(); @@ -1771,6 +1788,7 @@ private HoodieWriteConfig buildWriteConfigForEarlyConflictDetect(String markerTy if (markerType.equalsIgnoreCase(MarkerType.DIRECT.name())) { return getConfigBuilder() .withHeartbeatIntervalInMs(60 * 1000) + .withHeartbeatTolerableMisses(EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES) .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder() .withStorageType(FileSystemViewStorageType.MEMORY) .withSecondaryStorageType(FileSystemViewStorageType.MEMORY).build()) @@ -1791,6 +1809,7 @@ private HoodieWriteConfig buildWriteConfigForEarlyConflictDetect(String markerTy return getConfigBuilder() .withStorageConfig(HoodieStorageConfig.newBuilder().parquetMaxFileSize(20 * 1024).build()) .withHeartbeatIntervalInMs(60 * 1000) + .withHeartbeatTolerableMisses(EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES) .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder() .withStorageType(FileSystemViewStorageType.MEMORY) .withSecondaryStorageType(FileSystemViewStorageType.MEMORY).build()) From 3e671a234fa3314191b73dc1b7d87ca77447b74e Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Thu, 25 Jun 2026 19:24:00 -0700 Subject: [PATCH 097/255] fix(test): close TestingServer + write clients on all exit paths in TestHoodieClientMultiWriter (#19062) Test flakiness fix in TestHoodieClientMultiWriter.testHoodieClientBasicMultiWriterWithEarlyConflictDetection. Test-only change. No production code is touched. Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 014e8a281319059f900311f97bf28fdf453bd3c9) --- .../client/TestHoodieClientMultiWriter.java | 121 ++++++++++-------- 1 file changed, 67 insertions(+), 54 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index bebdf9d88e60d..a53f167c2efd4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -85,6 +85,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -500,71 +501,83 @@ private void testHoodieClientBasicMultiWriterWithEarlyConflictDetection(String t writeConfig = buildWriteConfigForEarlyConflictDetect(markerType, properties, InProcessLockProvider.class, earlyConflictDetectionStrategy); } - final SparkRDDWriteClient client1 = getHoodieWriteClient(writeConfig); + SparkRDDWriteClient client1 = null; + SparkRDDWriteClient client2 = null; + SparkRDDWriteClient client3 = null; + SparkRDDWriteClient client4 = null; + try { + client1 = getHoodieWriteClient(writeConfig); - // Create the first commit - final String nextCommitTime1 = "001"; - createCommitWithInserts(writeConfig, client1, "000", nextCommitTime1, 200); + // Create the first commit + final String nextCommitTime1 = "001"; + createCommitWithInserts(writeConfig, client1, "000", nextCommitTime1, 200); - final SparkRDDWriteClient client2 = getHoodieWriteClient(writeConfig); - final SparkRDDWriteClient client3 = getHoodieWriteClient(writeConfig); + client2 = getHoodieWriteClient(writeConfig); + client3 = getHoodieWriteClient(writeConfig); - final String nextCommitTime2 = "002"; + final String nextCommitTime2 = "002"; - // start to write commit 002 - final JavaRDD writeStatusList2 = startCommitForUpdate(writeConfig, client2, nextCommitTime2, 100); + // start to write commit 002 + final SparkRDDWriteClient finalClient2 = client2; + final JavaRDD writeStatusList2 = startCommitForUpdate(writeConfig, client2, nextCommitTime2, 100); - // start to write commit 003 - // this commit 003 will fail quickly because early conflict detection before create marker. - final String nextCommitTime3 = "003"; - assertThrows(SparkException.class, () -> { - final JavaRDD writeStatusList3 = - startCommitForUpdate(writeConfig, client3, nextCommitTime3, 100); - client3.commit(nextCommitTime3, writeStatusList3); - }, "Early conflict detected but cannot resolve conflicts for overlapping writes"); + // start to write commit 003 + // this commit 003 will fail quickly because early conflict detection before create marker. + final String nextCommitTime3 = "003"; + final SparkRDDWriteClient finalClient3 = client3; + assertThrows(SparkException.class, () -> { + final JavaRDD writeStatusList3 = + startCommitForUpdate(writeConfig, finalClient3, nextCommitTime3, 100); + finalClient3.commit(nextCommitTime3, writeStatusList3); + }, "Early conflict detected but cannot resolve conflicts for overlapping writes"); - // start to commit 002 and success - assertDoesNotThrow(() -> { - client2.commit(nextCommitTime2, writeStatusList2); - }); - - HoodieWriteConfig config4 = - HoodieWriteConfig.newBuilder().withProperties(writeConfig.getProps()) - .withHeartbeatIntervalInMs(heartBeatIntervalForCommit4).build(); - final SparkRDDWriteClient client4 = getHoodieWriteClient(config4); - - StoragePath heartbeatFilePath = new StoragePath( - HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + StoragePath.SEPARATOR + nextCommitTime3); - storage.create(heartbeatFilePath, true); + // start to commit 002 and success + assertDoesNotThrow(() -> { + finalClient2.commit(nextCommitTime2, writeStatusList2); + }); - // Wait for heart beat expired for failed commitTime3 "003" - // Otherwise commit4 still can see conflict between failed write 003. The early-conflict - // check treats 003 as alive until its heartbeat is older than - // (tolerable misses * heartbeat interval); tolerable misses is pinned in - // buildWriteConfigForEarlyConflictDetect, so wait one interval past that window. - Thread.sleep(heartBeatIntervalForCommit4 * (EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES + 1)); + HoodieWriteConfig config4 = + HoodieWriteConfig.newBuilder().withProperties(writeConfig.getProps()) + .withHeartbeatIntervalInMs(heartBeatIntervalForCommit4).build(); + client4 = getHoodieWriteClient(config4); - final String nextCommitTime4 = "004"; - assertDoesNotThrow(() -> { - final JavaRDD writeStatusList4 = - startCommitForUpdate(writeConfig, client4, nextCommitTime4, 100); - client4.commit(nextCommitTime4, writeStatusList4); - }); + StoragePath heartbeatFilePath = new StoragePath( + HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + StoragePath.SEPARATOR + nextCommitTime3); + storage.create(heartbeatFilePath, true); - List completedInstant = metaClient.reloadActiveTimeline().getCommitsTimeline() - .filterCompletedInstants().getInstants().stream() - .map(HoodieInstant::requestedTime).collect(Collectors.toList()); + // Wait for heart beat expired for failed commitTime3 "003" + // Otherwise commit4 still can see conflict between failed write 003. The early-conflict + // check treats 003 as alive until its heartbeat is older than + // (tolerable misses * heartbeat interval); tolerable misses is pinned in + // buildWriteConfigForEarlyConflictDetect, so wait one interval past that window. + Thread.sleep(heartBeatIntervalForCommit4 * (EARLY_CONFLICT_HEARTBEAT_TOLERABLE_MISSES + 1)); - assertEquals(3, completedInstant.size()); - assertTrue(completedInstant.contains(nextCommitTime1)); - assertTrue(completedInstant.contains(nextCommitTime2)); - assertTrue(completedInstant.contains(nextCommitTime4)); + final String nextCommitTime4 = "004"; + final SparkRDDWriteClient finalClient4 = client4; + assertDoesNotThrow(() -> { + final JavaRDD writeStatusList4 = + startCommitForUpdate(writeConfig, finalClient4, nextCommitTime4, 100); + finalClient4.commit(nextCommitTime4, writeStatusList4); + }); - FileIOUtils.deleteDirectory(new File(basePath)); - client1.close(); - client2.close(); - client3.close(); - client4.close(); + List completedInstant = metaClient.reloadActiveTimeline().getCommitsTimeline() + .filterCompletedInstants().getInstants().stream() + .map(HoodieInstant::requestedTime).collect(Collectors.toList()); + + assertEquals(3, completedInstant.size()); + assertTrue(completedInstant.contains(nextCommitTime1)); + assertTrue(completedInstant.contains(nextCommitTime2)); + assertTrue(completedInstant.contains(nextCommitTime4)); + + FileIOUtils.deleteDirectory(new File(basePath)); + } finally { + // Close the write clients on every exit path, including failed assertions. + // The TestingServer itself is closed by @AfterEach (see zkTestingServer above). + FileIOUtils.closeQuietly(client1 == null ? null : (Closeable) client1::close); + FileIOUtils.closeQuietly(client2 == null ? null : (Closeable) client2::close); + FileIOUtils.closeQuietly(client3 == null ? null : (Closeable) client3::close); + FileIOUtils.closeQuietly(client4 == null ? null : (Closeable) client4::close); + } } @Test From 1fc97453dcfea11bbf9439719e8c1a2034c1812c Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Thu, 25 Jun 2026 19:58:24 -0700 Subject: [PATCH 098/255] [MINOR] Guard detailed metadata size metrics with a config (#18803) * feat(metadata): guard detailed metadata size metrics behind a config Detailed size metrics in the metadata table cause FSV to be built for MDT in the driver. Add hoodie.metadata.enable.detailed.metrics (default false) to gate emission of these detailed size metrics, reducing driver memory pressure at scale. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit c6d707c6a8bda4a18edab98247ebd03ff1530111) --- .../HoodieBackedTableMetadataWriter.java | 6 ++++- .../action/index/RunIndexActionExecutor.java | 2 +- .../FlinkHoodieBackedTableMetadataWriter.java | 2 +- .../JavaHoodieBackedTableMetadataWriter.java | 2 +- .../testutils/TestHoodieMetadataBase.java | 1 + .../SparkHoodieBackedTableMetadataWriter.java | 2 +- ...kedTableMetadataWriterTableVersionSix.java | 2 +- .../functional/TestHoodieMetadataBase.java | 1 + .../common/config/HoodieMetadataConfig.java | 18 ++++++++++++++ .../hudi/metadata/BaseTableMetadata.java | 3 ++- .../hudi/metadata/HoodieMetadataMetrics.java | 8 ++++++- .../config/TestHoodieMetadataConfig.java | 24 +++++++++++++++++++ 12 files changed, 63 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 4789accc8caa9..df61ed996b1f0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1984,7 +1984,11 @@ protected void commitInternal(String instantTime, Map m.updateSizeMetrics(metadataMetaClient, metadata, dataMetaClient.getTableConfig().getMetadataPartitions())); + metrics.ifPresent(m -> { + if (m.isDetailedMetricsEnabled()) { + m.updateSizeMetrics(metadataMetaClient, metadata, dataMetaClient.getTableConfig().getMetadataPartitions()); + } + }); } protected abstract void bulkInsertAndCommit(BaseHoodieWriteClient writeClient, String instantTime, I preppedRecordInputs, Option bulkInsertPartitioner); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java index 6bb15573c8298..c0a29695bf455 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java @@ -98,7 +98,7 @@ public RunIndexActionExecutor(HoodieEngineContext context, HoodieWriteConfig con super(context, config, table, instantTime); this.txnManager = new TransactionManager(config, table.getStorage()); if (config.getMetadataConfig().isMetricsEnabled()) { - this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage(), config.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java index 268ae942b92e9..8cc44f068df54 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java @@ -98,7 +98,7 @@ public static HoodieTableMetadataWriter create(StorageConfiguration conf, protected void initRegistry() { if (metadataWriteConfig.isMetricsOn()) { // should support executor metrics - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java index 2a035f560d5ca..b5d9d2822f19e 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java @@ -82,7 +82,7 @@ public static HoodieTableMetadataWriter create(StorageConfiguration conf, @Override protected void initRegistry() { if (metadataWriteConfig.isMetricsOn()) { - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java index b555b76964613..792f742ed92a5 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java @@ -308,6 +308,7 @@ protected HoodieWriteConfig.Builder getWriteConfigBuilder(HoodieFailedWritesClea .enable(useFileListingMetadata) .withMetadataIndexColumnStats(false) .enableMetrics(enableMetrics) + .enableDetailedMetadataMetrics(enableMetrics) .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .withMetadataIndexColumnStats(false) // HUDI-8774 .withEngineType(EngineType.JAVA) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java index dfb295d5d16a3..7ea05926768d0 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java @@ -138,7 +138,7 @@ protected void initRegistry() { } else { registry = Registry.getRegistry("HoodieMetadata"); } - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java index 617b4b4e820f9..facd029657496 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java @@ -106,7 +106,7 @@ protected void initRegistry() { } else { registry = Registry.getRegistry("HoodieMetadata"); } - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java index f28de3266c7f0..4a210227bb743 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java @@ -344,6 +344,7 @@ protected HoodieWriteConfig.Builder getWriteConfigBuilder(HoodieFailedWritesClea .withMetadataConfig(HoodieMetadataConfig.newBuilder() .enable(useFileListingMetadata) .enableMetrics(enableMetrics) + .enableDetailedMetadataMetrics(enableMetrics) .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .build()) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index cd4d03ee4ee70..7ce1e447e65f5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -683,6 +683,15 @@ public final class HoodieMetadataConfig extends HoodieConfig { + "with the actual record count stored in the metadata table. This validation runs in a distributed manner " + "using the compute engine. Disabled by default as it adds overhead to the initialization process."); + public static final ConfigProperty ENABLE_DETAILED_METRICS = ConfigProperty + .key(METADATA_PREFIX + ".enable.detailed.metrics") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Enables detailed metadata table metrics — per-metadata-partition file size and base/log " + + "file counts. Emitting these requires building a HoodieTableFileSystemView for the metadata table on " + + "the driver, which adds memory pressure at scale; leave disabled unless you need the breakdown."); + public long getMaxLogFileSize() { return getLong(MAX_LOG_FILE_SIZE_BYTES_PROP); } @@ -1020,6 +1029,10 @@ public boolean isDropMetadataIndex(String indexName) { return subIndexNameToDrop.contains(indexName); } + public boolean isDetailedMetricsEnabled() { + return getBoolean(ENABLE_DETAILED_METRICS); + } + public static class Builder { private EngineType engineType = EngineType.SPARK; @@ -1349,6 +1362,11 @@ public Builder withAutoDeletePartitions(boolean autoDeletePartitions) { return this; } + public Builder enableDetailedMetadataMetrics(boolean enable) { + metadataConfig.setValue(ENABLE_DETAILED_METRICS, String.valueOf(enable)); + return this; + } + public HoodieMetadataConfig build() { metadataConfig.setDefaultValue(ENABLE, getDefaultMetadataEnable(engineType)); metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_COLUMN_STATS, getDefaultColStatsEnable(engineType)); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index f59a257dd7775..c849262d2d94a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -94,7 +94,8 @@ protected BaseTableMetadata(HoodieEngineContext engineContext, if (metadataConfig.isMetricsEnabled()) { this.metrics = Option.of(new HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder() - .fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(), dataMetaClient.getStorage())); + .fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(), dataMetaClient.getStorage(), + metadataConfig.isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java index c7244a30d094c..32f7eca343f36 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java @@ -85,10 +85,12 @@ public class HoodieMetadataMetrics implements Serializable { private final transient MetricRegistry metricsRegistry; private final transient Metrics metrics; + private final boolean detailedMetricsEnabled; - public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) { + public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage, boolean detailedMetricsEnabled) { this.metrics = Metrics.getInstance(metricsConfig, storage); this.metricsRegistry = metrics.getRegistry(); + this.detailedMetricsEnabled = detailedMetricsEnabled; } public Map getStats(boolean detailed, HoodieTableMetaClient metaClient, HoodieTableMetadata metadata, Set metadataPartitions) { @@ -101,6 +103,10 @@ public Map getStats(boolean detailed, HoodieTableMetaClient meta } } + public boolean isDetailedMetricsEnabled() { + return detailedMetricsEnabled; + } + private Map getStats(HoodieTableFileSystemView fsView, boolean detailed, HoodieTableMetadata tableMetadata, Set metadataPartitions) throws IOException { Map stats = new HashMap<>(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java index b66b432abc24d..f3091ed58fe31 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java @@ -269,4 +269,28 @@ void testTableServiceManagerEnabledWithEmptyActionsRejected() { .withTableServiceManagerEnabled(true) .build()); } + + @Test + void testMetricsConfig() { + // Test default value + HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().build(); + assertFalse(config.isMetricsEnabled()); + + Properties props = new Properties(); + props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true); + config = HoodieMetadataConfig.newBuilder() + .fromProperties(props) + .build(); + assertTrue(config.isMetricsEnabled()); + assertFalse(config.isDetailedMetricsEnabled()); + + props = new Properties(); + props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true); + props.put(HoodieMetadataConfig.ENABLE_DETAILED_METRICS.key(), true); + config = HoodieMetadataConfig.newBuilder() + .fromProperties(props) + .build(); + assertTrue(config.isMetricsEnabled()); + assertTrue(config.isDetailedMetricsEnabled()); + } } From 340deedcda6091aa9e4461d22f97eb7eee9cbc5f Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Thu, 25 Jun 2026 20:03:03 -0700 Subject: [PATCH 099/255] test(metadata): Add test coverage for deferred RLI init and bulk_insert (#18865) Follow-up to #18353 (defer RLI init for fresh tables) and #18836 (robust schema resolution during RLI bootstrap). Adds test coverage for the deferred RLI init flow: Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit e4ee7e3023efea1268b0b41510c82430fb6f2fcc) --- .../functional/TestRecordLevelIndex.scala | 116 +++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala index cf63ea9156307..12a6ad0d6da56 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala @@ -146,7 +146,8 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix "Metadata files partition count should be lower than data table file count after rebootstrap") } - def testRecordLevelIndex(tableType: HoodieTableType, streamingWriteEnabled: Boolean, holder: testRecordLevelIndexHolder): Unit = { + def testRecordLevelIndex(tableType: HoodieTableType, streamingWriteEnabled: Boolean, holder: testRecordLevelIndexHolder, + rliInitDeferred: Boolean = false): Unit = { val dataGen = new HoodieTestDataGenerator(); val inserts = dataGen.generateInserts("001", 5) val latestBatchDf = toDataset(spark, inserts) @@ -156,9 +157,10 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix RECORDKEY_FIELD.key -> "_row_key", PARTITIONPATH_FIELD.key -> "data_partition_path", HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", - HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key()-> "false", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true", HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key() -> streamingWriteEnabled.toString, + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() -> rliInitDeferred.toString, HoodieCompactionConfig.INLINE_COMPACT.key() -> "false", HoodieIndexConfig.INDEX_TYPE.key() -> RECORD_LEVEL_INDEX.name()) holder.options = options @@ -167,11 +169,19 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix .mode(SaveMode.Overwrite) .save(basePath) assertEquals(10, spark.read.format("hudi").load(basePath).count()) + if (rliInitDeferred) { + // With defer enabled, the first commit should NOT have initialized the RLI partition. + metaClient = HoodieTableMetaClient.reload(metaClient) + assertFalse(metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath), + "RLI partition should not be initialized after the first commit when defer is enabled") + } val props = TypedProperties.fromMap(JavaConverters.mapAsJavaMapConverter(options).asJava) val writeConfig = HoodieWriteConfig.newBuilder() .withProps(props) .withPath(basePath) .build() + // Constructing the metadata writer here will initialize RLI (lazily, on this second metadata-writer entry) + // when defer is enabled, since there is now 1 completed commit on the data table. var metadata = metadataWriter(writeConfig).getTableMetadata val recordKeys = inserts.asScala.map(i => i.getRecordKey).asJava.stream().collect(Collectors.toList()) holder.recordKeys = recordKeys @@ -301,6 +311,108 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix } } + @ParameterizedTest + @ValueSource(booleans = Array(true, false)) + def testPartitionedRecordLevelIndexDefer(streamingWriteEnabled: Boolean): Unit = { + val holder = new testRecordLevelIndexHolder + testRecordLevelIndex(HoodieTableType.MERGE_ON_READ, streamingWriteEnabled, holder, true) + assertEquals("deltacommit", metaClient.getActiveTimeline.lastInstant().get().getAction) + val writeConfig = getWriteConfig(holder.options) + var metadata = metadataWriter(writeConfig).getTableMetadata + doAllAssertions(holder, metadata) + val writeClient = new SparkRDDWriteClient(new HoodieSparkEngineContext(jsc), writeConfig) + val timeOpt = writeClient.scheduleCompaction(HOption.empty()) + assertTrue(timeOpt.isPresent) + writeClient.compact(timeOpt.get()) + metaClient.reloadActiveTimeline() + assertEquals("compaction", metaClient.getActiveTimeline.lastInstant().get().getAction) + metadata = metadataWriter(writeConfig).getTableMetadata + doAllAssertions(holder, metadata) + writeClient.close() + } + + @ParameterizedTest + @ValueSource(booleans = Array(true, false)) + def testPartitionedRecordLevelIndexDeferWithBulkInsert(streamingWriteEnabled: Boolean): Unit = { + val tableType = HoodieTableType.MERGE_ON_READ + val dataGen = new HoodieTestDataGenerator() + val inserts1 = dataGen.generateInserts("001", 5) + val batch1Df = toDataset(spark, inserts1) + val insertDf1 = batch1Df.withColumn("data_partition_path", lit("partition1")) + .union(batch1Df.withColumn("data_partition_path", lit("partition2"))) + + val options = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "data_partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true", + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key() -> streamingWriteEnabled.toString, + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key() -> "false", + HoodieIndexConfig.INDEX_TYPE.key() -> RECORD_LEVEL_INDEX.name()) + + // Commit #1: bulk_insert on a fresh table with defer enabled. + insertDf1.write.format("hudi") + .options(options) + .option(DataSourceWriteOptions.OPERATION.key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Overwrite) + .save(basePath) + assertEquals(10, spark.read.format("hudi").load(basePath).count()) + + // Defer should have kicked in: RLI partition is not initialized after the first bulk_insert. + metaClient = HoodieTableMetaClient.reload(metaClient) + assertFalse(metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath), + "RLI partition should not be initialized after the first bulk_insert when defer is enabled") + + // Commit #2: another bulk_insert into a new partition. New rows must use distinct record keys. + val inserts2 = dataGen.generateInserts("002", 5) + val batch2Df = toDataset(spark, inserts2) + val insertDf2 = batch2Df.withColumn("data_partition_path", lit("partition3")) + + insertDf2.write.format("hudi") + .options(options) + .option(DataSourceWriteOptions.OPERATION.key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + assertEquals(15, spark.read.format("hudi").load(basePath).count()) + + // Build metadata writer/reader; this entry will initialize RLI now that there is a completed commit. + val writeConfig = getWriteConfig(options) + val metadata = metadataWriter(writeConfig).getTableMetadata + + // RLI partition should now be present in the metadata table. + metaClient = HoodieTableMetaClient.reload(metaClient) + assertTrue(metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath), + "RLI partition should be initialized once a completed commit exists on the data table") + assertTrue(HoodieRecordIndex.isPartitioned( + metaClient.getIndexMetadata.get().getIndexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)), + "RLI should be initialized as partitioned RLI") + + // Validate record key -> location mapping for both batches against the data. + val tableRows = spark.read.format("hudi").load(basePath).collect() + + val batch1Keys = inserts1.asScala.map(_.getRecordKey).asJava.stream().collect(Collectors.toList()) + val partition1Locations = readRecordIndex(metadata, batch1Keys, HOption.of("partition1")) + assertEquals(5, partition1Locations.size) + validateDFWithLocations(tableRows, partition1Locations, "partition1") + val partition2Locations = readRecordIndex(metadata, batch1Keys, HOption.of("partition2")) + assertEquals(5, partition2Locations.size) + validateDFWithLocations(tableRows, partition2Locations, "partition2") + + val batch2Keys = inserts2.asScala.map(_.getRecordKey).asJava.stream().collect(Collectors.toList()) + val partition3Locations = readRecordIndex(metadata, batch2Keys, HOption.of("partition3")) + assertEquals(5, partition3Locations.size) + validateDFWithLocations(tableRows, partition3Locations, "partition3") + + // Cross-partition lookups for batch1 keys against partition3 (and vice versa) should be empty. + assertEquals(0, readRecordIndex(metadata, batch1Keys, HOption.of("partition3")).size) + assertEquals(0, readRecordIndex(metadata, batch2Keys, HOption.of("partition1")).size) + assertEquals(0, readRecordIndex(metadata, batch2Keys, HOption.of("partition2")).size) + } + @ParameterizedTest @ValueSource(booleans = Array(true, false)) def testPartitionedRecordLevelIndexCompact(streamingWriteEnabled: Boolean): Unit = { From 2396fb08d92fce60638f4eb9e31907eacf499afe Mon Sep 17 00:00:00 2001 From: Aditya Goenka <63430370+ad1happy2go@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:33:53 +0530 Subject: [PATCH 100/255] fix(spark): support consistent hashing clustering on non-partitioned tables (#18968) * fix(spark): support consistent hashing clustering on non-partitioned tables Consistent hashing clustering failed on non-partitioned tables because SingleSparkJobConsistentHashingExecutionStrategy rejected the empty ("") partition path with a "not null or empty" guard. Relax the guard to only reject null, since an empty partition path is valid for non-partitioned tables. Add a parameterized test (testResizingNonPartitioned) covering split and merge resizing on a non-partitioned table for both the single-job and multi-job execution strategies. Co-Authored-By: Claude Opus 4.8 * test(spark): merge non-partitioned resizing into testResizing per review Address review feedback on PR #18968: - Fold testResizingNonPartitioned into testResizing via a nonPartitioned parameter and resizingConfigParams() cross-product, since the two tests were identical apart from the partition dimension. - Update the stale HUDI-18161 reference to GitHub issue #18161. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 (cherry picked from commit 245f47f50352e826576814ea08fa68ac0a088c84) --- ...JobConsistentHashingExecutionStrategy.java | 6 ++-- .../TestSparkConsistentBucketClustering.java | 36 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java index 33b4a8d08ec5b..112273387b81b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java @@ -114,7 +114,8 @@ private List performBucketMergeForGroup(ReaderContextFactory rea Option> extraMetadata = clusteringGroup.getExtraMetadata(); ValidationUtils.checkArgument(extraMetadata.isPresent(), "Extra metadata should be present for consistent hashing operations"); String partition = extraMetadata.get().get(BaseConsistentHashingBucketClusteringPlanStrategy.METADATA_PARTITION_KEY); - ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(partition), "Partition should not be null or empty"); + // Note: partition can be an empty string for non-partitioned tables, so only check for null here. + ValidationUtils.checkArgument(partition != null, "Partition should not be null"); List nodes = decodeConsistentHashingNodes(clusteringGroup); Option newBucket = Option.fromJavaOptional(nodes.stream().filter(node -> node.getTag() == ConsistentHashingNode.NodeTag.REPLACE).findFirst()); ValidationUtils.checkArgument(newBucket.isPresent(), "New bucket should be present for merge operation"); @@ -200,7 +201,8 @@ private List performBucketSplitForGroup(ReaderContextFactory rea Option> extraMetadata = clusteringGroup.getExtraMetadata(); ValidationUtils.checkArgument(extraMetadata.isPresent(), "Extra metadata should be present for consistent hashing operations"); String partition = extraMetadata.get().get(BaseConsistentHashingBucketClusteringPlanStrategy.METADATA_PARTITION_KEY); - ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(partition), "Partition should not be null or empty"); + // Note: partition can be an empty string for non-partitioned tables, so only check for null here. + ValidationUtils.checkArgument(partition != null, "Partition should not be null"); List nodes = decodeConsistentHashingNodes(clusteringGroup); Integer seqNo = Integer.parseInt(extraMetadata.get().get(BaseConsistentHashingBucketClusteringPlanStrategy.METADATA_SEQUENCE_NUMBER_KEY)); HoodieConsistentHashingMetadata metadata = new HoodieConsistentHashingMetadata((short) 0, partition, instantTime, 0, seqNo + 1, Collections.emptyList()); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java index cd14ef61fb7fc..93e01b75ae282 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java @@ -34,6 +34,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; @@ -49,6 +50,7 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bucket.ConsistentBucketIndexUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.table.HoodieTable; @@ -102,6 +104,10 @@ public void setup(int maxFileSize, Map options) throws IOExcepti } public void setup(int maxFileSize, Map options, boolean singleJob) throws IOException { + setup(maxFileSize, options, singleJob, false); + } + + public void setup(int maxFileSize, Map options, boolean singleJob, boolean nonPartitioned) throws IOException { initPath(); initSparkContexts(); initTestDataGenerator(); @@ -109,6 +115,13 @@ public void setup(int maxFileSize, Map options, boolean singleJo Properties props = getPropertiesForKeyGen(true); props.putAll(options); props.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + if (nonPartitioned) { + // Non-partitioned tables produce records with an empty partition path and use the non-partition key generator. + dataGen = new HoodieTestDataGenerator(new String[] {HoodieTestDataGenerator.NO_PARTITION_PATH}); + props.setProperty(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.NON_PARTITION.name()); + props.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), ""); + props.remove(HoodieTableConfig.PARTITION_FIELDS.key()); + } metaClient = HoodieTestUtils.init(storageConf, basePath, HoodieTableType.MERGE_ON_READ, props); config = getConfigBuilder().withProps(props) .withIndexConfig(HoodieIndexConfig.newBuilder().fromProperties(props) @@ -130,16 +143,21 @@ public void tearDown() throws IOException { } /** - * Test resizing with bucket number upper bound and lower bound + * Test resizing with bucket number upper bound and lower bound, on both partitioned and non-partitioned tables. + * + *

    Non-partitioned coverage guards GitHub issue #18161: consistent hashing clustering used to fail on + * non-partitioned tables because the empty partition path was rejected by the execution strategy. For a + * non-partitioned table {@code dataGen.getPartitionPaths()} returns the single empty partition path, so the + * assertions below cover both cases without branching. * * @throws IOException */ @ParameterizedTest - @MethodSource("configParams") - public void testResizing(boolean isSplit, boolean rowWriterEnable, boolean single) throws IOException { + @MethodSource("resizingConfigParams") + public void testResizing(boolean isSplit, boolean rowWriterEnable, boolean single, boolean nonPartitioned) throws IOException { final int maxFileSize = isSplit ? 5120 : 128 * 1024 * 1024; final int targetBucketNum = isSplit ? 14 : 4; - setup(maxFileSize, Collections.emptyMap(), single); + setup(maxFileSize, Collections.emptyMap(), single, nonPartitioned); config.setValue("hoodie.datasource.write.row.writer.enable", String.valueOf(rowWriterEnable)); config.setValue("hoodie.metadata.enable", "false"); writeData(2000, true); @@ -384,6 +402,16 @@ private static Stream configParams() { ); } + // configParams crossed with the partitioned / non-partitioned dimension (isSplit, rowWriterEnable, single, nonPartitioned). + private static Stream resizingConfigParams() { + return configParams().flatMap(args -> { + Object[] a = args.get(); + return Stream.of( + Arguments.of(a[0], a[1], a[2], false), + Arguments.of(a[0], a[1], a[2], true)); + }); + } + private static Stream configParamsForSorting() { return Stream.of( Arguments.of("begin_lat", true), From 50236434dd54be440afc34835f5ae1ef128d7810 Mon Sep 17 00:00:00 2001 From: voonhous Date: Fri, 26 Jun 2026 21:03:46 +0800 Subject: [PATCH 101/255] chore(integ-test): bump trino-jdbc 390 to 481 (#19073) * build(integ-test): bump trino-jdbc 390 to 481 The TrinoQueryNode JDBC client lagged at Trino 390. Bump trino.version to 481 (current release). The integ-test only touches the driver via java.sql + Class.forName, so there is no compile-time coupling; the driver core is Java 11 bytecode, matching the project's java.version floor. Refs #19059 * fix(integ-test-bundle): exclude trino-jdbc aircompressor v3 (Java 25) classes from shade trino-jdbc 481 bundles aircompressor v3 compiled to Java 25 (class file major version 69). maven-shade-plugin 3.5.3 pins ASM 9.7, which cannot read major 69, so the integ-test-bundle shade fails with 'Unsupported class file major version 69'. These v3 classes are unreferenced by the JDBC client (the spooling DecompressionUtils binds the legacy Java 8 lz4/zstd codecs), so filter the whole compress/v3 subtree out of the bundle. The driver and the legacy codecs it actually uses are retained. Refs #19059 (cherry picked from commit 8195fb5d902298b5ba32f9a2e0e546610188e779) --- packaging/hudi-integ-test-bundle/pom.xml | 10 ++++++++++ pom.xml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packaging/hudi-integ-test-bundle/pom.xml b/packaging/hudi-integ-test-bundle/pom.xml index 1e27fe2c731f4..e1e15b7d24d9b 100644 --- a/packaging/hudi-integ-test-bundle/pom.xml +++ b/packaging/hudi-integ-test-bundle/pom.xml @@ -296,6 +296,16 @@ **/*.proto + + + io.trino:trino-jdbc + + io/trino/jdbc/$internal/airlift/compress/v3/** + + diff --git a/pom.xml b/pom.xml index a7b6e797efec1..35ac14994c179 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ 1.10.1 1.11.4 0.273 - 390 + 481 core 4.1.1 1.6.0 From 964d366224fc6feb48d6c01af9b829f96d527566 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Sat, 27 Jun 2026 00:27:48 +0800 Subject: [PATCH 102/255] Fix typo in PartitionTTLStrategyType#getPartitionTTLStrategyClassName (#19076) (cherry picked from commit 95ac693351aeb98f9f50185aae8b077037a42c59) --- .../strategy/PartitionTTLStrategyType.java | 3 +- .../TestPartitionTTLStrategyType.java | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategyType.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategyType.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategyType.java index 5dcf1e8bda38e..6a7ed12b3c059 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategyType.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategyType.java @@ -19,7 +19,6 @@ package org.apache.hudi.table.action.ttl.strategy; import org.apache.hudi.common.config.HoodieConfig; -import org.apache.hudi.keygen.constant.KeyGeneratorType; import lombok.Getter; @@ -67,7 +66,7 @@ public static String getPartitionTTLStrategyClassName(HoodieConfig config) { if (config.contains(PARTITION_TTL_STRATEGY_CLASS_NAME)) { return config.getString(PARTITION_TTL_STRATEGY_CLASS_NAME); } else if (config.contains(PARTITION_TTL_STRATEGY_TYPE)) { - return KeyGeneratorType.valueOf(config.getString(PARTITION_TTL_STRATEGY_TYPE)).getClassName(); + return PartitionTTLStrategyType.valueOf(config.getString(PARTITION_TTL_STRATEGY_TYPE)).getClassName(); } return null; } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategyType.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategyType.java new file mode 100644 index 0000000000000..276273cb8953e --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategyType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.ttl.strategy; + +import org.apache.hudi.common.config.HoodieConfig; +import org.apache.hudi.config.HoodieTTLConfig; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link PartitionTTLStrategyType}. + */ +public class TestPartitionTTLStrategyType { + + @Test + public void resolvesKeepByTimeFromType() { + HoodieConfig config = new HoodieConfig(); + config.setValue(HoodieTTLConfig.PARTITION_TTL_STRATEGY_TYPE, + PartitionTTLStrategyType.KEEP_BY_TIME.name()); + + assertEquals(PartitionTTLStrategyType.KEEP_BY_TIME.getClassName(), + PartitionTTLStrategyType.getPartitionTTLStrategyClassName(config)); + } + + @Test + public void resolvesKeepByCreationTimeFromType() { + HoodieConfig config = new HoodieConfig(); + config.setValue(HoodieTTLConfig.PARTITION_TTL_STRATEGY_TYPE, + PartitionTTLStrategyType.KEEP_BY_CREATION_TIME.name()); + + assertEquals(PartitionTTLStrategyType.KEEP_BY_CREATION_TIME.getClassName(), + PartitionTTLStrategyType.getPartitionTTLStrategyClassName(config)); + } + + @Test + public void throwsOnUnknownType() { + HoodieConfig config = new HoodieConfig(); + config.setValue(HoodieTTLConfig.PARTITION_TTL_STRATEGY_TYPE, "NOT_A_REAL_TYPE"); + + assertThrows(IllegalArgumentException.class, + () -> PartitionTTLStrategyType.getPartitionTTLStrategyClassName(config)); + } +} From e2467a4f57e86d2d3145f635dc359da476d59c49 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Fri, 26 Jun 2026 16:46:07 -0700 Subject: [PATCH 103/255] fix: Skip missing properties files gracefully in DFSPropertiesConfiguration (#18805) * fix: Skip missing properties files gracefully in DFSPropertiesConfiguration DFSPropertiesConfiguration.addPropsFromFile only checked file existence when the path equalled DEFAULT_PATH. For any other path (e.g. an include= directive pointing at a non-existent file), the writer would proceed to open() and surface a FileNotFoundException, breaking the load even though other includes and inline properties were valid. Remove the DEFAULT_PATH gate so a missing file is logged and skipped regardless of which path triggered the call. The behavior already documented in the existing log message ("Properties file ... not found. Ignoring to load props file") now applies uniformly. --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit aafbb638e3be6014ff710c03c3ff7869954007fb) --- .../config/DFSPropertiesConfiguration.java | 44 ++++++++-- .../util/TestDFSPropertiesConfiguration.java | 83 +++++++++++++++++++ 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java index 999cfdfed2afb..daabcf41dbb00 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java @@ -118,15 +118,18 @@ public static TypedProperties loadGlobalProps() { String.format("Failed to read %s from class loader", DEFAULT_PROPERTIES_FILE), ioe); } } - // Try loading the external config file from local file system + // Try loading the external config file from local file system. Both DEFAULT_PATH and + // HUDI_CONF_DIR are optional global config locations — use the tolerant overload so a + // missing file does not propagate as an exception (preserves prior silent-ignore behavior + // for optional global-defaults paths). try { - conf.addPropsFromFile(DEFAULT_PATH); + conf.addPropsFromFile(DEFAULT_PATH, true); } catch (Exception e) { log.warn("Cannot load default config file: {}", DEFAULT_PATH, e); } Option defaultConfPath = getConfPathFromEnv(); if (defaultConfPath.isPresent() && !defaultConfPath.get().equals(DEFAULT_PATH)) { - conf.addPropsFromFile(defaultConfPath.get()); + conf.addPropsFromFile(defaultConfPath.get(), true); } return conf.getProps(); } @@ -141,11 +144,31 @@ public static void clearGlobalProps() { } /** - * Add properties from external configuration files. + * Add properties from an external configuration file. A missing file is tolerated only when the + * caller's path equals {@link #DEFAULT_PATH} (the optional global {@code hudi-defaults.conf}); any + * other missing file fails fast with {@link HoodieIOException}, so a typo in an explicit + * user-supplied path (e.g. {@code --props /path/...}) is surfaced rather than silently loading + * empty properties. + * + *

    For include-resolved paths use {@link #addPropsFromFile(StoragePath, boolean)} with + * {@code tolerateMissing=true}. * * @param filePath file path for configuration file. */ public void addPropsFromFile(StoragePath filePath) { + addPropsFromFile(filePath, filePath.equals(DEFAULT_PATH)); + } + + /** + * Add properties from an external configuration file. + * + * @param filePath file path for configuration file. + * @param tolerateMissing when {@code true}, a missing file is logged at {@code debug} and + * ignored (used for optional global-defaults paths and {@code include=} + * recursion). When {@code false}, missing files raise + * {@link HoodieIOException} so explicit user-supplied paths fail fast. + */ + void addPropsFromFile(StoragePath filePath, boolean tolerateMissing) { if (visitedFilePaths.contains(filePath.toString())) { throw new IllegalStateException("Loop detected; file " + filePath + " already referenced"); } @@ -156,9 +179,12 @@ public void addPropsFromFile(StoragePath filePath) { ); try { - if (filePath.equals(DEFAULT_PATH) && !storage.exists(filePath)) { - log.debug("Properties file {} not found. Ignoring to load props file", filePath); - return; + if (!storage.exists(filePath)) { + if (tolerateMissing) { + log.debug("Properties file {} not found. Ignoring to load props file", filePath); + return; + } + throw new HoodieIOException("Properties file does not exist: " + filePath); } } catch (IOException ioe) { throw new HoodieIOException("Cannot check if the properties file exist: " + filePath, ioe); @@ -195,7 +221,9 @@ public void addPropsFromStream(BufferedReader reader, StoragePath cfgFilePath) t && cfgFilePath != null) { providedPath = new StoragePath(cfgFilePath.getParent(), split[1]); } - addPropsFromFile(providedPath); + // include= references may legitimately point to optional files (e.g. environment- + // specific overrides); skip silently when missing rather than failing the whole load. + addPropsFromFile(providedPath, true); } else { hoodieConfig.setValue(split[0], split[1]); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 9e5e45a00778d..fd2b550259858 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -271,4 +271,87 @@ public void testClassInitializationNeverThrows() { TypedProperties props = cfg.getProps(); assertEquals(5, props.size()); } + + @Test + public void testIncludeNonExistentFile() throws IOException { + // Create a properties file that includes a non-existent file + Path filePath = new Path(dfsBasePath + "/t5.props"); + writePropertiesFile(filePath, new String[] { + "existing.prop=value1", + "include=" + dfsBasePath + "/non-existent-file.props", + "another.prop=value2" + }); + + // Should not throw an exception, but log a warning and continue + DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), + new StoragePath(filePath.toUri())); + TypedProperties props = cfg.getProps(); + + // Properties before and after the non-existent include should still be loaded + assertEquals(2, props.size()); + assertEquals("value1", props.getString("existing.prop")); + assertEquals("value2", props.getString("another.prop")); + } + + @Test + public void testIncludeNonExistentRelativeFile() throws IOException { + // Create a properties file that includes a non-existent relative file + Path filePath = new Path(dfsBasePath + "/t6.props"); + writePropertiesFile(filePath, new String[] { + "prop1=val1", + "include=non-existent-relative.props", + "prop2=val2" + }); + + // Should not throw an exception for non-existent relative includes + DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), + new StoragePath(filePath.toUri())); + TypedProperties props = cfg.getProps(); + + // Properties before and after the non-existent include should still be loaded + assertEquals(2, props.size()); + assertEquals("val1", props.getString("prop1")); + assertEquals("val2", props.getString("prop2")); + } + + @Test + public void testMixedExistentAndNonExistentIncludes() throws IOException { + // Create a properties file with both existent and non-existent includes + Path filePath = new Path(dfsBasePath + "/t7.props"); + writePropertiesFile(filePath, new String[] { + "base.prop=base_value", + "include=" + dfsBasePath + "/non-existent-1.props", + "include=" + dfsBasePath + "/t1.props", // This exists + "include=" + dfsBasePath + "/non-existent-2.props", + "override.prop=override_value" + }); + + // Should load successfully, ignoring non-existent files + DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), + new StoragePath(filePath.toUri())); + TypedProperties props = cfg.getProps(); + + // Should have properties from t1.props and the main file + assertEquals("base_value", props.getString("base.prop")); + assertEquals("override_value", props.getString("override.prop")); + assertEquals(123, props.getInteger("int.prop")); // From t1.props + assertEquals("str", props.getString("string.prop")); // From t1.props + assertTrue(props.getBoolean("boolean.prop")); // From t1.props + } + + @Test + public void testExplicitMissingPropertiesFileThrows() { + // Explicit user-supplied paths (e.g. --props /typo.props) should fail fast rather than + // silently load empty properties. Only include= recursion and optional global-defaults + // paths are tolerated. + StoragePath missingPath = new StoragePath(dfsBasePath + "/this-file-does-not-exist.props"); + assertThrows(HoodieIOException.class, + () -> new DFSPropertiesConfiguration(dfs.getConf(), missingPath), + "Constructor should throw when the explicit properties file is missing"); + + DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(); + assertThrows(HoodieIOException.class, + () -> cfg.addPropsFromFile(missingPath), + "Public addPropsFromFile should throw when the explicit properties file is missing"); + } } From e70f8301601699173cf74290610bbdfd67077277 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Fri, 26 Jun 2026 16:49:29 -0700 Subject: [PATCH 104/255] [HUDI-18827] Fix per-task write token for MOR (table v6) rollback log files (#18828) fix(rollback): use per-task write tokens for MOR (table v6) rollback log files RollbackHelperV1 was overriding the per-task write token with the existing log file's token (often UNKNOWN_WRITE_TOKEN), causing retried rollbacks to collide on file name. Keep the per-task token and bump log version to latest + 1 instead. --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 17a76973a34e2430e7adc878888f07abd1d0b6d2) --- .../action/rollback/RollbackHelperV1.java | 35 ++- .../action/rollback/TestRollbackHelper.java | 25 +- .../TestMarkerBasedRollbackStrategy.java | 16 +- ...TestMergeOnReadRollbackActionExecutor.java | 235 ++++++++++++++++++ 4 files changed, 292 insertions(+), 19 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java index 1b288a70e8ba8..b24d85dd9e8d0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java @@ -141,7 +141,12 @@ Map> preComputeLogVersions( } } - Pair sentinel = Pair.of(HoodieLogFile.LOGFILE_BASE_VERSION, HoodieLogFormat.UNKNOWN_WRITE_TOKEN); + // Insert a sentinel for file groups with no existing log files so the caller can skip a + // redundant per-request FS listing. The sentinel uses a null write-token to distinguish + // it from a real entry whose token happens to equal UNKNOWN_WRITE_TOKEN; otherwise tests + // (and any callers that previously wrote logs with the legacy "1-0-1" token) would be + // indistinguishable from "no log file present". + Pair sentinel = Pair.of(HoodieLogFile.LOGFILE_BASE_VERSION, null); for (String expectedKey : expectedKeys) { logVersionMap.putIfAbsent(expectedKey, sentinel); } @@ -324,16 +329,30 @@ List> maybeDeleteAndCollectStats(HoodieEngineCo .withTableVersion(tableVersion) .withFileExtension(HoodieLogFile.DELTA_EXTENSION); - // Supply the pre-computed latest log version and its write token so that - // WriterBuilder.build() skips the per-request FSUtils.getLatestLogVersion() listing. - // This produces the same result: build() would discover (N, T_existing), construct - // path (N, T_existing), find it exists, and roll over to N+1. Pre-computation - // feeds the same (N, T_existing), triggering the identical rollover in getOutputStream(). + // Apply pre-computed log version if available. Always keep the per-task write token + // generated above (via CommonClientUtils.generateWriteToken) so that retried/repeated + // rollbacks do not collide on UNKNOWN_WRITE_TOKEN or inherit a prior log's write token. + // + // When doDelete=true, we actually create a new rollback log file: explicitly bump the + // version (latest + 1) so the new file is written with the per-task write token instead + // of rolling over and inheriting the existing log file's token. When doDelete=false we + // are only collecting stats (no append), so we let WriterBuilder.build() discover the + // existing version itself — bumping here would point to a non-existent path and break + // the downstream storage.getPathInfo lookup. + // + // The sentinel value (right == null) means "partition listed but no log file for this + // file group" — write at the base version. String logVersionKey = logVersionLookupKey(partitionPath, fileId, rollbackRequest.getLatestBaseInstant()); Pair preComputedVersion = logVersionMap.get(logVersionKey); if (preComputedVersion != null) { - writerBuilder.withLogVersion(preComputedVersion.getLeft()) - .withLogWriteToken(preComputedVersion.getRight()); + if (preComputedVersion.getRight() == null) { + writerBuilder.withLogVersion(HoodieLogFile.LOGFILE_BASE_VERSION); + } else if (doDelete) { + writerBuilder.withLogVersion(preComputedVersion.getLeft() + 1); + } else { + writerBuilder.withLogVersion(preComputedVersion.getLeft()) + .withLogWriteToken(preComputedVersion.getRight()); + } } writer = writerBuilder.build(); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java index 15c2421cd6df1..705aec57e66da 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java @@ -31,7 +31,6 @@ import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.testutils.FileCreateUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.collection.Triple; @@ -64,6 +63,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -681,7 +681,9 @@ void testPreComputeLogVersionsSentinelForMissingFileGroups() throws Exception { String missingKey = RollbackHelperV1.logVersionLookupKey(partition, "fileId-no-logs", baseInstant); assertTrue(result.containsKey(missingKey)); assertEquals(HoodieLogFile.LOGFILE_BASE_VERSION, (int) result.get(missingKey).getLeft()); - assertEquals(HoodieLogFormat.UNKNOWN_WRITE_TOKEN, result.get(missingKey).getRight()); + // Sentinel entries (no real log file) carry a null write token so they cannot be confused + // with a real log file that happens to use UNKNOWN_WRITE_TOKEN. + assertNull(result.get(missingKey).getRight()); } @Test @@ -698,10 +700,15 @@ void testV1MaybeDeleteAndCollectStatsWithMultipleRequestsPerFileGroup() throws I ctx.rollbackRequests, true, 5); validateStateAfterRollback(ctx.rollbackRequests); + // Rollback log files are written with a per-task write token from TaskContextSupplier. + // HoodieLocalEngineContext uses LocalTaskContextSupplier which returns 0/0/0 -> token "0-0-0". + String rollbackWriteToken = FSUtils.makeWriteToken(0, 0, 0); StoragePath rollbackLogPath1 = new StoragePath(new StoragePath(basePath, ctx.partition2), - FileCreateUtils.logFileName(ctx.baseInstantTimeOfLogFiles, ctx.logFileId1, 2)); + FSUtils.makeLogFileName(ctx.logFileId1, HoodieLogFile.DELTA_EXTENSION, + ctx.baseInstantTimeOfLogFiles, 2, rollbackWriteToken)); StoragePath rollbackLogPath2 = new StoragePath(new StoragePath(basePath, ctx.partition2), - FileCreateUtils.logFileName(ctx.baseInstantTimeOfLogFiles, ctx.logFileId2, ROLLBACK_LOG_VERSION)); + FSUtils.makeLogFileName(ctx.logFileId2, HoodieLogFile.DELTA_EXTENSION, + ctx.baseInstantTimeOfLogFiles, ROLLBACK_LOG_VERSION, rollbackWriteToken)); List> expected = buildExpectedBaseFileStats(ctx); expected.add(Pair.of(ctx.partition2, @@ -733,8 +740,10 @@ void testV1MaybeDeleteAndCollectStatsWithSingleRequestPerFileGroup() throws IOEx ctx.rollbackRequests, true, 5); validateStateAfterRollback(ctx.rollbackRequests); + String rollbackWriteToken = FSUtils.makeWriteToken(0, 0, 0); StoragePath rollbackLogPath = new StoragePath(new StoragePath(basePath, ctx.partition), - FileCreateUtils.logFileName(ctx.baseInstantTimeOfLogFiles, ctx.logFileId, ROLLBACK_LOG_VERSION)); + FSUtils.makeLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION, + ctx.baseInstantTimeOfLogFiles, ROLLBACK_LOG_VERSION, rollbackWriteToken)); List> expected = new ArrayList<>(); expected.add(Pair.of(ctx.partition, @@ -797,8 +806,12 @@ void testV1MaybeDeleteAndCollectStatsDoDeleteFalseForLogBlocks() throws IOExcept assertTrue(storage.exists(new StoragePath(partitionStoragePath, logFileName))); } + // doDelete=false: no rollback log file is created. The reported path is the existing latest + // log file (the WriterBuilder rediscovers the existing log when we don't explicitly bump the + // version), so it carries the existing log file's write token. StoragePath rollbackLogPath = new StoragePath(partitionStoragePath, - FileCreateUtils.logFileName(ctx.baseInstantTimeOfLogFiles, ctx.logFileId, ctx.logVersionCount)); + FSUtils.makeLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION, + ctx.baseInstantTimeOfLogFiles, ctx.logVersionCount, HoodieLogFormat.UNKNOWN_WRITE_TOKEN)); List> expected = Collections.singletonList( Pair.of(ctx.partition, HoodieRollbackStat.newBuilder() diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java index 182f65100b20b..bd9ddb23bf03a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.HoodieRollbackStat; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.IOType; @@ -43,6 +44,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.rollback.MarkerBasedRollbackStrategy; @@ -399,12 +401,16 @@ void testRollbackMultipleAppendLogFilesInOneFileGroupInMOR(HoodieTableVersion ta assertEquals(1, rollbackStats.size()); HoodieRollbackStat rollbackStat = rollbackStats.get(0); if (!tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT)) { - StoragePath rollbackLogPath = new StoragePath(new StoragePath(basePath, partition), - FileCreateUtils.logFileName(instantTime1, fileId, numLogFiles + 2)); + // The rollback log file's write token is determined by Spark's task context at runtime, + // so extract the actual path from the rollback stats rather than constructing it with a + // hardcoded write token. Verify the file exists and its version matches the expected bump. + StoragePathInfo rollbackLogPathInfo = + rollbackStat.getCommandBlocksCount().entrySet().stream().findFirst().get().getKey(); + StoragePath rollbackLogPath = rollbackLogPathInfo.getPath(); assertTrue(storage.exists(rollbackLogPath)); - assertEquals(rollbackLogPath.getPathWithoutSchemeAndAuthority(), - rollbackStat.getCommandBlocksCount().entrySet().stream().findFirst().get() - .getKey().getPath().getPathWithoutSchemeAndAuthority()); + HoodieLogFile rollbackLogFile = new HoodieLogFile(rollbackLogPathInfo); + assertEquals(fileId, rollbackLogFile.getFileId()); + assertEquals(numLogFiles + 2, rollbackLogFile.getLogVersion()); } assertEquals(partition, rollbackStat.getPartitionPath()); assertEquals( diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java index 7248078bbb889..467531e89a933 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java @@ -21,10 +21,12 @@ import org.apache.hudi.avro.model.HoodieRollbackMetadata; import org.apache.hudi.avro.model.HoodieRollbackPartitionMetadata; import org.apache.hudi.client.SparkRDDWriteClient; +import org.apache.hudi.client.WriteClientTestUtils; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.fs.ConsistencyGuardConfig; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieFileGroup; @@ -32,6 +34,10 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.marker.MarkerType; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; @@ -52,6 +58,12 @@ import org.apache.hudi.testutils.Assertions; import org.apache.hudi.testutils.MetadataMergeWriteStatus; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -62,17 +74,21 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_FILE_NAME_GENERATOR; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestMergeOnReadRollbackActionExecutor extends HoodieClientRollbackTestBase { @@ -487,4 +503,223 @@ public void testRollbackWhenFirstCommitFail() { client.rollback(newCommitTime); } } + + /** + * Tests that rollback operations generate unique write tokens for log files, preventing collisions + * during repeated rollback attempts. + * + *

    This test validates the fix for write token generation in metadata table rollbacks. Previously, + * rollback log files used the default UNKNOWN_WRITE_TOKEN ("1-0-1"), causing collisions when rollback + * was retried. Now, each rollback generates explicit write tokens based on Spark task context + * (format: {partitionId}-{stageId}-{attemptId}). + * + *

    Test flow: + *

      + *
    1. Create initial commit with inserts to establish base files
      2. + *
    2. Create second commit with updates to generate log files (MOR table)
      3. + *
    3. Backup commit timeline files and marker directory for repeated rollback simulation
      4. + *
    4. Execute first rollback and validate write tokens are NOT "1-0-1"
      5. + *
    5. Restore commit state (timeline files + markers) to simulate rollback retry scenario
      6. + *
    6. Execute second rollback and validate unique write tokens prevent collisions
      7. + *
    7. Verify exactly one new rollback log file per file group from second attempt
      8. + *
    + * + * @param enableMetadataTable runs the test both with and without metadata table enabled to + * ensure write-token generation is correct in both code paths + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRollbackWriteTokenGeneration(boolean enableMetadataTable) throws Exception { + // 1. Setup: Create a table-version-6 MOR table so the rollback exercises RollbackHelperV1 + // (which is what this test targets). On v8+ rollbacks delete files directly and don't + // produce rollback log files. + Properties props = new Properties(); + props.put(HoodieTableConfig.VERSION.key(), HoodieTableVersion.SIX.versionCode()); + tearDown(); + initPath(); + initSparkContexts(); + dataGen = new HoodieTestDataGenerator( + new String[] {DEFAULT_FIRST_PARTITION_PATH, DEFAULT_SECOND_PARTITION_PATH}); + initHoodieStorage(); + initMetaClient(HoodieTableType.MERGE_ON_READ, props); + + HoodieWriteConfig cfg = getConfigBuilder() + .withRollbackUsingMarkers(true) + .withMarkersType(MarkerType.DIRECT.name()) + .withWriteTableVersion(HoodieTableVersion.SIX.versionCode()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(enableMetadataTable).build()) + .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(0).build()) + .build(); + + HoodieTestDataGenerator.writePartitionMetadataDeprecated( + storage, new String[] {DEFAULT_FIRST_PARTITION_PATH}, basePath); + FileSystem fs = (FileSystem) storage.getFileSystem(); + SparkRDDWriteClient client = getHoodieWriteClient(cfg); + + // Write 1: Initial inserts + String commitTime1 = "001"; + WriteClientTestUtils.startCommitWithTime(client, commitTime1); + List records = dataGen.generateInsertsForPartition(commitTime1, 100, DEFAULT_FIRST_PARTITION_PATH); + JavaRDD writeRecords = jsc.parallelize(records, 1); + List statusList = client.upsert(writeRecords, commitTime1).collect(); + Assertions.assertNoWriteErrors(statusList); + client.commit(commitTime1, jsc.parallelize(statusList)); + + // Write 2: Updates to same partition to create log files. Use multiple Spark partitions to + // exercise multiple task contexts (so write tokens vary across tasks). + String commitTime2 = "002"; + WriteClientTestUtils.startCommitWithTime(client, commitTime2); + List updateRecords = dataGen.generateUpdates(commitTime2, records); + writeRecords = jsc.parallelize(updateRecords, 2); + statusList = client.upsert(writeRecords, commitTime2).collect(); + Assertions.assertNoWriteErrors(statusList); + // Intentionally leave commit 002 in inflight state so rollback exercises the inflight path. + + HoodieTable table = this.getHoodieTable(metaClient, cfg); + Map> logFileNames = collectLogFileNamesByFileId(fs, DEFAULT_FIRST_PARTITION_PATH); + assertFalse(logFileNames.isEmpty()); + + // Backup commit 002 timeline files + marker dir so the rollback retry below can replay the same input. + Path commit2RequestedPath = new Path(metaClient.getMetaPath().toString(), + commitTime2 + HoodieTimeline.REQUESTED_DELTA_COMMIT_EXTENSION); + Path commit2InflightPath = new Path(metaClient.getMetaPath().toString(), + commitTime2 + HoodieTimeline.INFLIGHT_DELTA_COMMIT_EXTENSION); + Path commit2MarkerDir = new Path(metaClient.getMarkerFolderPath(commitTime2)); + Path backupDir = new Path(basePath, ".backup_test"); + Path backupMarkerDir = new Path(backupDir, commitTime2); + fs.mkdirs(backupDir); + + boolean requestedExists = fs.exists(commit2RequestedPath); + boolean inflightExists = fs.exists(commit2InflightPath); + boolean markerDirExists = fs.exists(commit2MarkerDir); + + if (requestedExists) { + FileUtil.copy(fs, commit2RequestedPath, fs, + new Path(backupDir, commitTime2 + HoodieTimeline.REQUESTED_DELTA_COMMIT_EXTENSION), + false, fs.getConf()); + } + if (inflightExists) { + FileUtil.copy(fs, commit2InflightPath, fs, + new Path(backupDir, commitTime2 + HoodieTimeline.INFLIGHT_DELTA_COMMIT_EXTENSION), + false, fs.getConf()); + } + if (markerDirExists) { + FileUtil.copy(fs, commit2MarkerDir, fs, backupMarkerDir, false, fs.getConf()); + } + + // 3. Rollback commit 002 + String rollbackTime = "003"; + HoodieInstant rollBackInstant = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, commitTime2); + BaseRollbackPlanActionExecutor rollbackPlanExecutor = new BaseRollbackPlanActionExecutor( + context, cfg, table, rollbackTime, rollBackInstant, false, true, false); + rollbackPlanExecutor.execute().get(); + + MergeOnReadRollbackActionExecutor rollbackExecutor = new MergeOnReadRollbackActionExecutor( + context, cfg, table, rollbackTime, rollBackInstant, true, false); + Map rollbackMetadata = rollbackExecutor.execute().getPartitionMetadata(); + + assertEquals(1, rollbackMetadata.size()); + HoodieRollbackPartitionMetadata partitionMetadata = rollbackMetadata.get(DEFAULT_FIRST_PARTITION_PATH); + assertFalse(partitionMetadata.getRollbackLogFiles().isEmpty(), "Should have rollback log files"); + + metaClient = HoodieTableMetaClient.reload(metaClient); + table = this.getHoodieTable(metaClient, cfg); + + // Validate write tokens on the rollback log files are per-task generated (not UNKNOWN_WRITE_TOKEN "1-0-1"). + List rollbackFileSlices = table.getSliceView() + .getLatestFileSlices(DEFAULT_FIRST_PARTITION_PATH) + .collect(Collectors.toList()); + // FileSlice.getLogFiles() is sorted highest-version first (reverse comparator), + // so index 0 is the latest log file produced by the rollback. + List rollbackLogFiles = rollbackFileSlices.stream() + .flatMap(slice -> { + List logFiles = slice.getLogFiles().collect(Collectors.toList()); + return Collections.singleton(logFiles.get(0)).stream(); + }) + .collect(Collectors.toList()); + + assertFalse(rollbackLogFiles.isEmpty(), "Should have rollback log files with rollback instant time"); + for (HoodieLogFile logFile : rollbackLogFiles) { + String writeToken = logFile.getLogWriteToken(); + assertFalse(writeToken.isEmpty(), "Write token should not be empty"); + assertTrue(writeToken.matches("\\d+-\\d+-\\d+"), + String.format("Write token should match pattern partitionId-stageId-attemptId, but got: %s in file: %s", + writeToken, logFile.getFileName())); + assertNotEquals("1-0-1", writeToken); + } + + Map> logFileNamesPostRollback = collectLogFileNamesByFileId(fs, DEFAULT_FIRST_PARTITION_PATH); + + // Simulate rollback retry: remove rollback timeline files and restore commit 002 timeline + markers. + HoodieInstant lastRollbackInstant = metaClient.getActiveTimeline().getRollbackTimeline().lastInstant().get(); + String latestRollbackCompletedFileName = + INSTANT_FILE_NAME_GENERATOR.getFileName(lastRollbackInstant); + fs.delete(new Path(metaClient.getMetaPath().toString(), latestRollbackCompletedFileName), false); + fs.delete(new Path(metaClient.getMetaPath().toString(), + rollbackTime + HoodieTimeline.INFLIGHT_ROLLBACK_EXTENSION), false); + + if (requestedExists) { + FileUtil.copy(fs, new Path(backupDir, commitTime2 + HoodieTimeline.REQUESTED_DELTA_COMMIT_EXTENSION), + fs, commit2RequestedPath, false, fs.getConf()); + } + if (inflightExists) { + FileUtil.copy(fs, new Path(backupDir, commitTime2 + HoodieTimeline.INFLIGHT_DELTA_COMMIT_EXTENSION), + fs, commit2InflightPath, false, fs.getConf()); + } + if (markerDirExists) { + FileUtil.copy(fs, backupMarkerDir, fs, commit2MarkerDir.getParent(), false, fs.getConf()); + } + fs.delete(backupDir, true); + + metaClient = HoodieTableMetaClient.reload(metaClient); + table = this.getHoodieTable(metaClient, cfg); + + // Trigger second rollback - should create additional rollback log files with different write tokens. + MergeOnReadRollbackActionExecutor rollbackExecutor2 = new MergeOnReadRollbackActionExecutor( + context, cfg, table, rollbackTime, rollBackInstant, true, false); + Map rollbackMetadata2 = rollbackExecutor2.execute().getPartitionMetadata(); + + assertEquals(1, rollbackMetadata2.size()); + HoodieRollbackPartitionMetadata partitionMetadata2 = rollbackMetadata2.get(DEFAULT_FIRST_PARTITION_PATH); + assertFalse(partitionMetadata2.getRollbackLogFiles().isEmpty(), "Should have rollback log files"); + + metaClient = HoodieTableMetaClient.reload(metaClient); + + Map> logFileNamesPost2ndRollback = collectLogFileNamesByFileId(fs, DEFAULT_FIRST_PARTITION_PATH); + Map filesFrom2ndRollback = new HashMap<>(); + logFileNamesPost2ndRollback.forEach((fileId, fileNames) -> { + List previousFiles = logFileNamesPostRollback.getOrDefault(fileId, Collections.emptyList()); + for (String fileName : fileNames) { + if (!previousFiles.contains(fileName)) { + filesFrom2ndRollback.merge(fileId, 1, Integer::sum); + assertNotEquals("1-0-1", new HoodieLogFile(fileName).getLogWriteToken()); + } + } + }); + + assertFalse(filesFrom2ndRollback.isEmpty(), + "Second rollback should produce at least one new log file (no collision with first rollback)"); + assertEquals(logFileNames.size(), filesFrom2ndRollback.size()); + filesFrom2ndRollback.forEach((k, v) -> assertEquals(1, v)); + client.close(); + } + + /** + * Lists all log files in the given partition and groups their file names by file ID. + */ + private Map> collectLogFileNamesByFileId(FileSystem fs, String partitionPath) throws IOException { + Map> logFilesByFileId = new HashMap<>(); + RemoteIterator itr = fs.listFiles( + new Path(metaClient.getBasePath().toString() + "/" + partitionPath), false); + while (itr.hasNext()) { + FileStatus fileStatus = itr.next(); + String fileName = fileStatus.getPath().getName(); + if (FSUtils.isLogFile(fileName)) { + String fileId = FSUtils.getFileId(fileName); + logFilesByFileId.computeIfAbsent(fileId, k -> new ArrayList<>()).add(fileName); + } + } + return logFilesByFileId; + } } From 796104c079b58490bab14e584eb5bc32a5492eaa Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Fri, 26 Jun 2026 16:50:15 -0700 Subject: [PATCH 105/255] fix(spark): reject INSERT_OVERWRITE when overlapping with pending clustering (#18829) fix(spark): detect INSERT_OVERWRITE / pending-clustering overlap in update strategy Clustering update strategies only saw record-level updates and missed file groups being wholesale replaced by INSERT_OVERWRITE / INSERT_OVERWRITE_TABLE. Under the default SparkRejectUpdateStrategy the overwrite silently won the race against pending clustering instead of being rejected. Plumb fileGroupsToBeReplaced through UpdateStrategy (3-arg ctor kept for back-compat; loader falls back on NoSuchMethodException). Replace executors populate the set; SparkRejectUpdateStrategy unions it with explicit updates before the overlap check. Extended to the bulk-insert overwrite path via BaseDatasetBulkInsertCommitActionExecutor. Closes #19074 --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit e2cf7216fddf37f49cfdf10861be7aaa31e3feb5) --- .../cluster/strategy/UpdateStrategy.java | 17 +- .../strategy/BaseSparkUpdateStrategy.java | 13 +- .../strategy/SparkAllowUpdateStrategy.java | 11 +- ...nsistentBucketDuplicateUpdateStrategy.java | 8 +- .../strategy/SparkRejectUpdateStrategy.java | 17 +- .../commit/BaseSparkCommitActionExecutor.java | 53 +- ...rkInsertOverwriteCommitActionExecutor.java | 28 +- ...ertOverwriteTableCommitActionExecutor.java | 22 + .../TestInsertOverwriteWithClustering.java | 834 ++++++++++++++++++ .../ConsistentBucketUpdateStrategy.java | 2 +- .../FlinkConsistentBucketUpdateStrategy.java | 2 +- ...DatasetBulkInsertCommitActionExecutor.java | 89 ++ ...lkInsertOverwriteCommitActionExecutor.java | 44 + ...ertOverwriteTableCommitActionExecutor.java | 11 + 14 files changed, 1123 insertions(+), 28 deletions(-) create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestInsertOverwriteWithClustering.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java index 1c61db4b572e5..281aa97acfa57 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java @@ -24,6 +24,7 @@ import org.apache.hudi.table.HoodieTable; import java.io.Serializable; +import java.util.Collections; import java.util.Set; /** @@ -34,11 +35,25 @@ public abstract class UpdateStrategy implements Serializable { protected final transient HoodieEngineContext engineContext; protected HoodieTable table; protected Set fileGroupsInPendingClustering; + protected Set fileGroupsToBeReplaced; - public UpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, Set fileGroupsInPendingClustering) { + public UpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { this.engineContext = engineContext; this.table = table; this.fileGroupsInPendingClustering = fileGroupsInPendingClustering; + this.fileGroupsToBeReplaced = fileGroupsToBeReplaced; + } + + /** + * Backward-compatible 3-arg constructor for custom {@code hoodie.clustering.updates.strategy} + * classes that pre-date the addition of {@code fileGroupsToBeReplaced}. Delegates to the 4-arg + * form with an empty replaced set so the existing reflection lookup keeps working. + */ + public UpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, + Set fileGroupsInPendingClustering) { + this(engineContext, table, fileGroupsInPendingClustering, Collections.emptySet()); } /** diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/BaseSparkUpdateStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/BaseSparkUpdateStrategy.java index 751e2a2858bca..3ff660f7dcb10 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/BaseSparkUpdateStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/BaseSparkUpdateStrategy.java @@ -25,7 +25,7 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.cluster.strategy.UpdateStrategy; -import java.util.List; +import java.util.HashSet; import java.util.Set; /** @@ -35,8 +35,9 @@ public abstract class BaseSparkUpdateStrategy extends UpdateStrategy>> { public BaseSparkUpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, - Set fileGroupsInPendingClustering) { - super(engineContext, table, fileGroupsInPendingClustering); + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + super(engineContext, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); } /** @@ -44,9 +45,9 @@ public BaseSparkUpdateStrategy(HoodieEngineContext engineContext, HoodieTable ta * @param inputRecords the records to write, tagged with target file id * @return the records matched file group ids */ - protected List getGroupIdsWithUpdate(HoodieData> inputRecords) { - return inputRecords + protected Set getGroupIdsWithUpdate(HoodieData> inputRecords) { + return new HashSet<>(inputRecords .filter(record -> record.getCurrentLocation() != null) - .map(record -> new HoodieFileGroupId(record.getPartitionPath(), record.getCurrentLocation().getFileId())).distinct().collectAsList(); + .map(record -> new HoodieFileGroupId(record.getPartitionPath(), record.getCurrentLocation().getFileId())).distinct().collectAsList()); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkAllowUpdateStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkAllowUpdateStrategy.java index 7de85ae977871..7da558a5d5684 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkAllowUpdateStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkAllowUpdateStrategy.java @@ -25,7 +25,6 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.table.HoodieTable; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -35,13 +34,17 @@ public class SparkAllowUpdateStrategy extends BaseSparkUpdateStrategy { public SparkAllowUpdateStrategy( - HoodieEngineContext engineContext, HoodieTable table, Set fileGroupsInPendingClustering) { - super(engineContext, table, fileGroupsInPendingClustering); + HoodieEngineContext engineContext, HoodieTable table, + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + super(engineContext, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); } @Override public Pair>, Set> handleUpdate(HoodieData> taggedRecordsRDD) { - List fileGroupIdsWithRecordUpdate = getGroupIdsWithUpdate(taggedRecordsRDD); + // TODO: also consider fileGroupsToBeReplaced so INSERT_OVERWRITE overlapping with pending + // clustering can be rejected/handled here for users who set SparkAllowUpdateStrategy. + Set fileGroupIdsWithRecordUpdate = getGroupIdsWithUpdate(taggedRecordsRDD); Set fileGroupIdsWithUpdatesAndPendingClustering = fileGroupIdsWithRecordUpdate.stream() .filter(fileGroupsInPendingClustering::contains) .collect(Collectors.toSet()); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java index eb4b308a668af..d63fe860cdacd 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java @@ -49,12 +49,16 @@ */ public class SparkConsistentBucketDuplicateUpdateStrategy extends UpdateStrategy>> { - public SparkConsistentBucketDuplicateUpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, Set fileGroupsInPendingClustering) { - super(engineContext, table, fileGroupsInPendingClustering); + public SparkConsistentBucketDuplicateUpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + super(engineContext, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); } @Override public Pair>, Set> handleUpdate(HoodieData> taggedRecordsRDD) { + // TODO: also consider fileGroupsToBeReplaced so INSERT_OVERWRITE overlapping with pending + // clustering is handled here for the consistent-bucket duplicate-update strategy. if (fileGroupsInPendingClustering.isEmpty()) { return Pair.of(taggedRecordsRDD, Collections.emptySet()); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkRejectUpdateStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkRejectUpdateStrategy.java index 8f943d92fd7d4..0e98747059687 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkRejectUpdateStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkRejectUpdateStrategy.java @@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j; import java.util.Collections; -import java.util.List; import java.util.Set; /** @@ -39,18 +38,22 @@ @Slf4j public class SparkRejectUpdateStrategy extends BaseSparkUpdateStrategy { - public SparkRejectUpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, Set fileGroupsInPendingClustering) { - super(engineContext, table, fileGroupsInPendingClustering); + public SparkRejectUpdateStrategy(HoodieEngineContext engineContext, HoodieTable table, + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + super(engineContext, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); } @Override public Pair>, Set> handleUpdate(HoodieData> taggedRecordsRDD) { - List fileGroupIdsWithRecordUpdate = getGroupIdsWithUpdate(taggedRecordsRDD); - fileGroupIdsWithRecordUpdate.forEach(fileGroupIdWithRecordUpdate -> { - if (fileGroupsInPendingClustering.contains(fileGroupIdWithRecordUpdate)) { + Set allAffectedFileGroups = getGroupIdsWithUpdate(taggedRecordsRDD); + // also treat replaced file groups as potential conflict targets + allAffectedFileGroups.addAll(fileGroupsToBeReplaced); + allAffectedFileGroups.forEach(affectedFileGroup -> { + if (fileGroupsInPendingClustering.contains(affectedFileGroup)) { String msg = String.format("Not allowed to update the clustering file group %s. " + "For pending clustering operations, we are not going to support update for now.", - fileGroupIdWithRecordUpdate.toString()); + affectedFileGroup.toString()); log.error(msg); throw new HoodieClusteringUpdateException(msg); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java index e79ee136e98fd..ed0dcaf321be2 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java @@ -44,6 +44,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.data.HoodieJavaPairRDD; import org.apache.hudi.data.HoodieJavaRDD; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.execution.SparkLazyInsertIterable; import org.apache.hudi.index.HoodieIndex; @@ -121,9 +122,11 @@ protected HoodieData> clusteringHandleUpdate(HoodieData>> updateStrategy = (UpdateStrategy>>) ReflectionUtils - .loadClass(config.getClusteringUpdatesStrategyClass(), new Class[] {HoodieEngineContext.class, HoodieTable.class, Set.class}, - this.context, table, fileGroupsInPendingClustering); + // Get file groups that will be replaced by this operation (for INSERT_OVERWRITE, etc.) + Set fileGroupsToBeReplaced = getFileGroupsBeingReplaced(inputRecords); + + UpdateStrategy>> updateStrategy = + loadClusteringUpdateStrategy(fileGroupsInPendingClustering, fileGroupsToBeReplaced); // For SparkAllowUpdateStrategy with rollback pending clustering as false, need not handle // the file group intersection between current ingestion and pending clustering file groups. // This will be handled at the conflict resolution strategy. @@ -163,6 +166,50 @@ protected HoodieData> clusteringHandleUpdate(HoodieData getFileGroupsBeingReplaced(HoodieData> inputRecords) { + // Default implementation returns empty set. Subclasses should override as needed. + return Collections.emptySet(); + } + + /** + * Loads {@code hoodie.clustering.updates.strategy} via reflection, preferring the new 4-arg + * constructor (with {@code fileGroupsToBeReplaced}) and falling back to the legacy 3-arg + * constructor for custom strategies that pre-date this PR. + */ + @SuppressWarnings("unchecked") + private UpdateStrategy>> loadClusteringUpdateStrategy( + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + String strategyClass = config.getClusteringUpdatesStrategyClass(); + try { + return (UpdateStrategy>>) ReflectionUtils.loadClass( + strategyClass, + new Class[] {HoodieEngineContext.class, HoodieTable.class, Set.class, Set.class}, + this.context, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); + } catch (HoodieException ex) { + if (!(ex.getCause() instanceof NoSuchMethodException)) { + throw ex; + } + // Legacy custom strategies only have the 3-arg constructor. INSERT_OVERWRITE overlap with + // pending clustering will not be detected for these classes (they never see + // fileGroupsToBeReplaced); recommend bumping to the 4-arg constructor. + log.warn("Clustering update strategy {} is missing the 4-arg constructor with " + + "fileGroupsToBeReplaced; falling back to the 3-arg constructor. INSERT_OVERWRITE " + + "overlap with pending clustering will not be detected for this strategy.", strategyClass); + return (UpdateStrategy>>) ReflectionUtils.loadClass( + strategyClass, + new Class[] {HoodieEngineContext.class, HoodieTable.class, Set.class}, + this.context, table, fileGroupsInPendingClustering); + } + } + @Override public HoodieWriteMetadata> execute(HoodieData> inputRecords) { return this.execute(inputRecords, Option.empty()); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java index 6ac976f2e5442..0b699ae540e85 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -41,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class SparkInsertOverwriteCommitActionExecutor @@ -82,10 +84,10 @@ protected String getCommitActionType() { @Override protected Map> getPartitionToReplacedFileIds(HoodieWriteMetadata> writeMetadata) { - String staticOverwritePartition = config.getStringOrDefault(HoodieInternalConfig.STATIC_OVERWRITE_PARTITION_PATHS); - if (StringUtils.nonEmpty(staticOverwritePartition)) { + String staticOverwritePartitionPaths = config.getStringOrDefault(HoodieInternalConfig.STATIC_OVERWRITE_PARTITION_PATHS); + if (StringUtils.nonEmpty(staticOverwritePartitionPaths)) { // static insert overwrite partitions - List partitionPaths = Arrays.asList(staticOverwritePartition.split(",")); + List partitionPaths = Arrays.asList(staticOverwritePartitionPaths.split(",")); context.setJobStatus(this.getClass().getSimpleName(), "Getting ExistingFileIds of matching static partitions"); return HoodieJavaPairRDD.getJavaPairRDD(context.parallelize(partitionPaths, partitionPaths.size()).mapToPair( partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath)))).collectAsMap(); @@ -101,6 +103,26 @@ protected List getAllExistingFileIds(String partitionPath) { return table.getSliceView().getLatestFileSlices(partitionPath).map(FileSlice::getFileId).distinct().collect(Collectors.toList()); } + @Override + protected Set getFileGroupsBeingReplaced(HoodieData> inputRecords) { + String staticOverwritePartitionPaths = config.getStringOrDefault(HoodieInternalConfig.STATIC_OVERWRITE_PARTITION_PATHS); + List partitionPaths; + + if (StringUtils.nonEmpty(staticOverwritePartitionPaths)) { + // Static insert overwrite: use the configured partitions + partitionPaths = Arrays.asList(staticOverwritePartitionPaths.split(",")); + } else { + // Dynamic insert overwrite: determine partitions from input records + partitionPaths = inputRecords.map(HoodieRecord::getPartitionPath).distinct().collectAsList(); + } + + // Get all file groups in the partitions to be overwritten + return partitionPaths.stream() + .flatMap(partitionPath -> getAllExistingFileIds(partitionPath).stream() + .map(fileId -> new HoodieFileGroupId(partitionPath, fileId))) + .collect(Collectors.toSet()); + } + @Override protected Iterator> handleInsertPartition(String instantTime, Integer partition, Iterator recordItr, Broadcast bucketInfoGetter) { BucketInfo binfo = bucketInfoGetter.getValue().getBucketInfo(partition); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java index d300ea683a900..67f1ab3520731 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.collection.Pair; @@ -31,8 +32,10 @@ import org.apache.hudi.table.action.HoodieWriteMetadata; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class SparkInsertOverwriteTableCommitActionExecutor extends SparkInsertOverwriteCommitActionExecutor { @@ -53,4 +56,23 @@ protected Map> getPartitionToReplacedFileIds(HoodieWriteMet return HoodieJavaPairRDD.getJavaPairRDD(context.parallelize(partitionPaths, partitionPaths.size()).mapToPair( partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath)))).collectAsMap(); } + + @Override + protected Set getFileGroupsBeingReplaced(HoodieData> inputRecords) { + // INSERT_OVERWRITE_TABLE replaces every file group across every partition, not just the + // partitions present in the input records. Enumerate all partitions in parallel via the + // engine context (matches the parallelization in getPartitionToReplacedFileIds above and + // avoids a sequential driver-side walk for tables with many partitions whose file system + // view isn't fully cached). + List partitionPaths = FSUtils.getAllPartitionPaths(context, table.getMetaClient(), config.getMetadataConfig()); + if (partitionPaths == null || partitionPaths.isEmpty()) { + return Collections.emptySet(); + } + context.setJobStatus(this.getClass().getSimpleName(), "Resolving file groups being replaced across all partitions"); + return new HashSet<>(context.parallelize(partitionPaths, partitionPaths.size()) + .flatMap(partitionPath -> table.getSliceView().getLatestFileSlices(partitionPath) + .map(fileSlice -> new HoodieFileGroupId(partitionPath, fileSlice.getFileId())) + .iterator()) + .collectAsList()); + } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestInsertOverwriteWithClustering.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestInsertOverwriteWithClustering.java new file mode 100644 index 0000000000000..955a8160c264c --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestInsertOverwriteWithClustering.java @@ -0,0 +1,834 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.commit; + +import org.apache.hudi.client.HoodieWriteResult; +import org.apache.hudi.client.SparkRDDWriteClient; +import org.apache.hudi.client.WriteClientTestUtils; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.clustering.plan.strategy.SparkSingleFileSortPlanStrategy; +import org.apache.hudi.client.clustering.run.strategy.SparkSingleFileSortExecutionStrategy; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.util.ClusteringUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieClusteringConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.data.HoodieJavaRDD; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.table.HoodieSparkTable; +import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestBase; + +import org.apache.spark.api.java.JavaRDD; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for INSERT_OVERWRITE, INSERT_OVERWRITE_TABLE, and DELETE_PARTITION operations + * when there are pending clustering operations on the file groups being replaced. + */ +public class TestInsertOverwriteWithClustering extends HoodieClientTestBase { + + private HoodieTestDataGenerator dataGen; + + @BeforeEach + public void setUp() throws Exception { + initPath(); + initSparkContexts(); + initTestDataGenerator(); + initMetaClient(HoodieTableType.COPY_ON_WRITE); + dataGen = new HoodieTestDataGenerator(); + } + + @AfterEach + public void tearDown() throws Exception { + cleanupResources(); + } + + private HoodieClusteringConfig.Builder baseClusteringConfigBuilder(boolean rollbackPendingClustering) { + return HoodieClusteringConfig.newBuilder() + .withClusteringPlanStrategyClass(SparkSingleFileSortPlanStrategy.class.getName()) + .withClusteringExecutionStrategyClass(SparkSingleFileSortExecutionStrategy.class.getName()) + .withClusteringMaxNumGroups(10) + .withRollbackPendingClustering(rollbackPendingClustering); + } + + private HoodieWriteConfig.Builder getConfigBuilder(boolean rollbackPendingClustering) { + return HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withSchema(TRIP_EXAMPLE_SCHEMA) + .withParallelism(2, 2) + .withBulkInsertParallelism(2) + .withFinalizeWriteParallelism(2) + .withDeleteParallelism(2) + .withRollbackParallelism(2) + .withClusteringConfig(baseClusteringConfigBuilder(rollbackPendingClustering).build()); + } + + private HoodieWriteConfig.Builder getConfigBuilderWithPartitionFilter(boolean rollbackPendingClustering, String partitionFilter) { + return HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withSchema(TRIP_EXAMPLE_SCHEMA) + .withParallelism(2, 2) + .withBulkInsertParallelism(2) + .withFinalizeWriteParallelism(2) + .withDeleteParallelism(2) + .withRollbackParallelism(2) + .withClusteringConfig(baseClusteringConfigBuilder(rollbackPendingClustering) + .withClusteringPartitionSelected(partitionFilter) + .build()); + } + + /** + * Test that INSERT_OVERWRITE operation throws an exception when file groups + * to be replaced conflict with pending clustering. + */ + @Test + public void testStaticInsertOverwriteWithPendingClusteringRejectsUpdate() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Initial insert to create some data + String instant1 = nextInstant(); + List records1 = dataGen.generateInserts(instant1, 100); + JavaRDD writeRecords1 = jsc.parallelize(records1, 2); + commitInsert(client, writeRecords1, instant1); + + // Get partition path from the first record + String partitionPath = records1.get(0).getPartitionPath(); + + // Step 2: Schedule clustering for the partition + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline() + .filterPendingClusteringTimeline(); + assertEquals(1, pendingReplaceTimeline.countInstants()); + + // Get file groups involved in pending clustering + Set pendingClusteringFileGroups = ClusteringUtils + .getAllFileGroupsInPendingClusteringPlans(metaClient).keySet(); + assertFalse(pendingClusteringFileGroups.isEmpty()); + + // Step 3: Perform static INSERT_OVERWRITE on the same partition + // This should throw HoodieUpsertException because file groups to be replaced + // conflict with pending clustering + String instant3 = nextInstant(); + List records3 = dataGen.generateInsertsForPartition(instant3, 50, partitionPath); + JavaRDD writeRecords3 = jsc.parallelize(records3, 2); + + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieUpsertException upsertException = assertThrows(HoodieUpsertException.class, () -> + commitInsertOverwrite(client, writeRecords3, instant3) + ); + assertTrue(upsertException.getCause().getMessage().contains("Not allowed to update the clustering file group")); + + // Verify clustering is still pending (NOT rolled back) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertEquals(1, pendingReplaceTimeline.countInstants(), + "Pending clustering should remain pending after failed INSERT_OVERWRITE"); + assertTrue(pendingReplaceTimeline.containsInstant(clusteringInstant)); + + // Verify the INSERT_OVERWRITE did NOT complete + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertFalse(completedReplaceTimeline.containsInstant(instant3)); + } + + /** + * Test that dynamic INSERT_OVERWRITE_TABLE operation gets aborted when it overlaps w/ pending clustering. + */ + @Test + public void testDynamicInsertOverwriteWithPendingClustering() throws Exception { + HoodieWriteConfig config = getConfigBuilder(true).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Initial insert to create data in multiple partitions + String instant1 = nextInstant(); + List records1 = dataGen.generateInserts(instant1, 100); + JavaRDD writeRecords1 = jsc.parallelize(records1, 2); + commitInsert(client, writeRecords1, instant1); + + // Get partitions that have data + Set partitionsWithData = records1.stream() + .map(HoodieRecord::getPartitionPath) + .collect(Collectors.toSet()); + + // Step 2: Schedule clustering + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants()); + + // Get file groups involved in pending clustering + Set pendingClusteringFileGroups = ClusteringUtils + .getAllFileGroupsInPendingClusteringPlans(metaClient).keySet(); + assertFalse(pendingClusteringFileGroups.isEmpty()); + + // Step 3: Perform dynamic INSERT_OVERWRITE_TABLE on overlapping partitions + String instant3 = nextInstant(); + String targetPartition = partitionsWithData.iterator().next(); + List records3 = dataGen.generateInsertsForPartition(instant3, 50, targetPartition); + JavaRDD writeRecords3 = jsc.parallelize(records3, 2); + HoodieUpsertException upsertException = assertThrows(HoodieUpsertException.class, () -> + commitInsertOverwrite(client, writeRecords3, instant3) + ); + assertTrue(upsertException.getCause().getMessage().contains("Not allowed to update the clustering file group")); + + // Verify clustering was never rolled back. + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertTrue(pendingReplaceTimeline.containsInstant(clusteringInstant), + "Pending clustering should not be rolled back"); + + // Verify the INSERT_OVERWRITE did NOT complete + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertFalse(completedReplaceTimeline.containsInstant(instant3)); + } + + /** + * Test that DELETE_PARTITION operation succeeds when pending clustering does not overlap. + */ + @Test + public void testDeletePartitionWithNonOverlappingPendingClustering() throws Exception { + HoodieWriteConfig config = getConfigBuilderWithPartitionFilter(false, "partition1").build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: insert into partition1 + String instant1 = nextInstant(); + List records1 = dataGen.generateInsertsForPartition(instant1, 100, "partition1"); + JavaRDD writeRecords1 = jsc.parallelize(records1, 2); + commitInsert(client, writeRecords1, instant1); + + String instant2 = nextInstant(); + List records2 = dataGen.generateInsertsForPartition(instant2, 100, "partition2"); + JavaRDD writeRecords2 = jsc.parallelize(records2, 2); + commitInsert(client, writeRecords2, instant2); + + // Step 3: Schedule clustering for the partition1 + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants()); + + // Get file groups involved in pending clustering + Set pendingClusteringFileGroups = ClusteringUtils + .getAllFileGroupsInPendingClusteringPlans(metaClient).keySet(); + assertFalse(pendingClusteringFileGroups.isEmpty()); + + client.close(); + SparkRDDWriteClient client2 = getHoodieWriteClient(config); + + // Step 3: Delete the partition2 + String instant3 = nextInstant(); + commitDeletePartitions(client2, Arrays.asList("partition2"), instant3); + + // Verify clustering is still pending (NOT rolled back) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertTrue(pendingReplaceTimeline.containsInstant(clusteringInstant), + "Pending clustering should remain pending after successful DELETE_PARTITION"); + + // Verify the DELETE_PARTITION is completed + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertTrue(completedReplaceTimeline.containsInstant(instant3)); + } + + /** + * Test getFileGroupsBeingReplaced method in SparkInsertOverwriteCommitActionExecutor + * for static INSERT_OVERWRITE scenario. + */ + @Test + public void testGetFileGroupsBeingReplacedForStaticOverwrite() throws Exception { + HoodieWriteConfig config = getConfigBuilder(true).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Initial insert to create some data + String instant1 = nextInstant(); + String partitionPath = "2023/01/01"; + List records1 = dataGen.generateInsertsForPartition(instant1, 100, partitionPath); + JavaRDD writeRecords1 = jsc.parallelize(records1, 2); + commitInsert(client, writeRecords1, instant1); + + // Step 2: Get the file groups in the partition + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTable table = HoodieSparkTable.create(config, context, metaClient); + List existingFileIds = table.getSliceView() + .getLatestFileSlices(partitionPath) + .map(fileSlice -> fileSlice.getFileId()) + .collect(Collectors.toList()); + assertFalse(existingFileIds.isEmpty(), "Should have at least one file group"); + + // Step 3: Create INSERT_OVERWRITE executor and test getFileGroupsBeingReplaced + String instant2 = nextInstant(); + List records = dataGen.generateInsertsForPartition(instant2, 10, partitionPath); + HoodieData inputRecords = HoodieJavaRDD.of(jsc.parallelize(records, 1)); + + SparkInsertOverwriteCommitActionExecutor executor = + new SparkInsertOverwriteCommitActionExecutor( + context, config, table, instant2, inputRecords); + + // Invoke the method - it's protected so we test it indirectly through the workflow + Set fileGroupsBeingReplaced = executor.getFileGroupsBeingReplaced(inputRecords); + + // Verify that the file groups in the partition are identified as being replaced + assertEquals(existingFileIds.size(), fileGroupsBeingReplaced.size(), + "Should identify all existing file groups in the partition as being replaced"); + assertTrue(fileGroupsBeingReplaced.stream() + .allMatch(fg -> fg.getPartitionPath().equals(partitionPath)), + "All identified file groups should be in the target partition"); + } + + /** + * Test that INSERT_OVERWRITE on a non-overlapping partition succeeds + * even when there is pending clustering on a different partition. + */ + @Test + public void testInsertOverwriteNonOverlappingPartitionWithPendingClustering() throws Exception { + HoodieWriteConfig config = getConfigBuilder(true) + .withClusteringConfig(HoodieClusteringConfig.newBuilder() + .withClusteringPlanStrategyClass(SparkSingleFileSortPlanStrategy.class.getName()) + .withClusteringExecutionStrategyClass(SparkSingleFileSortExecutionStrategy.class.getName()) + .withClusteringMaxNumGroups(10) + .withRollbackPendingClustering(true) + .withClusteringPartitionSelected("partition1") + .build()) + .build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into partition1 + String instant1 = nextInstant(); + List records1 = dataGen.generateInsertsForPartition(instant1, 100, "partition1"); + JavaRDD writeRecords1 = jsc.parallelize(records1, 2); + commitInsert(client, writeRecords1, instant1); + + String instant2 = nextInstant(); + List records2 = dataGen.generateInsertsForPartition(instant1, 100, "partition2"); + JavaRDD writeRecords2 = jsc.parallelize(records2, 2); + commitInsert(client, writeRecords2, instant2); + + // Step 2: Schedule clustering for partition1 + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants()); + + // Step 3: Perform INSERT_OVERWRITE on partition2 (non-overlapping) + String instant3 = nextInstant(); + List records3 = dataGen.generateInsertsForPartition(instant3, 50, "partition2"); + JavaRDD writeRecords3 = jsc.parallelize(records3, 2); + commitInsertOverwrite(client, writeRecords3, instant3); + + // Verify clustering was NOT rolled back (no overlap) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertEquals(1, pendingReplaceTimeline.countInstants(), + "Pending clustering should NOT be rolled back for non-overlapping partition"); + + // Verify the INSERT_OVERWRITE completed successfully + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertTrue(completedReplaceTimeline.containsInstant(instant3)); + } + + /** + * Test dynamic INSERT_OVERWRITE that determines partitions from input records. + * If input records target a partition with pending clustering, it should detect the conflict and abort. + */ + @Test + public void testDynamicInsertOverwriteDetectsOverlap() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into multiple partitions + String instant1 = nextInstant(); + List partition1Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/01"); + List partition2Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/02"); + List partition3Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/03"); + List allRecords = new ArrayList<>(); + allRecords.addAll(partition1Records); + allRecords.addAll(partition2Records); + allRecords.addAll(partition3Records); + JavaRDD writeRecords1 = jsc.parallelize(allRecords, 2); + commitInsert(client, writeRecords1, instant1); + + // Step 2: Schedule clustering + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants()); + + // Get partitions in clustering + Set clusteringFileGroups = ClusteringUtils + .getAllFileGroupsInPendingClusteringPlans(metaClient).keySet(); + Set clusteringPartitions = clusteringFileGroups.stream() + .map(HoodieFileGroupId::getPartitionPath) + .collect(Collectors.toSet()); + assertFalse(clusteringPartitions.isEmpty()); + + // Step 3: Perform dynamic INSERT_OVERWRITE_TABLE with records in overlapping partition + // This should fail + metaClient = HoodieTableMetaClient.reload(this.metaClient); + String instant3 = nextInstant(); + String targetPartition = clusteringPartitions.iterator().next(); + List records3 = dataGen.generateInsertsForPartition(instant3, 50, targetPartition); + JavaRDD writeRecords3 = jsc.parallelize(records3, 2); + HoodieUpsertException upsertException = assertThrows(HoodieUpsertException.class, () -> + commitInsertOverwrite(client, writeRecords3, instant3) + ); + assertTrue(upsertException.getCause().getMessage().contains("Not allowed to update the clustering file group")); + + // Verify clustering is still pending (NOT rolled back) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertEquals(1, pendingReplaceTimeline.countInstants(), + "Pending clustering should remain pending after failed INSERT_OVERWRITE_TABLE"); + assertTrue(pendingReplaceTimeline.containsInstant(clusteringInstant)); + + // Verify the INSERT_OVERWRITE_TABLE did NOT complete + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertFalse(completedReplaceTimeline.containsInstant(instant3)); + } + + /** + * Test multiple concurrent INSERT_OVERWRITE operations on different partitions + * with one partition having pending clustering. + */ + @Test + public void testMultipleInsertOverwriteWithSelectiveOverlap() throws Exception { + HoodieWriteConfig config = getConfigBuilderWithPartitionFilter(false,"2023/01/01,2023/01/02").build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into three partitions + String instant1 = nextInstant(); + List partition1Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/01"); + List partition2Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/02"); + List partition3Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/03"); + List allRecords = new ArrayList<>(); + allRecords.addAll(partition1Records); + allRecords.addAll(partition2Records); + allRecords.addAll(partition3Records); + JavaRDD writeRecords1 = jsc.parallelize(allRecords, 2); + commitInsert(client, writeRecords1, instant1); + + // Step 2: Schedule clustering (will cluster partition1 and partition2) + Option clusteringInstantOpt = client.scheduleClustering(Option.empty()); + assertTrue(clusteringInstantOpt.isPresent(), "Expected clustering to be scheduled but returned empty"); + String clusteringInstant = clusteringInstantOpt.get(); + + // Verify clustering is pending + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants()); + + client.close(); + SparkRDDWriteClient client2 = getHoodieWriteClient(config); + + // Step 3: Perform INSERT_OVERWRITE on partition3 (no overlap) - should succeed without rollback + String instant3 = nextInstant(); + List records3 = dataGen.generateInsertsForPartition(instant3, 50, "2023/01/03"); + JavaRDD writeRecords3 = jsc.parallelize(records3, 2); + commitInsertOverwrite(client2, writeRecords3, instant3); + + // Verify clustering is still pending (no rollback for non-overlapping partition) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + assertEquals(1, metaClient.getActiveTimeline().filterPendingClusteringTimeline().countInstants(), + "Clustering should still be pending after INSERT_OVERWRITE on non-overlapping partition"); + + client2.close(); + SparkRDDWriteClient client3 = getHoodieWriteClient(config); + // Step 4: Now perform INSERT_OVERWRITE on partition1 (with overlap) - should fail + String instant4 = nextInstant(); + List records4 = dataGen.generateInsertsForPartition(instant4, 50, "2023/01/01"); + JavaRDD writeRecords4 = jsc.parallelize(records4, 2); + HoodieUpsertException upsertException = assertThrows(HoodieUpsertException.class, () -> + commitInsertOverwrite(client3, writeRecords4, instant4) + ); + assertTrue(upsertException.getCause().getMessage().contains("Not allowed to update the clustering file group")); + + // Verify clustering is still pending (NOT rolled back) + metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTimeline pendingReplaceTimeline = metaClient.getActiveTimeline().filterPendingClusteringTimeline(); + assertTrue(pendingReplaceTimeline.containsInstant(clusteringInstant), "Pending clustering should remain pending after failed INSERT_OVERWRITE"); + + // Verify the INSERT_OVERWRITE did NOT complete + HoodieTimeline completedReplaceTimeline = metaClient.getCommitTimeline() + .filterCompletedInstants(); + assertFalse(completedReplaceTimeline.containsInstant(instant4)); + } + + /** + * Test getPartitionToReplacedFileIds for static INSERT_OVERWRITE. + * Static overwrite uses configured partition paths, not input records. + */ + @Test + public void testGetPartitionToReplacedFileIdsForStaticOverwrite() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into multiple partitions + String instant1 = nextInstant(); + List partition1Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/01"); + List partition2Records = dataGen.generateInsertsForPartition(instant1, 100, "2023/01/02"); + List allRecords = new ArrayList<>(); + allRecords.addAll(partition1Records); + allRecords.addAll(partition2Records); + commitInsert(client, jsc.parallelize(allRecords, 2), instant1); + + // Insert more data to create multiple file groups per partition + String instant2 = nextInstant(); + List moreRecords1 = dataGen.generateInsertsForPartition(instant2, 50, "2023/01/01"); + List moreRecords2 = dataGen.generateInsertsForPartition(instant2, 50, "2023/01/02"); + List moreRecords = new ArrayList<>(); + moreRecords.addAll(moreRecords1); + moreRecords.addAll(moreRecords2); + commitInsert(client, jsc.parallelize(moreRecords, 2), instant2); + + // Get existing file IDs before overwrite + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTable table = HoodieSparkTable.create(config, context, metaClient); + List partition1FileIds = table.getSliceView() + .getLatestFileSlices("2023/01/01") + .map(slice -> slice.getFileId()) + .collect(Collectors.toList()); + List partition2FileIds = table.getSliceView() + .getLatestFileSlices("2023/01/02") + .map(slice -> slice.getFileId()) + .collect(Collectors.toList()); + + assertFalse(partition1FileIds.isEmpty(), "Partition 2023/01/01 should have file groups"); + assertFalse(partition2FileIds.isEmpty(), "Partition 2023/01/02 should have file groups"); + + // Step 2: Perform static INSERT_OVERWRITE on partition 2023/01/01 only + String instant3 = nextInstant(); + List overwriteRecords = dataGen.generateInsertsForPartition(instant3, 30, "2023/01/01"); + commitInsertOverwrite(client, jsc.parallelize(overwriteRecords, 2), instant3); + + // Step 3: Verify replaced file IDs - should only include partition 2023/01/01 + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieInstant instant3Instant = metaClient.getActiveTimeline().filterCompletedInstants() + .filter(i -> i.requestedTime().equals(instant3)).firstInstant().get(); + HoodieReplaceCommitMetadata replaceMetadata = metaClient.getActiveTimeline() + .readReplaceCommitMetadata(instant3Instant); + + Map> partitionToReplacedFileIds = replaceMetadata.getPartitionToReplaceFileIds(); + + // Verify partition 2023/01/01 is in replaced file IDs + assertTrue(partitionToReplacedFileIds.containsKey("2023/01/01"), + "Partition 2023/01/01 should be in replaced file IDs"); + + // Verify all file IDs from partition 2023/01/01 are marked as replaced + List replacedFileIds = partitionToReplacedFileIds.get("2023/01/01"); + assertEquals(partition1FileIds.size(), replacedFileIds.size(), + "All file IDs from partition 2023/01/01 should be marked as replaced"); + assertTrue(replacedFileIds.containsAll(partition1FileIds), + "Replaced file IDs should match original file IDs in partition 2023/01/01"); + + // Verify partition 2023/01/02 is NOT in replaced file IDs (was not overwritten) + assertFalse(partitionToReplacedFileIds.containsKey("2023/01/02"), + "Partition 2023/01/02 should NOT be in replaced file IDs"); + } + + /** + * Test getPartitionToReplacedFileIds for dynamic INSERT_OVERWRITE. + * Dynamic overwrite determines partitions from input records. + */ + @Test + public void testGetPartitionToReplacedFileIdsForDynamicOverwrite() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into three partitions + String instant1 = nextInstant(); + List partition1Records = dataGen.generateInsertsForPartition(instant1, 50, "2023/01/01"); + List partition2Records = dataGen.generateInsertsForPartition(instant1, 50, "2023/01/02"); + List partition3Records = dataGen.generateInsertsForPartition(instant1, 50, "2023/01/03"); + List allRecords = new ArrayList<>(); + allRecords.addAll(partition1Records); + allRecords.addAll(partition2Records); + allRecords.addAll(partition3Records); + commitInsert(client, jsc.parallelize(allRecords, 2), instant1); + + // Get existing file IDs + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTable table = HoodieSparkTable.create(config, context, metaClient); + List partition1FileIds = table.getSliceView() + .getLatestFileSlices("2023/01/01") + .map(slice -> slice.getFileId()) + .collect(Collectors.toList()); + List partition2FileIds = table.getSliceView() + .getLatestFileSlices("2023/01/02") + .map(slice -> slice.getFileId()) + .collect(Collectors.toList()); + + // Step 2: Perform dynamic INSERT_OVERWRITE with records for partitions 01/01 and 01/02 + String instant2 = nextInstant(); + List overwriteRecords = new ArrayList<>(); + overwriteRecords.addAll(dataGen.generateInsertsForPartition(instant2, 30, "2023/01/01")); + overwriteRecords.addAll(dataGen.generateInsertsForPartition(instant2, 30, "2023/01/02")); + commitInsertOverwrite(client, jsc.parallelize(overwriteRecords, 2), instant2); + + // Step 3: Verify replaced file IDs include both partitions + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieInstant instant2Instant = metaClient.getActiveTimeline().filterCompletedInstants() + .filter(i -> i.requestedTime().equals(instant2)).firstInstant().get(); + HoodieReplaceCommitMetadata replaceMetadata = metaClient.getActiveTimeline() + .readReplaceCommitMetadata(instant2Instant); + + Map> partitionToReplacedFileIds = replaceMetadata.getPartitionToReplaceFileIds(); + + // Verify both partitions are in replaced file IDs + assertTrue(partitionToReplacedFileIds.containsKey("2023/01/01"), + "Partition 2023/01/01 should be in replaced file IDs"); + assertTrue(partitionToReplacedFileIds.containsKey("2023/01/02"), + "Partition 2023/01/02 should be in replaced file IDs"); + + // Verify file IDs match + assertEquals(partition1FileIds.size(), partitionToReplacedFileIds.get("2023/01/01").size()); + assertEquals(partition2FileIds.size(), partitionToReplacedFileIds.get("2023/01/02").size()); + + // Verify partition 2023/01/03 is NOT in replaced file IDs + assertFalse(partitionToReplacedFileIds.containsKey("2023/01/03"), + "Partition 2023/01/03 should NOT be in replaced file IDs"); + } + + /** + * Test getPartitionToReplacedFileIds when overwriting a partition with multiple file groups. + */ + @Test + public void testGetPartitionToReplacedFileIdsWithMultipleFileGroups() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Create multiple file groups in a single partition through multiple inserts + String partitionPath = "2023/01/01"; + for (int i = 1; i <= 3; i++) { + String instant = nextInstant(); + List records = dataGen.generateInsertsForPartition(instant, 50, partitionPath); + commitBulkInsert(client, jsc.parallelize(records, 2), instant); + } + + // Get all file IDs in the partition + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTable table = HoodieSparkTable.create(config, context, metaClient); + List existingFileIds = table.getSliceView() + .getLatestFileSlices(partitionPath) + .map(slice -> slice.getFileId()) + .distinct() + .collect(Collectors.toList()); + + assertTrue(existingFileIds.size() >= 2, + "Should have multiple file groups in partition from multiple inserts"); + + // Step 2: Perform INSERT_OVERWRITE + metaClient = HoodieTableMetaClient.reload(metaClient); + String overwriteInstant = nextInstant(); + List overwriteRecords = dataGen.generateInsertsForPartition(overwriteInstant, 100, partitionPath); + commitInsertOverwrite(client, jsc.parallelize(overwriteRecords, 2), overwriteInstant); + + // Step 3: Verify all file groups are marked as replaced + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieInstant overwriteInstantObj = metaClient.getActiveTimeline().filterCompletedInstants() + .filter(i -> i.requestedTime().equals(overwriteInstant)).firstInstant().get(); + HoodieReplaceCommitMetadata replaceMetadata = metaClient.getActiveTimeline() + .readReplaceCommitMetadata(overwriteInstantObj); + + Map> partitionToReplacedFileIds = replaceMetadata.getPartitionToReplaceFileIds(); + + assertTrue(partitionToReplacedFileIds.containsKey(partitionPath)); + List replacedFileIds = partitionToReplacedFileIds.get(partitionPath); + + // All existing file IDs should be marked as replaced + assertEquals(existingFileIds.size(), replacedFileIds.size(), + "All file groups should be marked as replaced"); + assertTrue(replacedFileIds.containsAll(existingFileIds), + "Replaced file IDs should include all original file groups"); + } + + /** + * Test getPartitionToReplacedFileIds when overwriting an empty partition. + */ + @Test + public void testGetPartitionToReplacedFileIdsForEmptyPartition() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data into partition1 + String instant1 = nextInstant(); + String partition1 = "2023/01/01"; + List records1 = dataGen.generateInsertsForPartition(instant1, 100, partition1); + commitInsert(client, jsc.parallelize(records1, 2), instant1); + + // Step 2: Perform INSERT_OVERWRITE on a different partition (empty partition2) + String instant2 = nextInstant(); + String partition2 = "2023/01/02"; + List overwriteRecords = dataGen.generateInsertsForPartition(instant2, 50, partition2); + commitInsertOverwrite(client, jsc.parallelize(overwriteRecords, 2), instant2); + + // Step 3: Verify partition2 is in replaced file IDs even though it was empty + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieInstant instant2Instant = metaClient.getActiveTimeline().filterCompletedInstants() + .filter(i -> i.requestedTime().equals(instant2)).firstInstant().get(); + HoodieReplaceCommitMetadata replaceMetadata = metaClient.getActiveTimeline() + .readReplaceCommitMetadata(instant2Instant); + Map> partitionToReplacedFileIds = replaceMetadata.getPartitionToReplaceFileIds(); + + // partition2 should be present with empty list (no existing files to replace) + assertTrue(partitionToReplacedFileIds.containsKey(partition2), + "Empty partition should still be in replaced file IDs map"); + assertTrue(partitionToReplacedFileIds.get(partition2).isEmpty(), + "Empty partition should have empty list of replaced file IDs"); + + // partition1 should NOT be in replaced file IDs + assertFalse(partitionToReplacedFileIds.containsKey(partition1), + "Non-overwritten partition should not be in replaced file IDs"); + } + + /** + * Test getPartitionToReplacedFileIds validates actual file IDs, not just counts. + * Ensures the specific file IDs returned match the file groups that existed. + */ + @Test + public void testGetPartitionToReplacedFileIdsValidatesActualFileIds() throws Exception { + HoodieWriteConfig config = getConfigBuilder(false).build(); + SparkRDDWriteClient client = getHoodieWriteClient(config); + + // Step 1: Insert data + String instant1 = nextInstant(); + String partitionPath = "2023/01/01"; + List records = dataGen.generateInsertsForPartition(instant1, 100, partitionPath); + commitInsert(client, jsc.parallelize(records, 2), instant1); + + // Step 2: Insert more data to create additional file groups + String instant2 = nextInstant(); + List moreRecords = dataGen.generateInsertsForPartition(instant2, 50, partitionPath); + commitInsert(client, jsc.parallelize(moreRecords, 2), instant2); + + // Capture exact file IDs before overwrite + HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(this.metaClient); + HoodieTable table = HoodieSparkTable.create(config, context, metaClient); + Set expectedFileIds = table.getSliceView() + .getLatestFileSlices(partitionPath) + .map(slice -> slice.getFileId()) + .collect(Collectors.toSet()); + + assertFalse(expectedFileIds.isEmpty(), "Should have file groups before overwrite"); + + // Step 3: Perform INSERT_OVERWRITE + String instant3 = nextInstant(); + List overwriteRecords = dataGen.generateInsertsForPartition(instant3, 75, partitionPath); + commitInsertOverwrite(client, jsc.parallelize(overwriteRecords, 2), instant3); + + // Step 4: Validate exact file IDs in replaced list + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieInstant instant3Instant = metaClient.getActiveTimeline().filterCompletedInstants() + .filter(i -> i.requestedTime().equals(instant3)).firstInstant().get(); + HoodieReplaceCommitMetadata replaceMetadata = metaClient.getActiveTimeline() + .readReplaceCommitMetadata(instant3Instant); + Map> partitionToReplacedFileIds = replaceMetadata.getPartitionToReplaceFileIds(); + + Set actualReplacedFileIds = new HashSet<>(partitionToReplacedFileIds.get(partitionPath)); + + // Validate exact file IDs, not just counts + assertEquals(expectedFileIds, actualReplacedFileIds, + "Replaced file IDs should exactly match the file groups that existed before overwrite"); + } + + // Hudi instant times are millisecond-resolution; sleep briefly between generations so + // back-to-back instants in a single test method are strictly ordered. + private String nextInstant() throws InterruptedException { + Thread.sleep(2); + return WriteClientTestUtils.createNewInstantTime(); + } + + private JavaRDD commitInsert(SparkRDDWriteClient client, JavaRDD records, String instantTime) { + WriteClientTestUtils.startCommitWithTime(client, instantTime); + JavaRDD writeStatuses = client.insert(records, instantTime); + List statusList = writeStatuses.collect(); + client.commit(instantTime, jsc.parallelize(statusList, 1)); + return writeStatuses; + } + + private JavaRDD commitBulkInsert(SparkRDDWriteClient client, JavaRDD records, String instantTime) { + WriteClientTestUtils.startCommitWithTime(client, instantTime); + JavaRDD writeStatuses = client.bulkInsert(records, instantTime); + List statusList = writeStatuses.collect(); + client.commit(instantTime, jsc.parallelize(statusList, 1)); + return writeStatuses; + } + + private HoodieWriteResult commitInsertOverwrite(SparkRDDWriteClient client, JavaRDD records, String instantTime) { + WriteClientTestUtils.startCommitWithTime(client, instantTime, HoodieTimeline.REPLACE_COMMIT_ACTION); + HoodieWriteResult result = client.insertOverwrite(records, instantTime); + List statusList = result.getWriteStatuses().collect(); + client.commit(instantTime, jsc.parallelize(statusList, 1), Option.empty(), HoodieTimeline.REPLACE_COMMIT_ACTION, + result.getPartitionToReplaceFileIds()); + return result; + } + + private HoodieWriteResult commitDeletePartitions(SparkRDDWriteClient client, List partitions, String instantTime) { + WriteClientTestUtils.startCommitWithTime(client, instantTime, HoodieTimeline.REPLACE_COMMIT_ACTION); + HoodieWriteResult result = client.deletePartitions(partitions, instantTime); + List statusList = result.getWriteStatuses().collect(); + client.commit(instantTime, jsc.parallelize(statusList, 1), Option.empty(), HoodieTimeline.REPLACE_COMMIT_ACTION, + result.getPartitionToReplaceFileIds()); + return result; + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/ConsistentBucketUpdateStrategy.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/ConsistentBucketUpdateStrategy.java index c7ea6d8747f63..3f4515e1b2f15 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/ConsistentBucketUpdateStrategy.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/ConsistentBucketUpdateStrategy.java @@ -62,7 +62,7 @@ public class ConsistentBucketUpdateStrategy extends UpdateStrategy indexKeyFields) { - super(writeClient.getEngineContext(), writeClient.getHoodieTable(), Collections.emptySet()); + super(writeClient.getEngineContext(), writeClient.getHoodieTable(), Collections.emptySet(), Collections.emptySet()); this.indexKeyFields = indexKeyFields; this.partitionToIdentifier = new HashMap<>(); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/FlinkConsistentBucketUpdateStrategy.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/FlinkConsistentBucketUpdateStrategy.java index b6159f411bd29..9272653096da5 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/FlinkConsistentBucketUpdateStrategy.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/FlinkConsistentBucketUpdateStrategy.java @@ -62,7 +62,7 @@ public class FlinkConsistentBucketUpdateStrategy private String lastRefreshInstant = HoodieTimeline.INIT_INSTANT_TS; public FlinkConsistentBucketUpdateStrategy(HoodieFlinkWriteClient writeClient, List indexKeyFields) { - super(writeClient.getEngineContext(), writeClient.getHoodieTable(), Collections.emptySet()); + super(writeClient.getEngineContext(), writeClient.getHoodieTable(), Collections.emptySet(), Collections.emptySet()); this.indexKeyFields = indexKeyFields; this.partitionToIdentifier = new HashMap<>(); } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java index 75b5e6637f9a5..8b6466d5117d6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java @@ -24,12 +24,18 @@ import org.apache.hudi.client.HoodieWriteResult; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.clustering.update.strategy.SparkAllowUpdateStrategy; import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.CommitUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.data.HoodieJavaRDD; import org.apache.hudi.exception.HoodieException; @@ -41,8 +47,10 @@ import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; +import org.apache.hudi.table.action.cluster.strategy.UpdateStrategy; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -51,9 +59,12 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.apache.hudi.config.HoodieWriteConfig.WRITE_STATUS_STORAGE_LEVEL_VALUE; +@Slf4j public abstract class BaseDatasetBulkInsertCommitActionExecutor implements Serializable { protected final transient HoodieWriteConfig writeConfig; @@ -109,6 +120,11 @@ public final HoodieWriteResult execute(Dataset records, boolean isTablePart BulkInsertPartitioner> bulkInsertPartitionerRows = getPartitioner(populateMetaFields, isTablePartitioned); Dataset hoodieDF = HoodieDatasetBulkInsertHelper.prepareForBulkInsert(records, writeConfig, table.getMetaClient().getTableConfig(), bulkInsertPartitionerRows, instantTime); + // Reject INSERT_OVERWRITE / INSERT_OVERWRITE_TABLE against partitions with pending + // clustering before any writes materialize. Subclasses override getFileGroupsBeingReplaced; + // default is a no-op (empty set) which preserves the non-overwrite paths. + rejectIfOverlappingPendingClustering(hoodieDF); + HoodieWriteMetadata> result = buildHoodieWriteMetadata(doExecute(hoodieDF, bulkInsertPartitionerRows.arePartitionRecordsSorted())); afterExecute(result); @@ -141,4 +157,77 @@ protected BulkInsertPartitioner> getPartitioner(boolean populateMet } protected abstract Map> getPartitionToReplacedFileIds(HoodieData writeStatuses); + + /** + * Returns the file groups this operation will replace. Default is empty (non-overwrite paths). + * Bulk-insert overwrite executors override this so the overlap-with-pending-clustering check + * can fire before any writes materialize. + * + * @param preparedRecords the dataset after {@code HoodieDatasetBulkInsertHelper.prepareForBulkInsert} + * has populated the {@code _hoodie_partition_path} meta field, so dynamic + * partition resolution can read it. + */ + protected Set getFileGroupsBeingReplaced(Dataset preparedRecords) { + return Collections.emptySet(); + } + + /** + * Mirrors {@code BaseSparkCommitActionExecutor#clusteringHandleUpdate} for the bulk-insert row + * path: if any of the file groups this operation will replace are in pending clustering, route + * through the configured {@code hoodie.clustering.updates.strategy}. With the default + * {@code SparkRejectUpdateStrategy} this throws {@code HoodieClusteringUpdateException}; with + * {@code SparkAllowUpdateStrategy} (and {@code !isRollbackPendingClustering()}) the overlap is + * deferred to the conflict-resolution strategy, matching the existing Spark-side behavior. + */ + protected void rejectIfOverlappingPendingClustering(Dataset preparedRecords) { + Set fileGroupsInPendingClustering = table.getFileSystemView() + .getFileGroupsInPendingClustering().map(Pair::getKey).collect(Collectors.toSet()); + if (fileGroupsInPendingClustering.isEmpty()) { + return; + } + Set fileGroupsToBeReplaced = getFileGroupsBeingReplaced(preparedRecords); + if (fileGroupsToBeReplaced.isEmpty()) { + return; + } + + HoodieEngineContext engineContext = writeClient.getEngineContext(); + UpdateStrategy> updateStrategy = loadClusteringUpdateStrategy( + engineContext, fileGroupsInPendingClustering, fileGroupsToBeReplaced); + if (updateStrategy instanceof SparkAllowUpdateStrategy && !writeConfig.isRollbackPendingClustering()) { + return; + } + // handleUpdate consumes only fileGroupsToBeReplaced on this path (no tagged records to + // inspect for the bulk-insert overwrite case), so pass an empty HoodieData. + updateStrategy.handleUpdate(engineContext.emptyHoodieData()); + } + + /** + * Loads {@code hoodie.clustering.updates.strategy} via reflection, preferring the 4-arg + * constructor (with {@code fileGroupsToBeReplaced}) and falling back to the legacy 3-arg + * constructor for custom strategies that pre-date this PR. + */ + @SuppressWarnings("unchecked") + private UpdateStrategy> loadClusteringUpdateStrategy( + HoodieEngineContext engineContext, + Set fileGroupsInPendingClustering, + Set fileGroupsToBeReplaced) { + String strategyClass = writeConfig.getClusteringUpdatesStrategyClass(); + try { + return (UpdateStrategy>) ReflectionUtils.loadClass( + strategyClass, + new Class[] {HoodieEngineContext.class, HoodieTable.class, Set.class, Set.class}, + engineContext, table, fileGroupsInPendingClustering, fileGroupsToBeReplaced); + } catch (HoodieException ex) { + if (!(ex.getCause() instanceof NoSuchMethodException)) { + throw ex; + } + log.warn("Clustering update strategy {} is missing the 4-arg constructor with " + + "fileGroupsToBeReplaced; falling back to the 3-arg constructor. INSERT_OVERWRITE " + + "overlap with pending clustering will not be detected for this strategy.", strategyClass); + return (UpdateStrategy>) ReflectionUtils.loadClass( + strategyClass, + new Class[] {HoodieEngineContext.class, HoodieTable.class, Set.class}, + engineContext, table, fileGroupsInPendingClustering); + } + } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java index deaeac4df45f4..397407adb6b31 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java @@ -23,6 +23,8 @@ import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; @@ -33,12 +35,14 @@ import org.apache.hudi.data.HoodieJavaPairRDD; import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class DatasetBulkInsertOverwriteCommitActionExecutor extends BaseDatasetBulkInsertCommitActionExecutor { @@ -56,6 +60,46 @@ protected Option> doExecute(Dataset records, boolea .bulkInsert(records, instantTime, table, writeConfig, arePartitionRecordsSorted, false)); } + /** + * For INSERT_OVERWRITE: enumerate latest file groups in the targeted partitions so the caller + * can reject overlap with pending clustering before the bulk-insert materializes. Mirrors + * {@code SparkInsertOverwriteCommitActionExecutor#getFileGroupsBeingReplaced}; called by the + * base class on the prepared dataset (after {@code prepareForBulkInsert} populates the + * {@code _hoodie_partition_path} meta field), so dynamic partition resolution can read it. + */ + @Override + protected Set getFileGroupsBeingReplaced(Dataset preparedRecords) { + List partitionPaths = resolveTargetPartitions(preparedRecords); + if (partitionPaths.isEmpty()) { + return Collections.emptySet(); + } + return partitionPaths.stream() + .flatMap(partitionPath -> table.getSliceView().getLatestFileSlices(partitionPath) + .map(FileSlice::getFileGroupId)) + .collect(Collectors.toSet()); + } + + /** + * Resolves the partition paths this overwrite will replace. Subclasses override for the + * table-wide variant (enumerate every partition). + */ + protected List resolveTargetPartitions(Dataset preparedRecords) { + if (!table.isPartitioned()) { + return Collections.singletonList(StringUtils.EMPTY_STRING); + } + String staticOverwritePartitionPaths = writeConfig.getStringOrDefault(HoodieInternalConfig.STATIC_OVERWRITE_PARTITION_PATHS); + if (StringUtils.nonEmpty(staticOverwritePartitionPaths)) { + return Arrays.asList(staticOverwritePartitionPaths.split(",")); + } + // Dynamic partition path: read the populated _hoodie_partition_path meta field. The base + // class invokes this hook after HoodieDatasetBulkInsertHelper.prepareForBulkInsert, so the + // field is guaranteed to be present and populated by the configured key generator. + return preparedRecords.select(HoodieRecord.PARTITION_PATH_METADATA_FIELD) + .distinct() + .as(Encoders.STRING()) + .collectAsList(); + } + @Override public WriteOperationType getWriteOperationType() { return WriteOperationType.INSERT_OVERWRITE; diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java index d35325360745f..3dc1df5dc2700 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java @@ -28,6 +28,9 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.data.HoodieJavaPairRDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -44,6 +47,14 @@ public WriteOperationType getWriteOperationType() { return WriteOperationType.INSERT_OVERWRITE_TABLE; } + @Override + protected List resolveTargetPartitions(Dataset preparedRecords) { + // Table-wide overwrite replaces every file group in every partition; enumerate them all. + List partitionPaths = FSUtils.getAllPartitionPaths(writeClient.getEngineContext(), + table.getMetaClient(), writeConfig.getMetadataConfig()); + return partitionPaths == null ? Collections.emptyList() : partitionPaths; + } + @Override protected Map> getPartitionToReplacedFileIds(HoodieData writeStatuses) { HoodieEngineContext context = writeClient.getEngineContext(); From 4570ecdf2db7d314021ec2aad45fcca38e3736a0 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Fri, 26 Jun 2026 16:51:20 -0700 Subject: [PATCH 106/255] fix(streamer): override all deserialize() overloads in KafkaAvroSchemaDeserializer (#18892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(streamer): prevent ArrayIndexOutOfBoundsException on Kafka Avro schema evolution KafkaAvroSchemaDeserializer only overrode deserialize(String, Boolean, byte[], Schema). The other overloads — deserialize(String, byte[]), (String, byte[], Schema), and (String, Headers, byte[]) — bypassed sourceSchema injection and fell back to the writer's schema, causing ArrayIndexOutOfBoundsException when consuming records serialized with an older (fewer-field) schema while the deserializer was configured with an evolved reader schema. Override all four overloads to route through super.deserialize(topic, false, bytes, sourceSchema) so Avro resolution consistently uses the configured reader schema regardless of the Kafka client's entry point. Closes #18891 --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 4efc74739158b9210548aaeae276fcd7dc95f41d) --- .../deser/KafkaAvroSchemaDeserializer.java | 16 ++ .../TestKafkaAvroSchemaDeserializer.java | 205 ++++++++++++++ .../resources/schema/cdc_envelope_new.avsc | 261 ++++++++++++++++++ .../resources/schema/cdc_envelope_old.avsc | 237 ++++++++++++++++ 4 files changed, 719 insertions(+) create mode 100644 hudi-utilities/src/test/resources/schema/cdc_envelope_new.avsc create mode 100644 hudi-utilities/src/test/resources/schema/cdc_envelope_old.avsc diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java index a2ffb4878454e..b1d0589150c2e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deser/KafkaAvroSchemaDeserializer.java @@ -26,6 +26,7 @@ import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.header.Headers; import java.util.Map; import java.util.Map.Entry; @@ -57,6 +58,21 @@ public void configure(Map configs, boolean isKey) { } } + @Override + public Object deserialize(String topic, byte[] bytes) { + return this.deserialize(topic, false, bytes, sourceSchema); + } + + @Override + public Object deserialize(String topic, byte[] bytes, Schema readerSchema) { + return this.deserialize(topic, false, bytes, sourceSchema); + } + + @Override + public Object deserialize(String topic, Headers headers, byte[] bytes) { + return super.deserialize(topic, false, bytes, sourceSchema); + } + /** * We need to inject sourceSchema instead of reader schema during deserialization or later stages of the pipeline. * diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deser/TestKafkaAvroSchemaDeserializer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deser/TestKafkaAvroSchemaDeserializer.java index 6846724704337..01887573a91f4 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deser/TestKafkaAvroSchemaDeserializer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deser/TestKafkaAvroSchemaDeserializer.java @@ -30,8 +30,12 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.kafka.common.header.internals.RecordHeaders; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Properties; @@ -125,4 +129,205 @@ public void testKafkaAvroSchemaDeserializer() { assertEquals(HoodieSchema.fromAvroSchema(actualRec.getSchema()), evolSchema); assertNull(genericRecord.get("age")); } + + private Schema loadSchemaFromResource(String resourcePath) throws IOException { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) { + return new Schema.Parser().parse(is); + } + } + + private static Schema getRecordTypeFromUnion(Schema unionSchema) { + return unionSchema.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.RECORD) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No record type in union")); + } + + private GenericRecord createCdcSourceRecord(Schema envelopeSchema) { + Schema sourceSchema = envelopeSchema.getField("source").schema(); + GenericRecord source = new GenericData.Record(sourceSchema); + source.put("version", "2.3.0.Final"); + source.put("connector", "mysql"); + source.put("name", "cdc"); + source.put("ts_ms", 1700000000000L); + source.put("snapshot", "false"); + source.put("db", "testdb"); + source.put("sequence", null); + source.put("table", "item"); + source.put("server_id", 1L); + source.put("gtid", null); + source.put("file", "binlog.000001"); + source.put("pos", 12345L); + source.put("row", 0); + source.put("thread", null); + source.put("query", null); + return source; + } + + private GenericRecord createCdcValueRecord(Schema envelopeSchema) { + Schema valueSchema = getRecordTypeFromUnion(envelopeSchema.getField("before").schema()); + GenericRecord value = new GenericData.Record(valueSchema); + value.put("id", ByteBuffer.wrap(new byte[]{1, 2, 3, 4})); + value.put("account_id", 42); + value.put("title", "Test Item"); + value.put("query", "test query"); + value.put("request_query_id", null); + value.put("page", null); + value.put("request_page_id", null); + value.put("source_rank_id", null); + value.put("property_id", 100); + value.put("created", "2024-01-01T00:00:00Z"); + value.put("created_by", 1); + value.put("updated", "2024-01-02T00:00:00Z"); + value.put("updated_by", 2); + value.put("version_id", null); + value.put("related_urls", null); + value.put("assignee", 5); + value.put("status", "IN_PROGRESS"); + value.put("tags", null); + value.put("deleted", false); + value.put("profile_id", null); + return value; + } + + private GenericRecord createCdcValueRecordBefore(Schema envelopeSchema) { + Schema valueSchema = getRecordTypeFromUnion(envelopeSchema.getField("before").schema()); + GenericRecord value = new GenericData.Record(valueSchema); + value.put("id", ByteBuffer.wrap(new byte[]{1, 2, 3, 4})); + value.put("account_id", 42); + value.put("title", "Old Item Title"); + value.put("query", "old query"); + value.put("request_query_id", null); + value.put("page", "https://example.com/old"); + value.put("request_page_id", null); + value.put("source_rank_id", 10); + value.put("property_id", 100); + value.put("created", "2024-01-01T00:00:00Z"); + value.put("created_by", 1); + value.put("updated", "2024-01-02T00:00:00Z"); + value.put("updated_by", 1); + value.put("version_id", "v1"); + value.put("related_urls", "[\"https://example.com\"]"); + value.put("assignee", 5); + value.put("status", "TO_DO"); + value.put("tags", null); + value.put("deleted", false); + value.put("profile_id", null); + return value; + } + + private GenericRecord createCdcEnvelopeRecord(Schema envelopeSchema, GenericRecord beforeRecord, GenericRecord afterRecord) { + GenericRecord envelope = new GenericData.Record(envelopeSchema); + envelope.put("before", beforeRecord); + envelope.put("after", afterRecord); + envelope.put("source", createCdcSourceRecord(envelopeSchema)); + envelope.put("op", "u"); + envelope.put("ts_ms", 1700000000000L); + envelope.put("transaction", null); + return envelope; + } + + /** + * Tests deserialization of a CDC (Debezium) envelope schema with schema evolution + * across all deserialize method overloads. The old schema lacks 4 fields + * (notes, search_engine_id, locale_id, language_id) in the nested Value record. + * When deserializing old records with the evolved schema, those new fields should default to null. + * + * Exercises: + * - deserialize(String topic, Boolean isKey, byte[] payload, Schema readerSchema) + * - deserialize(String topic, byte[] bytes) + * - deserialize(String topic, byte[] bytes, Schema readerSchema) + * - deserialize(String topic, Headers headers, byte[] bytes) + */ + @Test + public void testKafkaAvroSchemaDeserializerWithCdcEnvelopeEvolution() throws IOException { + Schema cdcOldSchema = loadSchemaFromResource("schema/cdc_envelope_old.avsc"); + Schema cdcNewSchema = loadSchemaFromResource("schema/cdc_envelope_new.avsc"); + + // Create and serialize records with old schema (no notes/search_engine_id/locale_id/language_id) + GenericRecord oldBefore = createCdcValueRecordBefore(cdcOldSchema); + GenericRecord oldAfter = createCdcValueRecord(cdcOldSchema); + GenericRecord oldEnvelope = createCdcEnvelopeRecord(cdcOldSchema, oldBefore, oldAfter); + byte[] bytesOldRecord = avroSerializer.serialize(topic, oldEnvelope); + + // Create and serialize records with evolved schema (new fields populated) + GenericRecord newBefore = createCdcValueRecordBefore(cdcNewSchema); + GenericRecord newAfter = createCdcValueRecord(cdcNewSchema); + newAfter.put("notes", "[{\"note\": \"test\"}]"); + newAfter.put("search_engine_id", "[\"default\"]"); + newAfter.put("locale_id", 1); + newAfter.put("language_id", 2); + GenericRecord newEnvelope = createCdcEnvelopeRecord(cdcNewSchema, newBefore, newAfter); + byte[] bytesNewRecord = avroSerializer.serialize(topic, newEnvelope); + + // Configure deserializer with evolved schema + config.put(AvroKafkaSource.KAFKA_AVRO_VALUE_DESERIALIZER_SCHEMA, cdcNewSchema.toString()); + KafkaAvroSchemaDeserializer deserializer = new KafkaAvroSchemaDeserializer(schemaRegistry, new HashMap(config)); + deserializer.configure(new HashMap(config), false); + + // === 1. deserialize(String, Boolean, byte[], Schema) — the existing override === + IndexedRecord evolvedDeserialized = (IndexedRecord) deserializer.deserialize(topic, false, bytesOldRecord, cdcNewSchema); + GenericRecord evolvedEnvelope = (GenericRecord) evolvedDeserialized; + assertEquals(cdcNewSchema, evolvedEnvelope.getSchema()); + assertEquals("u", evolvedEnvelope.get("op").toString()); + // Validate before record + GenericRecord beforeRecord = (GenericRecord) evolvedEnvelope.get("before"); + assertEquals("Old Item Title", beforeRecord.get("title").toString()); + assertEquals(42, beforeRecord.get("account_id")); + assertEquals("TO_DO", beforeRecord.get("status").toString()); + assertNull(beforeRecord.get(20)); + assertNull(beforeRecord.get(21)); + assertNull(beforeRecord.get(22)); + assertNull(beforeRecord.get(23)); + // Validate after record + GenericRecord afterRecord = (GenericRecord) evolvedEnvelope.get("after"); + assertEquals("Test Item", afterRecord.get("title").toString()); + assertNull(afterRecord.get("notes")); + assertNull(afterRecord.get("search_engine_id")); + assertNull(afterRecord.get("locale_id")); + assertNull(afterRecord.get("language_id")); + + // Evolved record via same method — should round-trip exactly + IndexedRecord newDeserialized = (IndexedRecord) deserializer.deserialize(topic, false, bytesNewRecord, cdcNewSchema); + assertEquals(newEnvelope, newDeserialized); + + // === 2. deserialize(String, byte[]) === + GenericRecord topicBytesResult = (GenericRecord) deserializer.deserialize(topic, bytesOldRecord); + GenericRecord tbBefore = (GenericRecord) topicBytesResult.get("before"); + assertEquals("Old Item Title", tbBefore.get("title").toString()); + assertNull(tbBefore.get(20)); + assertNull(tbBefore.get(21)); + assertNull(tbBefore.get(22)); + assertNull(tbBefore.get(23)); + + // === 3. deserialize(String, byte[], Schema) === + GenericRecord topicBytesSchemaResult = (GenericRecord) deserializer.deserialize(topic, bytesOldRecord, cdcNewSchema); + GenericRecord tbsBefore = (GenericRecord) topicBytesSchemaResult.get("before"); + assertEquals("Old Item Title", tbsBefore.get("title").toString()); + assertNull(tbsBefore.get(20)); + assertNull(tbsBefore.get(21)); + assertNull(tbsBefore.get(22)); + assertNull(tbsBefore.get(23)); + + // === 4. deserialize(String, Headers, byte[]) === + RecordHeaders headers = new RecordHeaders(); + GenericRecord headersResult = (GenericRecord) deserializer.deserialize(topic, headers, bytesOldRecord); + assertEquals(cdcNewSchema, headersResult.getSchema()); + GenericRecord hdrBefore = (GenericRecord) headersResult.get("before"); + assertEquals("Old Item Title", hdrBefore.get("title").toString()); + assertEquals(42, hdrBefore.get("account_id")); + assertNull(hdrBefore.get(20)); + assertNull(hdrBefore.get(21)); + assertNull(hdrBefore.get(22)); + assertNull(hdrBefore.get(23)); + + // New record via headers method + GenericRecord newHdrResult = (GenericRecord) deserializer.deserialize(topic, headers, bytesNewRecord); + GenericRecord newHdrAfter = (GenericRecord) newHdrResult.get("after"); + assertEquals("[{\"note\": \"test\"}]", newHdrAfter.get("notes").toString()); + assertEquals("[\"default\"]", newHdrAfter.get("search_engine_id").toString()); + assertEquals(1, newHdrAfter.get("locale_id")); + assertEquals(2, newHdrAfter.get("language_id")); + } + } diff --git a/hudi-utilities/src/test/resources/schema/cdc_envelope_new.avsc b/hudi-utilities/src/test/resources/schema/cdc_envelope_new.avsc new file mode 100644 index 0000000000000..bf3ed9c0432d4 --- /dev/null +++ b/hudi-utilities/src/test/resources/schema/cdc_envelope_new.avsc @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "type" : "record", + "name" : "Envelope", + "namespace" : "cdc.test.inventory.item", + "fields" : [ { + "name" : "before", + "type" : [ "null", { + "type" : "record", + "name" : "Value", + "fields" : [ { + "name" : "id", + "type" : "bytes" + }, { + "name" : "account_id", + "type" : "int" + }, { + "name" : "title", + "type" : "string" + }, { + "name" : "query", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "request_query_id", + "type" : [ "null", "bytes" ], + "default" : null + }, { + "name" : "page", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "request_page_id", + "type" : [ "null", "bytes" ], + "default" : null + }, { + "name" : "source_rank_id", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "property_id", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "created", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.default" : "1970-01-01T00:00:00Z", + "connect.name" : "io.debezium.time.ZonedTimestamp" + }, + "default" : "1970-01-01T00:00:00Z" + }, { + "name" : "created_by", + "type" : "int" + }, { + "name" : "updated", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.default" : "1970-01-01T00:00:00Z", + "connect.name" : "io.debezium.time.ZonedTimestamp" + }, + "default" : "1970-01-01T00:00:00Z" + }, { + "name" : "updated_by", + "type" : "int" + }, { + "name" : "version_id", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "related_urls", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "assignee", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "status", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.parameters" : { + "allowed" : "TO_DO,IN_PROGRESS,IN_REVIEW,APPROVED,PUBLISHED" + }, + "connect.default" : "TO_DO", + "connect.name" : "io.debezium.data.Enum" + }, + "default" : "TO_DO" + }, { + "name" : "tags", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "deleted", + "type" : { + "type" : "boolean", + "connect.default" : false + }, + "default" : false + }, { + "name" : "profile_id", + "type" : [ "null", "bytes" ], + "default" : null + }, { + "name" : "notes", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "search_engine_id", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "locale_id", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "language_id", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "cdc.test.inventory.item.Value" + } ], + "default" : null + }, { + "name" : "after", + "type" : [ "null", "Value" ], + "default" : null + }, { + "name" : "source", + "type" : { + "type" : "record", + "name" : "Source", + "namespace" : "io.debezium.connector.mysql", + "fields" : [ { + "name" : "version", + "type" : "string" + }, { + "name" : "connector", + "type" : "string" + }, { + "name" : "name", + "type" : "string" + }, { + "name" : "ts_ms", + "type" : "long" + }, { + "name" : "snapshot", + "type" : [ { + "type" : "string", + "connect.version" : 1, + "connect.parameters" : { + "allowed" : "true,last,false,incremental" + }, + "connect.default" : "false", + "connect.name" : "io.debezium.data.Enum" + }, "null" ], + "default" : "false" + }, { + "name" : "db", + "type" : "string" + }, { + "name" : "sequence", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "table", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "server_id", + "type" : "long" + }, { + "name" : "gtid", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "file", + "type" : "string" + }, { + "name" : "pos", + "type" : "long" + }, { + "name" : "row", + "type" : "int" + }, { + "name" : "thread", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "query", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.debezium.connector.mysql.Source" + } + }, { + "name" : "op", + "type" : "string" + }, { + "name" : "ts_ms", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "transaction", + "type" : [ "null", { + "type" : "record", + "name" : "block", + "namespace" : "event", + "fields" : [ { + "name" : "id", + "type" : "string" + }, { + "name" : "total_order", + "type" : "long" + }, { + "name" : "data_collection_order", + "type" : "long" + } ], + "connect.version" : 1, + "connect.name" : "event.block" + } ], + "default" : null + } ], + "connect.version" : 1, + "connect.name" : "cdc.test.inventory.item.Envelope" +} diff --git a/hudi-utilities/src/test/resources/schema/cdc_envelope_old.avsc b/hudi-utilities/src/test/resources/schema/cdc_envelope_old.avsc new file mode 100644 index 0000000000000..3d2a832d5d3ce --- /dev/null +++ b/hudi-utilities/src/test/resources/schema/cdc_envelope_old.avsc @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "type" : "record", + "name" : "Envelope", + "namespace" : "cdc.test.inventory.item", + "fields" : [ { + "name" : "before", + "type" : [ "null", { + "type" : "record", + "name" : "Value", + "fields" : [ { + "name" : "id", + "type" : "bytes" + }, { + "name" : "account_id", + "type" : "int" + }, { + "name" : "title", + "type" : "string" + }, { + "name" : "query", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "request_query_id", + "type" : [ "null", "bytes" ], + "default" : null + }, { + "name" : "page", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "request_page_id", + "type" : [ "null", "bytes" ], + "default" : null + }, { + "name" : "source_rank_id", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "property_id", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "created", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.default" : "1970-01-01T00:00:00Z", + "connect.name" : "io.debezium.time.ZonedTimestamp" + }, + "default" : "1970-01-01T00:00:00Z" + }, { + "name" : "created_by", + "type" : "int" + }, { + "name" : "updated", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.default" : "1970-01-01T00:00:00Z", + "connect.name" : "io.debezium.time.ZonedTimestamp" + }, + "default" : "1970-01-01T00:00:00Z" + }, { + "name" : "updated_by", + "type" : "int" + }, { + "name" : "version_id", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "related_urls", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "assignee", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "status", + "type" : { + "type" : "string", + "connect.version" : 1, + "connect.parameters" : { + "allowed" : "TO_DO,IN_PROGRESS,IN_REVIEW,APPROVED,PUBLISHED" + }, + "connect.default" : "TO_DO", + "connect.name" : "io.debezium.data.Enum" + }, + "default" : "TO_DO" + }, { + "name" : "tags", + "type" : [ "null", { + "type" : "string", + "connect.version" : 1, + "connect.name" : "io.debezium.data.Json" + } ], + "default" : null + }, { + "name" : "deleted", + "type" : { + "type" : "boolean", + "connect.default" : false + }, + "default" : false + }, { + "name" : "profile_id", + "type" : [ "null", "bytes" ], + "default" : null + } ], + "connect.name" : "cdc.test.inventory.item.Value" + } ], + "default" : null + }, { + "name" : "after", + "type" : [ "null", "Value" ], + "default" : null + }, { + "name" : "source", + "type" : { + "type" : "record", + "name" : "Source", + "namespace" : "io.debezium.connector.mysql", + "fields" : [ { + "name" : "version", + "type" : "string" + }, { + "name" : "connector", + "type" : "string" + }, { + "name" : "name", + "type" : "string" + }, { + "name" : "ts_ms", + "type" : "long" + }, { + "name" : "snapshot", + "type" : [ { + "type" : "string", + "connect.version" : 1, + "connect.parameters" : { + "allowed" : "true,last,false,incremental" + }, + "connect.default" : "false", + "connect.name" : "io.debezium.data.Enum" + }, "null" ], + "default" : "false" + }, { + "name" : "db", + "type" : "string" + }, { + "name" : "sequence", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "table", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "server_id", + "type" : "long" + }, { + "name" : "gtid", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "file", + "type" : "string" + }, { + "name" : "pos", + "type" : "long" + }, { + "name" : "row", + "type" : "int" + }, { + "name" : "thread", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "query", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.debezium.connector.mysql.Source" + } + }, { + "name" : "op", + "type" : "string" + }, { + "name" : "ts_ms", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "transaction", + "type" : [ "null", { + "type" : "record", + "name" : "block", + "namespace" : "event", + "fields" : [ { + "name" : "id", + "type" : "string" + }, { + "name" : "total_order", + "type" : "long" + }, { + "name" : "data_collection_order", + "type" : "long" + } ], + "connect.version" : 1, + "connect.name" : "event.block" + } ], + "default" : null + } ], + "connect.version" : 1, + "connect.name" : "cdc.test.inventory.item.Envelope" +} From fbd320236009709513a72aa9e8d787294a1c4274 Mon Sep 17 00:00:00 2001 From: Nada Date: Sat, 27 Jun 2026 16:08:48 -0400 Subject: [PATCH 107/255] fix(spark): preserve Spark's native unresolved-column errors in HoodieAnalysis (#18147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(spark): preserve Spark's native unresolved-column errors in HoodieAnalysis Closes #18151 HoodieAnalysis was catching Spark's AnalysisException during column resolution and rethrowing it as a generic HoodieException, which stripped the structured error class (UNRESOLVED_COLUMN.*) Spark exposes to user tooling and downgraded the error to a flat message. Tests on Spark 3.5+ that assert on the error class — and users relying on Spark's error classification for retry / alerting — both broke. Let the original AnalysisException propagate; only wrap unrelated exceptions. Tighten the matching test assertions to the Spark error class instead of a substring match so future regressions surface immediately. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: sivabalan Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Rahil Chertara (cherry picked from commit 1f479c890cc5d9b335520a7677fcce7db60a904d) --- .../analysis/HoodieSparkBaseAnalysis.scala | 9 +- .../sql/hudi/analysis/HoodieAnalysis.scala | 7 +- .../TestHoodieAnalysisErrorHandling.scala | 196 ++++++++++++++++++ 3 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodieAnalysisErrorHandling.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala index 88ad684c1e2fd..5160706e415cb 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala @@ -183,7 +183,14 @@ case class ResolveReferences(spark: SparkSession) extends Rule[LogicalPlan] val sourceTable = if (sourceTableO.resolved) sourceTableO else analyzer.execute(sourceTableO) val m = mO.asInstanceOf[MergeIntoTable].copy(targetTable = targetTable, sourceTable = sourceTable) // END: custom Hudi change - EliminateSubqueryAliases(targetTable) match { + // If the source table still has unresolved references (e.g. a non-existent + // column in the source query, or a missing source table), return the + // partially-resolved MIT and let Spark's CheckAnalysis surface the error. + // Continuing into the resolve-assignments path would either lose the column + // context or throw a less informative UnresolvedException. + if (!sourceTable.resolved) { + m + } else EliminateSubqueryAliases(targetTable) match { case r: NamedRelation if r.skipSchemaResolution => // Do not resolve the expression if the target table accepts any schema. // This allows data sources to customize their own resolution logic using diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala index 46f9ed36ad780..41d6c2b9ae81c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala @@ -322,7 +322,12 @@ object HoodieAnalysis extends SparkAdapterSupport { analyzer.execute(plan) } - if (resolved.output.exists(attr => isMetaField(attr.name))) { + // If the plan is still not fully resolved (e.g., it references non-existent + // tables or columns), fall through. Spark's CheckAnalysis runs later and + // produces precise UNRESOLVED_COLUMN / TABLE_OR_VIEW_NOT_FOUND errors with + // "did you mean" suggestions; intercepting UnresolvedException here would + // discard that context. Only inspect the output once the plan is resolved. + if (resolved.resolved && resolved.output.exists(attr => isMetaField(attr.name))) { Some(resolved.output) } else { None diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodieAnalysisErrorHandling.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodieAnalysisErrorHandling.scala new file mode 100644 index 0000000000000..06e31163df43c --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodieAnalysisErrorHandling.scala @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.hudi.analysis + +import org.apache.hudi.HoodieSparkUtils + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase + +/** + * Regression tests covering how Hudi's analysis surface unresolved references in + * Spark SQL queries. Hudi's [[ProducesHudiMetaFields]] extractor and the MERGE INTO + * resolution path in [[HoodieSparkBaseAnalysis]] used to swallow the + * [[org.apache.spark.sql.catalyst.analysis.UnresolvedException]] and rewrite it as a + * generic Hudi error, which lost Spark's "did you mean" suggestions. Both sites now + * fall through to Spark's CheckAnalysis so the user-facing error remains + * Spark-native (e.g. `UNRESOLVED_COLUMN.WITH_SUGGESTION`). + */ +class TestHoodieAnalysisErrorHandling extends HoodieSparkSqlTestBase { + + test("MERGE INTO with unresolved column in source query surfaces Spark's native error") { + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + |CREATE TABLE $tableName ( + | id INT, + | name STRING, + | price DOUBLE, + | ts INT + |) USING hudi + |LOCATION '${tmp.getCanonicalPath}' + |TBLPROPERTIES ( + | primaryKey = 'id', + | preCombineField = 'ts' + |) + """.stripMargin) + + spark.sql(s"INSERT INTO $tableName VALUES (1, 'a1', 10.0, 1000)") + + // Source query references a non-existent column. Spark's analyzer + // should produce a precise error naming the missing column. + val ex = intercept[AnalysisException] { + spark.sql( + s""" + |MERGE INTO $tableName AS target + |USING ( + | SELECT 1 AS id, 'updated' AS name, 20.0 AS price, 2000 AS ts, nonexistent_column AS extra + |) AS source + |ON target.id = source.id + |WHEN MATCHED THEN UPDATE SET * + |WHEN NOT MATCHED THEN INSERT * + """.stripMargin) + } + val msg = ex.getMessage + assertNativeUnresolvedColumn(msg, "nonexistent_column") + assertNoHudiGenericRewrite(msg) + } + } + + test("INSERT INTO from non-existent source table surfaces Spark's native error") { + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + |CREATE TABLE $tableName ( + | id INT, + | name STRING, + | price DOUBLE, + | ts INT + |) USING hudi + |LOCATION '${tmp.getCanonicalPath}' + |TBLPROPERTIES ( + | primaryKey = 'id', + | preCombineField = 'ts' + |) + """.stripMargin) + + val ex = intercept[AnalysisException] { + spark.sql( + s""" + |INSERT INTO $tableName + |SELECT * FROM nonexistent_source_table + """.stripMargin) + } + val msg = ex.getMessage + assertNativeTableNotFound(msg, "nonexistent_source_table") + assertNoHudiGenericRewrite(msg) + } + } + + test("MERGE INTO with unresolved column in ON predicate surfaces Spark's native error") { + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + |CREATE TABLE $tableName ( + | id INT, name STRING, price DOUBLE, ts INT + |) USING hudi + |LOCATION '${tmp.getCanonicalPath}' + |TBLPROPERTIES (primaryKey = 'id', preCombineField = 'ts') + """.stripMargin) + spark.sql(s"INSERT INTO $tableName VALUES (1, 'a1', 10.0, 1000)") + + val ex = intercept[AnalysisException] { + spark.sql( + s""" + |MERGE INTO $tableName AS target + |USING (SELECT 1 AS id, 'u' AS name, 20.0 AS price, 2000 AS ts) AS source + |ON target.nonexistent_id = source.id + |WHEN MATCHED THEN UPDATE SET * + |WHEN NOT MATCHED THEN INSERT * + """.stripMargin) + } + val msg = ex.getMessage + assertNativeUnresolvedColumn(msg, "nonexistent_id") + assertNoHudiGenericRewrite(msg) + } + } + + /** + * Assert the failure is Spark's native unresolved-column error and not a Hudi rewrite. + * + * On Spark 3.4+ we require the structured `UNRESOLVED_COLUMN` error-class token. That + * token is produced only by Spark's own `CheckAnalysis`; the pre-PR Hudi path rewrote + * the failure as a generic "Failed to resolve query ..." message that never carried it, + * so requiring the token here actually distinguishes the new behavior from the old (the + * looser "cannot be resolved" substring would have matched either way — see + * https://github.com/apache/hudi/pull/18147#discussion_r2795763747). On Spark 3.3, which + * predates error classes, the phrasing varies by code path — accept the legacy + * "cannot resolve" / "cannot be resolved" forms as well as "Column '...' does not exist" + * (what Spark 3.3 emits for the unresolved references in these queries). + * + * In all cases require the offending column name to appear, so we know the precise + * column was reported rather than some unrelated resolution failure. + */ + private def assertNativeUnresolvedColumn(msg: String, columnName: String): Unit = { + if (HoodieSparkUtils.gteqSpark3_4) { + assert(msg.contains("UNRESOLVED_COLUMN"), + s"Expected Spark's structured UNRESOLVED_COLUMN error class; got: $msg") + } else { + assert(msg.contains("cannot resolve") || msg.contains("cannot be resolved") || + msg.contains("does not exist"), + s"Expected Spark's native unresolved-column error; got: $msg") + } + assert(msg.contains(columnName), + s"Expected error to mention column '$columnName'; got: $msg") + } + + /** + * Assert the failure is Spark's native table-not-found error. On Spark 3.4+ require the + * structured `TABLE_OR_VIEW_NOT_FOUND` error-class token (same reasoning as + * [[assertNativeUnresolvedColumn]]); on Spark 3.3 fall back to the legacy phrasing. + */ + private def assertNativeTableNotFound(msg: String, tableName: String): Unit = { + if (HoodieSparkUtils.gteqSpark3_4) { + assert(msg.contains("TABLE_OR_VIEW_NOT_FOUND"), + s"Expected Spark's structured TABLE_OR_VIEW_NOT_FOUND error class; got: $msg") + } else { + assert(msg.toLowerCase.contains("table or view not found"), + s"Expected Spark's native table-not-found error; got: $msg") + } + assert(msg.contains(tableName), + s"Expected error to mention table '$tableName'; got: $msg") + } + + /** + * The earlier version of this PR caught UnresolvedException and rewrote it as + * "Failed to resolve query. The query contains unresolved columns or tables. + * Please check for: (1) typos ...". Make sure we no longer mask Spark's + * native message with that generic Hudi wrapper. + */ + private def assertNoHudiGenericRewrite(msg: String): Unit = { + assert(!msg.contains("Failed to resolve query"), + s"Hudi should not rewrap Spark's analysis error; got: $msg") + assert(!msg.contains("Please check for: (1) typos"), + s"Hudi should not rewrap Spark's analysis error; got: $msg") + } +} From 4ce7ef834e2dd50e9ac2cf8cad8a74f6394dfe6c Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Sat, 27 Jun 2026 13:09:27 -0700 Subject: [PATCH 108/255] [MINOR] Forward spark.hoodie.* SparkConf to write path (parity with read path) (#18650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [MINOR] Forward spark.hoodie.* SparkConf to writes (parity with reads) The read side of DefaultSource collects hoodie.* and spark.hoodie.* from SparkConf and strips the spark. prefix before handing to the write path. The write side dropped them entirely — configs set via --conf spark.hoodie.X=Y had no effect on writes (e.g. hoodie.datasource.hive_sync.use_spark_catalog). Extract collectHoodieAndSparkHoodieConfs + normalizeSparkHoodiePrefix in DataSourceOptionsHelper. Wire both into DefaultSource.createRelation (write) and into parametersWithWriteDefaults so callers bypassing the DefaultSource entry point (SQL ALTER TABLE, HoodieCLIUtils) get the same parity. Explicit .option(...) calls still win over SparkConf. Closes #18649 Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.7 Co-authored-by: sivabalan Co-authored-by: Rahil Chertara (cherry picked from commit 01f1496ff565381f95fbeb2096d0fa3c208c4fdc) --- .../org/apache/hudi/DataSourceOptions.scala | 88 +++++++++++- .../scala/org/apache/hudi/DefaultSource.scala | 22 +-- .../org/apache/hudi/HoodieWriterUtils.scala | 8 +- .../apache/hudi/TestDataSourceOptions.scala | 131 +++++++++++++++++- 4 files changed, 231 insertions(+), 18 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala index 445f052ee4f6a..b2548029e6216 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -35,6 +35,7 @@ import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory.{getKeyGene import org.apache.hudi.sync.common.HoodieSyncConfig import org.apache.hudi.util.{JFunction, SparkConfigUtils} +import org.apache.spark.sql.SQLContext import org.apache.spark.sql.execution.datasources.{DataSourceUtils => SparkDataSourceUtils} import org.apache.spark.sql.hudi.HoodieSqlCommonUtils import org.slf4j.LoggerFactory @@ -1033,9 +1034,87 @@ object DataSourceOptionsHelper { private val log = LoggerFactory.getLogger(DataSourceOptionsHelper.getClass) // Prefix constants for config normalization + private val HOODIE_PREFIX = "hoodie." private val SPARK_HOODIE_PREFIX = "spark.hoodie." private val SPARK_PREFIX = "spark." + /** + * Collects `hoodie.*` and `spark.hoodie.*` configs from the SparkConf, normalizes the + * `spark.hoodie.*` keys to canonical `hoodie.*`, and merges with explicit DataFrame + * options. Explicit options win over SparkConf. + * + * This is the read-path entry point: reads have always picked up session-level `hoodie.*` + * confs (e.g. `hoodie.datasource.query.type`), so both prefixes are forwarded here. + * Do NOT use this for writes — see `collectSparkHoodieConfs` for why ambient `hoodie.*` + * confs must not be forwarded to the write path. + * + * Example (SparkConf has both prefixes set; explicit options override): + * {{{ + * SparkConf: spark.hoodie.X = "a", hoodie.Y = "b" + * optParams: hoodie.X = "c" + * result: hoodie.X = "c" // explicit wins over both prefixes + * hoodie.Y = "b" + * }}} + */ + def collectHoodieAndSparkHoodieConfs(sqlContext: SQLContext, + optParams: Map[String, String]): Map[String, String] = + collectConfsByPrefix(sqlContext, optParams, includeHoodiePrefix = true) + + /** + * Collects only `spark.hoodie.*` configs from the SparkConf, normalizes them to canonical + * `hoodie.*`, and merges with explicit DataFrame options. Explicit options win over SparkConf. + * + * This is the write-path entry point. It deliberately does NOT forward bare `hoodie.*` + * session confs: unlike reads, the DataFrame write path historically honored only the + * explicit `.option(...)` map, so injecting ambient `hoodie.*` session state (e.g. a + * session-level `hoodie.datasource.write.operation` or `hoodie.logfile.data.block.format`) + * would silently change every `df.write`. The bug this addresses (HUDI-#18649) is about + * `--conf spark.hoodie.X=Y` being dropped on writes, which only requires forwarding the + * `spark.hoodie.*` form. + * + * Example: + * {{{ + * SparkConf: spark.hoodie.X = "a", hoodie.Y = "b" // bare hoodie.Y is NOT forwarded + * optParams: hoodie.Z = "c" + * result: hoodie.X = "a" + * hoodie.Z = "c" + * }}} + */ + def collectSparkHoodieConfs(sqlContext: SQLContext, + optParams: Map[String, String]): Map[String, String] = + collectConfsByPrefix(sqlContext, optParams, includeHoodiePrefix = false) + + private def collectConfsByPrefix(sqlContext: SQLContext, + optParams: Map[String, String], + includeHoodiePrefix: Boolean): Map[String, String] = { + val sparkConfs = sqlContext.getAllConfs.filter { + case (key, _) => + key.startsWith(SPARK_HOODIE_PREFIX) || (includeHoodiePrefix && key.startsWith(HOODIE_PREFIX)) + } + normalizeSparkHoodiePrefix(sparkConfs) ++ optParams + } + + /** + * Strips the `spark.` prefix from `spark.hoodie.*` keys so downstream code only sees + * canonical `hoodie.*` keys. If both `spark.hoodie.X` and `hoodie.X` are present, the + * latter wins (explicit options/configs override the SparkConf-prefixed form). + * + * The function is idempotent: running it on an already-normalized map is a no-op. + * Both `collectHoodieAndSparkHoodieConfs` (the entry-point helper) and + * `parametersWithReadDefaults` / `parametersWithWriteDefaults` (the per-path defaulting + * helpers) call it; this defense-in-depth ensures callers that bypass + * `collectHoodieAndSparkHoodieConfs` (e.g., SQL `ALTER TABLE` paths) still get + * normalized configs. + */ + def normalizeSparkHoodiePrefix(parameters: Map[String, String]): Map[String, String] = { + val rekeyedSparkHoodie = parameters.collect { + case (key, value) if key.startsWith(SPARK_HOODIE_PREFIX) => + (key.stripPrefix(SPARK_PREFIX), value) + } + val nonSparkHoodie = parameters.filterNot(_._1.startsWith(SPARK_HOODIE_PREFIX)) + rekeyedSparkHoodie ++ nonSparkHoodie + } + // put all the configs with alternatives here private val allConfigsWithAlternatives = List( DataSourceReadOptions.QUERY_TYPE, @@ -1132,13 +1211,8 @@ object DataSourceOptionsHelper { // 2) spark.hoodie.* (normalized to hoodie.*) // 3) hoodie.* / explicit data source options // NOTE: If both spark.hoodie.X and hoodie.X are set, hoodie.X wins. - val normalizedSparkHoodieConfigs = parameters.collect { - case (key, value) if key.startsWith(SPARK_HOODIE_PREFIX) => (key.stripPrefix(SPARK_PREFIX), value) - } - val paramsWithoutSparkHoodie = parameters.filterNot(_._1.startsWith(SPARK_HOODIE_PREFIX)) - val paramsWithGlobalProps = DFSPropertiesConfiguration.getGlobalProps.asScala.toMap ++ - normalizedSparkHoodieConfigs ++ - paramsWithoutSparkHoodie + val normalized = normalizeSparkHoodiePrefix(parameters) + val paramsWithGlobalProps = DFSPropertiesConfiguration.getGlobalProps.asScala.toMap ++ normalized val queryType = paramsWithGlobalProps.get(IS_QUERY_AS_RO_TABLE) .map(is => if (is.toBoolean) QUERY_TYPE_READ_OPTIMIZED_OPT_VAL else QUERY_TYPE_SNAPSHOT_OPT_VAL) .getOrElse(paramsWithGlobalProps.getOrElse(QUERY_TYPE.key, QUERY_TYPE.defaultValue())) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala index 2682de70f1fab..48fb989a2c9f9 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -107,17 +107,14 @@ class DefaultSource extends RelationProvider throw new HoodieException("Glob paths are not supported for read paths as of Hudi 1.2.0") } - val hoodieAndSparkHoodieSqlConfs = sqlContext.getAllConfs.filter { - case (key, _) => key.startsWith("hoodie.") || key.startsWith("spark.hoodie.") - } // Add default options for unspecified read options keys. // Effective precedence (low -> high): // 1) global DFS props - // 2) spark.hoodie.* SQL confs (normalized in parametersWithReadDefaults) + // 2) spark.hoodie.* SQL confs (normalized to hoodie.* in collectHoodieAndSparkHoodieConfs) // 3) hoodie.* SQL confs // 4) explicit DataFrame/DataSource options val parameters = DataSourceOptionsHelper.parametersWithReadDefaults( - hoodieAndSparkHoodieSqlConfs ++ optParams) + DataSourceOptionsHelper.collectHoodieAndSparkHoodieConfs(sqlContext, optParams)) // Get the table base path val tablePath = DataSourceUtils.getTablePath(storage, Seq(new StoragePath(path.get)).asJava) @@ -173,11 +170,20 @@ class DefaultSource extends RelationProvider mode: SaveMode, optParams: Map[String, String], df: DataFrame): BaseRelation = { + // Pull `spark.hoodie.*` from SparkConf, normalize to canonical `hoodie.*`, and merge + // with explicit options (explicit options win), so configs like + // `--conf spark.hoodie.datasource.hive_sync.use_spark_catalog=true` are honored on + // writes too. Unlike the read path, we deliberately do NOT forward bare `hoodie.*` + // session confs here: the DataFrame write path historically honored only the explicit + // `.option(...)` map, and injecting ambient session `hoodie.*` state would silently + // change every write. `HoodieSparkSqlWriter` and downstream callers see only canonical keys. + val effectiveOpts = + DataSourceOptionsHelper.collectSparkHoodieConfs(sqlContext, optParams) try { - if (optParams.get(OPERATION.key).contains(BOOTSTRAP_OPERATION_OPT_VAL)) { - HoodieSparkSqlWriter.bootstrap(sqlContext, mode, optParams, df) + if (effectiveOpts.get(OPERATION.key).contains(BOOTSTRAP_OPERATION_OPT_VAL)) { + HoodieSparkSqlWriter.bootstrap(sqlContext, mode, effectiveOpts, df) } else { - val (success, _, _, _, _, _) = HoodieSparkSqlWriter.write(sqlContext, mode, optParams, df) + val (success, _, _, _, _, _) = HoodieSparkSqlWriter.write(sqlContext, mode, effectiveOpts, df) if (!success) { throw new HoodieException("Failed to write to Hudi") } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala index e76af4a2733d8..54459ff61e00a 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala @@ -53,8 +53,12 @@ object HoodieWriterUtils { * Add default options for unspecified write options keys. */ def parametersWithWriteDefaults(parameters: Map[String, String]): Map[String, String] = { + // Strip the `spark.` prefix from `spark.hoodie.*` keys so write/hive_sync configs + // forwarded from SparkConf reach Hudi under their canonical names. Mirrors what + // parametersWithReadDefaults does for the read path. + val normalizedParams = DataSourceOptionsHelper.normalizeSparkHoodiePrefix(parameters) val globalProps = DFSPropertiesConfiguration.getGlobalProps.asScala - val props = TypedProperties.fromMap(parameters.asJava) + val props = TypedProperties.fromMap(normalizedParams.asJava) val hoodieConfig: HoodieConfig = new HoodieConfig(props) hoodieConfig.setDefaultValue(OPERATION) hoodieConfig.setDefaultValue(TABLE_TYPE) @@ -87,7 +91,7 @@ object HoodieWriterUtils { hoodieConfig.setDefaultValue(RECONCILE_SCHEMA) hoodieConfig.setDefaultValue(DROP_PARTITION_COLUMNS) hoodieConfig.setDefaultValue(KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED) - Map() ++ hoodieConfig.getProps.asScala ++ globalProps ++ DataSourceOptionsHelper.translateConfigurations(parameters) + Map() ++ hoodieConfig.getProps.asScala ++ globalProps ++ DataSourceOptionsHelper.translateConfigurations(normalizedParams) } /** diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala index 20d61973f213b..7dfce2faf5714 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala @@ -22,9 +22,11 @@ package org.apache.hudi import org.apache.hudi.common.config.{DFSPropertiesConfiguration, HoodieCommonConfig} import org.apache.hudi.common.table.HoodieTableConfig +import org.apache.spark.sql.SQLContext import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.Test +import org.mockito.Mockito.{mock, when} class TestDataSourceOptions { @Test @@ -91,6 +93,133 @@ class TestDataSourceOptions { assertEquals(DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL, params3(DataSourceReadOptions.QUERY_TYPE.key)) } + @Test + def testNormalizeSparkHoodiePrefixStripsSparkPrefix(): Unit = { + val result = DataSourceOptionsHelper.normalizeSparkHoodiePrefix(Map( + "spark.hoodie.datasource.query.type" -> "snapshot", + "spark.hoodie.datasource.hive_sync.use_spark_catalog" -> "true", + "non.hoodie.key" -> "ignored" + )) + + assertEquals("snapshot", result("hoodie.datasource.query.type")) + assertEquals("true", result("hoodie.datasource.hive_sync.use_spark_catalog")) + assertEquals("ignored", result("non.hoodie.key")) + assertFalse(result.contains("spark.hoodie.datasource.query.type")) + assertFalse(result.contains("spark.hoodie.datasource.hive_sync.use_spark_catalog")) + } + + @Test + def testNormalizeSparkHoodiePrefixPrefersHoodieOverSparkHoodie(): Unit = { + val result = DataSourceOptionsHelper.normalizeSparkHoodiePrefix(Map( + "spark.hoodie.datasource.query.type" -> DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL, + "hoodie.datasource.query.type" -> DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL + )) + + assertEquals(DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL, + result("hoodie.datasource.query.type")) + assertFalse(result.contains("spark.hoodie.datasource.query.type")) + } + + @Test + def testNormalizeSparkHoodiePrefixIsIdempotent(): Unit = { + val once = DataSourceOptionsHelper.normalizeSparkHoodiePrefix(Map( + "spark.hoodie.datasource.query.type" -> "snapshot", + "hoodie.other.key" -> "v" + )) + val twice = DataSourceOptionsHelper.normalizeSparkHoodiePrefix(once) + + assertEquals(once, twice) + } + + @Test + def testCollectHoodieAndSparkHoodieConfsReturnsCanonicalKeys(): Unit = { + val sqlContext = mock(classOf[SQLContext]) + when(sqlContext.getAllConfs).thenReturn(Map( + "spark.hoodie.datasource.query.type" -> DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL, + "hoodie.datasource.write.operation" -> "upsert", + "spark.sql.shuffle.partitions" -> "200" // non-hoodie, must be filtered out + )) + + val result = DataSourceOptionsHelper.collectHoodieAndSparkHoodieConfs(sqlContext, Map.empty) + + assertEquals(DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL, + result("hoodie.datasource.query.type")) + assertEquals("upsert", result("hoodie.datasource.write.operation")) + assertFalse(result.contains("spark.hoodie.datasource.query.type")) + assertFalse(result.contains("spark.sql.shuffle.partitions")) + } + + @Test + def testCollectHoodieAndSparkHoodieConfsExplicitOptionsWin(): Unit = { + val sqlContext = mock(classOf[SQLContext]) + when(sqlContext.getAllConfs).thenReturn(Map( + "spark.hoodie.datasource.write.operation" -> "insert", + "hoodie.datasource.write.precombine.field" -> "ts_from_conf" + )) + + val result = DataSourceOptionsHelper.collectHoodieAndSparkHoodieConfs(sqlContext, Map( + "hoodie.datasource.write.operation" -> "upsert", // explicit overrides spark.hoodie.* + "hoodie.datasource.write.precombine.field" -> "ts_from_options" // explicit overrides hoodie.* + )) + + assertEquals("upsert", result("hoodie.datasource.write.operation")) + assertEquals("ts_from_options", result("hoodie.datasource.write.precombine.field")) + } + + @Test + def testCollectSparkHoodieConfsForwardsOnlySparkPrefix(): Unit = { + val sqlContext = mock(classOf[SQLContext]) + when(sqlContext.getAllConfs).thenReturn(Map( + "spark.hoodie.datasource.hive_sync.use_spark_catalog" -> "true", // forwarded (the --conf use case) + "hoodie.logfile.data.block.format" -> "parquet", // bare hoodie.* must NOT leak into writes + "hoodie.datasource.write.operation" -> "bulk_insert", // bare hoodie.* must NOT leak into writes + "spark.sql.shuffle.partitions" -> "200" // non-hoodie, filtered out + )) + + val result = DataSourceOptionsHelper.collectSparkHoodieConfs(sqlContext, Map.empty) + + assertEquals("true", result("hoodie.datasource.hive_sync.use_spark_catalog")) + assertFalse(result.contains("hoodie.logfile.data.block.format")) + assertFalse(result.contains("hoodie.datasource.write.operation")) + assertFalse(result.contains("spark.sql.shuffle.partitions")) + } + + @Test + def testCollectSparkHoodieConfsExplicitOptionsWin(): Unit = { + val sqlContext = mock(classOf[SQLContext]) + when(sqlContext.getAllConfs).thenReturn(Map( + "spark.hoodie.datasource.write.operation" -> "insert" + )) + + val result = DataSourceOptionsHelper.collectSparkHoodieConfs(sqlContext, Map( + "hoodie.datasource.write.operation" -> "upsert" // explicit overrides spark.hoodie.* + )) + + assertEquals("upsert", result("hoodie.datasource.write.operation")) + assertFalse(result.contains("spark.hoodie.datasource.write.operation")) + } + + @Test + def testWriteDefaultsSupportSparkHoodieConfigs(): Unit = { + val params = HoodieWriterUtils.parametersWithWriteDefaults(Map( + "spark.hoodie.datasource.write.operation" -> "upsert" + )) + + assertEquals("upsert", params(DataSourceWriteOptions.OPERATION.key)) + assertFalse(params.contains("spark.hoodie.datasource.write.operation")) + } + + @Test + def testWriteDefaultsPreferHoodieOverSparkHoodieWhenBothSet(): Unit = { + val params = HoodieWriterUtils.parametersWithWriteDefaults(Map( + "spark.hoodie.datasource.write.operation" -> "insert", + "hoodie.datasource.write.operation" -> "upsert" + )) + + assertEquals("upsert", params(DataSourceWriteOptions.OPERATION.key)) + assertFalse(params.contains("spark.hoodie.datasource.write.operation")) + } + @AfterEach def cleanup(): Unit = { DFSPropertiesConfiguration.clearGlobalProps() From a29a48831df970f911b7b1dc71247dad97673983 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Mon, 29 Jun 2026 22:29:37 +0800 Subject: [PATCH 109/255] fix(partition-ttl): Fix the integer overflow issue when TTL exceeded 24 days (#19075) (cherry picked from commit d4d7e20771b1bf3ec20c11346452a5d2640523a5) --- .../ttl/strategy/KeepByTimeStrategy.java | 3 +- .../strategy/TestPartitionTTLStrategy.java | 82 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategy.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java index 7f07c4b735d2b..329fee22cf60d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java @@ -31,6 +31,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator.fixInstantTimeCompatibility; @@ -46,7 +47,7 @@ public class KeepByTimeStrategy extends PartitionTTLStrategy { public KeepByTimeStrategy(HoodieTable hoodieTable, String instantTime) { super(hoodieTable, instantTime); - this.ttlInMilis = writeConfig.getPartitionTTLStrategyDaysRetain() * 1000 * 3600 * 24; + this.ttlInMilis = TimeUnit.DAYS.toMillis(writeConfig.getPartitionTTLStrategyDaysRetain()); } @Override diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategy.java new file mode 100644 index 0000000000000..4cad31289ec40 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategy.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.ttl.strategy; + +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestPartitionTTLStrategy { + + private static final long MILLIS_PER_DAY = 24L * 3600L * 1000L; + + private KeepByTimeStrategy newStrategy(int daysRetain) { + HoodieTable hoodieTable = mock(HoodieTable.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(hoodieTable.getConfig()).thenReturn(writeConfig); + when(writeConfig.getPartitionTTLStrategyDaysRetain()).thenReturn(daysRetain); + return new KeepByTimeStrategy(hoodieTable, ""); + } + + /** + * Verify that ttlInMilis is computed in long arithmetic, so values that would + * overflow a 32-bit int multiplication (daysRetain > 24) still yield the + * correct positive millisecond value. + * + *

    Without the {@code (long)} cast on {@code getPartitionTTLStrategyDaysRetain()}, + * the expression {@code daysRetain * 1000 * 3600 * 24} is evaluated as int and + * overflows once daysRetain * 86_400_000 exceeds Integer.MAX_VALUE + * (i.e., for any daysRetain >= 25). + */ + @Test + public void testKeepByTimeStrategyTTLInMilis() { + // Small value: no overflow either way, sanity check. + assertEquals(1L * MILLIS_PER_DAY, newStrategy(1).ttlInMilis); + + // Largest value that does NOT overflow int: 24 * 86_400_000 = 2_073_600_000. + assertEquals(24L * MILLIS_PER_DAY, newStrategy(24).ttlInMilis); + + // 25 days: 25 * 86_400_000 = 2_160_000_000, exceeds Integer.MAX_VALUE. + // With the (long) cast the result is correct; without it the int expression + // would overflow to a negative value. + int days = 25; + long expected = (long) days * MILLIS_PER_DAY; + assertEquals(expected, newStrategy(days).ttlInMilis); + assertTrue(newStrategy(days).ttlInMilis > 0, + "ttlInMilis must stay positive for daysRetain beyond the int-overflow boundary"); + // Guard against regression: the unfixed int expression would produce this + // (overflowed) value. The fixed code must NOT match it. + int overflowed = days * 1000 * 3600 * 24; + assertEquals(-2_134_967_296, overflowed, "sanity: confirm the int expression overflows for 25 days"); + assertTrue(newStrategy(days).ttlInMilis != overflowed, + "ttlInMilis must not equal the int-overflowed value"); + + // A clearly large value (e.g., 365 days) to ensure long arithmetic holds. + assertEquals(365L * MILLIS_PER_DAY, newStrategy(365).ttlInMilis); + + // Zero / disabled. + assertEquals(0L, newStrategy(0).ttlInMilis); + } +} From 4a2c490dfdefc9e7ab245de495dbafe7675903e9 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 30 Jun 2026 09:27:55 +0800 Subject: [PATCH 110/255] perf(common): Avoid per-record HoodieSchema rebuilds on Avro read/merge paths (#18967) * perf(common): Avoid per-call HoodieSchema rebuild in AvroRecordContext field access getFieldValueFromIndexedRecord wrapped record.getSchema() in a fresh HoodieSchema on every call, which rebuilt the full field list and field map (one HoodieSchemaField per column plus a HashMap collect) and split the field path, per record per accessed field in the file group reader merge path. Intern the wrapper through HoodieSchemaCache instead, so the canonical instance's lazily built field list and field map are reused across calls and the per-record cost drops to a cache hit. Single-segment field names, the overwhelmingly common case, also skip the path split. Lookup semantics are unchanged since the traversal still goes through HoodieSchema#getNonNullType and #getField, keeping HoodieSchema as the type system facade. Since interned instances are shared across executor task threads, HoodieSchema's lazily built field list and field map are now published through immutable wrappers (final-field freeze) so a racing reader can never observe a non-null map with invisible entries and silently miss an existing field. * review: Guard lazy field list/field map initialization with double-checked locking Addresses review feedback on the safe-publication change: make the cache fields volatile and take the monitor only on the miss path, so reads stay lock-free on the hot path while initialization is lock-guarded (no duplicate builds). Same pattern as org.apache.hudi.common.util.Lazy#get. * review: Drop the single-segment fast path String.split already fast-paths the two-character pattern, so after interning the fast path only saved one small array allocation per call; not worth the extra branch. * review: Drop the DCL local variables The cache fields are volatile and write-once, so reading them directly after the null check is safe; the locals only saved a volatile re-read. * review: Add Avro-schema-keyed intern overload so the hot path skips wrapper construction HoodieSchema.fromAvroSchema still ran per record to build the intern probe key. HoodieSchemaCache.intern(Schema) is backed by a weak identity-keyed cache: records of one file share the same Avro Schema instance, so the per-record path becomes a single cache hit with no wrapper allocation or type dispatch. Misses convert and value-intern, so equal but distinct Avro schema instances still converge on one canonical HoodieSchema. * refactor(common): Extract Avro-schema-keyed cache into AvroToHoodieSchemaCache Move the Avro Schema -> HoodieSchema cache out of HoodieSchemaCache into a dedicated AvroToHoodieSchemaCache class; misses still value-intern through HoodieSchemaCache. AvroRecordContext now uses the new class. HoodieSchemaCache is back to interning HoodieSchema only. * review: Drop synchronized DCL for lazy fields in favor of a benign racy single-check getFields()/getFieldMap() build an immutable, deterministic view of the schema's fields, so concurrent first-callers can each build it once and converge on equal results. Keep the volatile fields so the unmodifiable collections (wrapping non-final ArrayList/HashMap) are still published safely; drop the synchronized blocks. * perf(common): Intern HoodieSchema at remaining per-record fromAvroSchema sites Audit of all HoodieSchema.fromAvroSchema(...) call sites for per-record rebuilds (follow-up to the AvroRecordContext change). Switch the genuinely per-record sites to AvroToHoodieSchemaCache.intern(...): - SparkFileFormatInternalRecordContext.convertAvroRecord - FlinkRecordContext.convertAvroRecord - RealtimeCompactedRecordReader.mergeRecord (two calls) - HoodieAvroUtils.getRecordColumnValues - HoodieJsonPayload.getInsertValue - ExpressionPayload MERGE-INTO eval paths And hoist the loop-invariant fromAvroSchema(schema) out of the per-record write loop in HoodieAvroDataBlock#getBytes. Interning returns an equal canonical HoodieSchema and improves downstream schema-keyed cache hit rates; cold/one-time and per-block sites are left unchanged. * review(common): correct lazy field-cache comment (benign race, not DCL) * review(common): intern the loop-invariant HoodieSchema in getBytes Switch the hoisted HoodieSchema.fromAvroSchema(schema) in getBytes to AvroToHoodieSchemaCache.intern(schema). This matches every other site touched in this PR and reuses one cached, value-interned instance across getBytes calls, keeping HoodieSchema identity stable for the downstream per-record caches instead of rebuilding per block. * review(common): rename AvroToHoodieSchemaCache to HoodieAvroSchemaCache Match the Hoodie-prefix convention used by every other class in the org.apache.hudi.common.schema package (HoodieSchema, HoodieSchemaCache, HoodieSchemaField, ...). Pure rename of the class and its 8 referencing files; no behavior change. (cherry picked from commit 57e557535b514723139fd9e5493b94b888456dac) --- .../hudi/table/format/FlinkRecordContext.java | 3 +- ...SparkFileFormatInternalRecordContext.scala | 4 +- .../apache/hudi/avro/AvroRecordContext.java | 6 +- .../org/apache/hudi/avro/HoodieAvroUtils.java | 3 +- .../apache/hudi/common/HoodieJsonPayload.java | 5 +- .../common/schema/HoodieAvroSchemaCache.java | 52 ++++++++++++++ .../hudi/common/schema/HoodieSchema.java | 14 ++-- .../table/log/block/HoodieAvroDataBlock.java | 5 +- .../hudi/avro/TestAvroRecordContext.java | 68 +++++++++++++++++++ .../RealtimeCompactedRecordReader.java | 5 +- .../command/payload/ExpressionPayload.scala | 14 ++-- 11 files changed, 157 insertions(+), 22 deletions(-) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieAvroSchemaCache.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java index d242dcbfb9d60..e2406b829693e 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.table.HoodieTableConfig; @@ -125,7 +126,7 @@ public RowData getDeleteRow(String recordKey) { @Override public RowData convertAvroRecord(IndexedRecord avroRecord) { Schema recordSchema = avroRecord.getSchema(); - AvroToRowDataConverters.AvroToRowDataConverter converter = RowDataQueryContexts.fromSchema(HoodieSchema.fromAvroSchema(recordSchema), utcTimezone).getAvroToRowDataConverter(); + AvroToRowDataConverters.AvroToRowDataConverter converter = RowDataQueryContexts.fromSchema(HoodieAvroSchemaCache.intern(recordSchema), utcTimezone).getAvroToRowDataConverter(); RowData rowData = (RowData) converter.convert(avroRecord); Schema.Field operationField = recordSchema.getField(HoodieRecord.OPERATION_METADATA_FIELD); if (operationField != null) { diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRecordContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRecordContext.scala index 3a4cf4642bb8e..edf0a4eee3dbd 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRecordContext.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRecordContext.scala @@ -21,7 +21,7 @@ package org.apache.hudi import org.apache.avro.generic.{GenericRecord, IndexedRecord} import org.apache.hudi.common.engine.RecordContext -import org.apache.hudi.common.schema.HoodieSchema +import org.apache.hudi.common.schema.{HoodieAvroSchemaCache, HoodieSchema} import org.apache.hudi.common.table.HoodieTableConfig import org.apache.spark.sql.HoodieInternalRowUtils import org.apache.spark.sql.avro.{HoodieAvroDeserializer, HoodieAvroSerializer} @@ -47,7 +47,7 @@ trait SparkFileFormatInternalRecordContext extends BaseSparkInternalRecordContex * @return An [[InternalRow]]. */ override def convertAvroRecord(avroRecord: IndexedRecord): InternalRow = { - val schema = HoodieSchema.fromAvroSchema(avroRecord.getSchema) + val schema = HoodieAvroSchemaCache.intern(avroRecord.getSchema) val structType = HoodieInternalRowUtils.getCachedSchema(schema) val deserializer = deserializerMap.getOrElseUpdate(schema, { sparkAdapter.createAvroDeserializer(schema, structType) diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java index def6e6a7003fc..fceeeaf84b97a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieEmptyRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.table.HoodieTableConfig; @@ -70,7 +71,10 @@ public AvroRecordContext() { public static Object getFieldValueFromIndexedRecord( IndexedRecord record, String fieldName) { - HoodieSchema currentSchema = HoodieSchema.fromAvroSchema(record.getSchema()); + // Interning returns the canonical wrapper for this schema, whose lazily built field list and + // field map survive across calls, so the per-record cost is a cache hit instead of an + // O(schema width) wrapper rebuild. + HoodieSchema currentSchema = HoodieAvroSchemaCache.intern(record.getSchema()); IndexedRecord currentRecord = record; String[] path = fieldName.split("\\."); for (int i = 0; i < path.length; i++) { diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java index 7c55f441d2009..c52960c4abb06 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java @@ -19,6 +19,7 @@ package org.apache.hudi.avro; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.util.DateTimeUtils; @@ -834,7 +835,7 @@ public static Object[] getRecordColumnValues(HoodieRecord record, Schema schema, boolean consistentLogicalTimestampEnabled) { try { - GenericRecord genericRecord = (GenericRecord) (record.toIndexedRecord(HoodieSchema.fromAvroSchema(schema), new Properties()).get()).getData(); + GenericRecord genericRecord = (GenericRecord) (record.toIndexedRecord(HoodieAvroSchemaCache.intern(schema), new Properties()).get()).getData(); List list = new ArrayList<>(); for (String col : columns) { list.add(HoodieAvroUtils.getNestedFieldVal(genericRecord, col, true, consistentLogicalTimestampEnabled)); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java index e6667384f582b..e89b10cccf283 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java @@ -20,8 +20,7 @@ import org.apache.hudi.avro.MercifulJsonConverter; import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.schema.HoodieSchema; -import org.apache.hudi.io.util.FileIOUtils; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; @@ -65,7 +64,7 @@ public Option combineAndGetUpdateValue(IndexedRecord oldRec, Sche @Override public Option getInsertValue(Schema schema) throws IOException { MercifulJsonConverter jsonConverter = new MercifulJsonConverter(); - return Option.of(jsonConverter.convert(getJsonData(), HoodieSchema.fromAvroSchema(schema))); + return Option.of(jsonConverter.convert(getJsonData(), HoodieAvroSchemaCache.intern(schema))); } private String getJsonData() throws IOException { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieAvroSchemaCache.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieAvroSchemaCache.java new file mode 100644 index 0000000000000..8b153eb4487e4 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieAvroSchemaCache.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.schema; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import org.apache.avro.Schema; + +/** + * A global cache mapping Avro {@link Schema} instances to their canonical {@link HoodieSchema}. + * + *

    This is an Avro-schema-keyed view onto {@link HoodieSchemaCache} for per-record call sites: + * {@code weakKeys} gives identity-based lookups (records of one file share the same {@link Schema} + * instance), so the hot path is a single cache hit with no wrapper allocation or type dispatch. + * Misses convert and then value-intern through {@link HoodieSchemaCache}, so equal but distinct Avro + * schema instances still converge on one canonical {@link HoodieSchema}. + * + *

    This is a global cache which works for a JVM lifecycle. + */ +public class HoodieAvroSchemaCache { + + private static final LoadingCache AVRO_SCHEMA_CACHE = + Caffeine.newBuilder().weakKeys().maximumSize(1024) + .build(avroSchema -> HoodieSchemaCache.intern(HoodieSchema.fromAvroSchema(avroSchema))); + + /** + * Returns the canonical {@link HoodieSchema} wrapping the given Avro schema, converting and + * interning it on first use. + * + * @param avroSchema Avro schema to look up + * @return the canonical HoodieSchema for the given Avro schema + */ + public static HoodieSchema intern(Schema avroSchema) { + return AVRO_SCHEMA_CACHE.get(avroSchema); + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java index 0b0978d552a23..f38b9e2c93ed1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java @@ -313,8 +313,11 @@ private static void addVectorColumnName(String s, int start, int end, Set fields; - private transient Map fieldMap; + // interned instances are shared across threads, so the lazily built caches use a benign racy + // single-check (see getFields()/getFieldMap()): lock-free volatile reads, and volatile gives + // safe publication of the immutable, deterministic result + private transient volatile List fields; + private transient volatile Map fieldMap; // Register the Variant logical type with Avro static { @@ -1152,6 +1155,8 @@ public List getFields() { if (!hasFields()) { throw new IllegalStateException("Cannot get fields from schema type: " + type); } + // Benign race: the result is an immutable, deterministic view of avroSchema's fields, so concurrent + // callers may each build it once but converge on equal lists; the volatile field makes publication safe. if (fields == null) { fields = Collections.unmodifiableList(avroSchema.getFields().stream().map(HoodieSchemaField::new).collect(Collectors.toList())); } @@ -1195,9 +1200,10 @@ public Option getField(String name) { } private Map getFieldMap() { + // Benign race, same rationale as getFields(): deterministic immutable result, volatile for safe publication. if (fieldMap == null) { - fieldMap = getFields().stream() - .collect(Collectors.toMap(HoodieSchemaField::name, field -> field)); + fieldMap = Collections.unmodifiableMap(getFields().stream() + .collect(Collectors.toMap(HoodieSchemaField::name, field -> field))); } return fieldMap; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java index 9f265d4f26878..764528bfc3f06 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaCache; import org.apache.hudi.common.util.CollectionUtils; @@ -507,9 +508,11 @@ public byte[] getBytes(Schema schema) throws IOException { output.writeInt(records.size()); // 3. Write the records + // schema is loop-invariant; intern it once (shared, cached) instead of rebuilding the HoodieSchema per record + HoodieSchema hoodieSchema = HoodieAvroSchemaCache.intern(schema); Iterator> itr = records.iterator(); while (itr.hasNext()) { - IndexedRecord s = itr.next().toIndexedRecord(HoodieSchema.fromAvroSchema(schema), new Properties()).get().getData(); + IndexedRecord s = itr.next().toIndexedRecord(hoodieSchema, new Properties()).get().getData(); ByteArrayOutputStream temp = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(temp, encoderCache.get()); encoderCache.set(encoder); diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java index b84738684b16c..04c7ae3c2bda7 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java @@ -19,14 +19,21 @@ package org.apache.hudi.avro; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.apache.avro.util.Utf8; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.util.stream.Stream; +import static org.apache.hudi.avro.AvroRecordContext.getFieldValueFromIndexedRecord; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; class TestAvroRecordContext { @@ -44,4 +51,65 @@ void testConvertValueToEngineType(Comparable input, Comparable expected) { Comparable actual = AvroRecordContext.getFieldAccessorInstance().convertValueToEngineType(input); assertEquals(expected, actual); } + + private static final Schema RECORD_SCHEMA = new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"top\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"int\"}," + + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"address\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"address\",\"fields\":[" + + "{\"name\":\"city\",\"type\":\"string\"}," + + "{\"name\":\"zip\",\"type\":[\"null\",\"int\"],\"default\":null}]}],\"default\":null}," + + "{\"name\":\"multi\",\"type\":[\"null\",\"string\",\"int\"],\"default\":null}]}"); + + private static GenericRecord buildRecord() { + GenericRecord address = new GenericData.Record(RECORD_SCHEMA.getField("address").schema().getTypes().get(1)); + address.put("city", new Utf8("sf")); + address.put("zip", 94105); + GenericRecord record = new GenericData.Record(RECORD_SCHEMA); + record.put("id", 1); + record.put("name", new Utf8("alice")); + record.put("address", address); + return record; + } + + @Test + void testGetFieldValueTopLevel() { + GenericRecord record = buildRecord(); + assertEquals(1, getFieldValueFromIndexedRecord(record, "id")); + assertEquals(new Utf8("alice"), getFieldValueFromIndexedRecord(record, "name")); + assertNull(getFieldValueFromIndexedRecord(record, "multi")); + assertNull(getFieldValueFromIndexedRecord(record, "missing")); + } + + @Test + void testGetFieldValueNested() { + GenericRecord record = buildRecord(); + // intermediate segment unwraps the [null, record] union + assertEquals(new Utf8("sf"), getFieldValueFromIndexedRecord(record, "address.city")); + assertEquals(94105, getFieldValueFromIndexedRecord(record, "address.zip")); + assertNull(getFieldValueFromIndexedRecord(record, "address.missing")); + assertNull(getFieldValueFromIndexedRecord(record, "missing.nested")); + } + + @Test + void testGetFieldValueErrorCases() { + GenericRecord record = buildRecord(); + // a union that is not [null, T] does not support field lookups + assertThrows(IllegalStateException.class, () -> getFieldValueFromIndexedRecord(record, "multi.sub")); + assertThrows(IllegalArgumentException.class, () -> getFieldValueFromIndexedRecord(record, "")); + } + + @Test + void testGetFieldValueAcrossEqualSchemaInstances() { + // records from different files carry equal but distinct schema instances; both must intern to + // the same canonical wrapper and resolve identically + Schema schemaCopy = new Schema.Parser().parse(RECORD_SCHEMA.toString()); + GenericRecord record = buildRecord(); + GenericRecord recordWithCopy = new GenericData.Record(schemaCopy); + for (Schema.Field field : RECORD_SCHEMA.getFields()) { + recordWithCopy.put(field.pos(), record.get(field.pos())); + } + assertEquals(getFieldValueFromIndexedRecord(record, "id"), getFieldValueFromIndexedRecord(recordWithCopy, "id")); + assertEquals(getFieldValueFromIndexedRecord(record, "address.city"), getFieldValueFromIndexedRecord(recordWithCopy, "address.city")); + } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java index 8d0d1765eb5e7..c68b984e5686c 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieAvroRecordMerger; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.read.BufferedRecord; @@ -204,8 +205,8 @@ private Option mergeRecord(HoodieRecord newRecord, A // once presto on hudi have its own mor reader, we can remove the rewrite logical. GenericRecord genericRecord = HiveAvroSerializer.rewriteRecordIgnoreResultCheck(oldRecord, getLogScannerReaderSchema()); RecordContext recordContext = AvroRecordContext.getFieldAccessorInstance(); - BufferedRecord record = BufferedRecords.fromEngineRecord(genericRecord, HoodieSchema.fromAvroSchema(genericRecord.getSchema()), recordContext, orderingFields, newRecord.getRecordKey(), false); - BufferedRecord newBufferedRecord = BufferedRecords.fromHoodieRecord(newRecord, HoodieSchema.fromAvroSchema(getLogScannerReaderSchema().toAvroSchema()), + BufferedRecord record = BufferedRecords.fromEngineRecord(genericRecord, HoodieAvroSchemaCache.intern(genericRecord.getSchema()), recordContext, orderingFields, newRecord.getRecordKey(), false); + BufferedRecord newBufferedRecord = BufferedRecords.fromHoodieRecord(newRecord, HoodieAvroSchemaCache.intern(getLogScannerReaderSchema().toAvroSchema()), recordContext, payloadProps, orderingFields, deleteContext); BufferedRecord mergeResult = merger.merge(record, newBufferedRecord, recordContext, payloadProps); if (mergeResult.isDelete()) { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala index 7fab437b294f9..a33fe583a6dbf 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala @@ -22,7 +22,7 @@ import org.apache.hudi.HoodieSchemaConversionUtils.{convertHoodieSchemaToDataTyp import org.apache.hudi.SparkAdapterSupport.sparkAdapter import org.apache.hudi.avro.HoodieAvroUtils import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodiePayloadProps, HoodieRecord, HoodieRecordPayload, OverwriteWithLatestAvroPayload} -import org.apache.hudi.common.schema.{HoodieSchema, HoodieSchemaUtils} +import org.apache.hudi.common.schema.{HoodieAvroSchemaCache, HoodieSchema, HoodieSchemaUtils} import org.apache.hudi.common.util.{BinaryUtil, ConfigUtils, HoodieRecordUtils, Option => HOption, OrderingValues, StringUtils, ValidationUtils} import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.config.HoodieWriteConfig @@ -116,7 +116,7 @@ class ExpressionPayload(@transient record: GenericRecord, // Get the Evaluator for each condition and update assignments. val updateConditionAndAssignments = - getEvaluator(updateConditionAndAssignmentsText.toString, HoodieSchema.fromAvroSchema(inputRecord.asAvro.getSchema)) + getEvaluator(updateConditionAndAssignmentsText.toString, HoodieAvroSchemaCache.intern(inputRecord.asAvro.getSchema)) for ((conditionEvaluator, assignmentEvaluator) <- updateConditionAndAssignments if resultRecordOpt == null) { @@ -145,7 +145,7 @@ class ExpressionPayload(@transient record: GenericRecord, // Process delete val deleteConditionText = properties.get(ExpressionPayload.PAYLOAD_DELETE_CONDITION) if (deleteConditionText != null) { - val (deleteConditionEvaluator, _) = getEvaluator(deleteConditionText.toString, HoodieSchema.fromAvroSchema(inputRecord.asAvro.getSchema)).head + val (deleteConditionEvaluator, _) = getEvaluator(deleteConditionText.toString, HoodieAvroSchemaCache.intern(inputRecord.asAvro.getSchema)).head val deleteConditionEvalResult = deleteConditionEvaluator.apply(inputRecord.asRow) .get(0, BooleanType) .asInstanceOf[Boolean] @@ -206,7 +206,7 @@ class ExpressionPayload(@transient record: GenericRecord, * multiple times for different expression evaluation invocations */ case class ConvertibleRecord(private val avro: GenericRecord) extends Logging { - private lazy val row: InternalRow = getAvroDeserializerFor(HoodieSchema.fromAvroSchema(avro.getSchema)).deserialize(avro) match { + private lazy val row: InternalRow = getAvroDeserializerFor(HoodieAvroSchemaCache.intern(avro.getSchema)).deserialize(avro) match { case Some(row) => row.asInstanceOf[InternalRow] case None => logError(s"Failed to deserialize Avro record `${avro.toString}` as Catalyst row") @@ -231,7 +231,7 @@ class ExpressionPayload(@transient record: GenericRecord, properties.get(ExpressionPayload.PAYLOAD_INSERT_CONDITION_AND_ASSIGNMENTS).toString // Get the evaluator for each condition and insert assignment. val insertConditionAndAssignments = - ExpressionPayload.getEvaluator(insertConditionAndAssignmentsText, HoodieSchema.fromAvroSchema(inputRecord.asAvro.getSchema)) + ExpressionPayload.getEvaluator(insertConditionAndAssignmentsText, HoodieAvroSchemaCache.intern(inputRecord.asAvro.getSchema)) var resultRecordOpt: HOption[IndexedRecord] = null for ((conditionEvaluator, assignmentEvaluator) <- insertConditionAndAssignments if resultRecordOpt == null) { @@ -243,7 +243,7 @@ class ExpressionPayload(@transient record: GenericRecord, if (conditionEvalResult) { val writerSchema = getWriterSchema(properties, false) val resultingRow = assignmentEvaluator.apply(inputRecord.asRow) - val resultingAvroRecord = getAvroSerializerFor(HoodieSchema.fromAvroSchema(writerSchema.getAvroSchema)) + val resultingAvroRecord = getAvroSerializerFor(HoodieAvroSchemaCache.intern(writerSchema.getAvroSchema)) .serialize(resultingRow) .asInstanceOf[GenericRecord] @@ -315,7 +315,7 @@ class ExpressionPayload(@transient record: GenericRecord, */ private def joinRecord(sourceRecord: IndexedRecord, targetRecord: IndexedRecord, props: Properties): GenericRecord = { val leftSchema = sourceRecord.getSchema - val joinSchema = getMergedSchema(HoodieSchema.fromAvroSchema(leftSchema), HoodieSchema.fromAvroSchema(targetRecord.getSchema)) + val joinSchema = getMergedSchema(HoodieAvroSchemaCache.intern(leftSchema), HoodieAvroSchemaCache.intern(targetRecord.getSchema)) // TODO rebase onto JoinRecord val values = new Array[AnyRef](joinSchema.getFields.size()) From d35272a7447c74156bcd09814560e07c801b1743 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 30 Jun 2026 08:30:52 +0700 Subject: [PATCH 111/255] test(flink): de-flake testLookupJoin lookup-join IT (#19093) The streaming lookup join in ITTestHoodieDataSource.testLookupJoin reads its dimension table through HoodieLookupFunction, whose cache is populated lazily on the first probe row. A teardown / commit-visibility race can occasionally make the join emit no rows; execInsertSql awaits the insert but swallows the exception, so the empty result is committed to t2 and the read then asserts against an empty table. Both tables use uuid as the record key, so re-running the insert is an idempotent upsert. Retry the insert-and-read up to MAX_STREAM_READ_ATTEMPTS until the expected rows materialize, mirroring the existing stream-read de-flake pattern (submitAndFetchWithRetry). Co-authored-by: Vova Kolmakov (cherry picked from commit 848d33e11823f1c6bcfc237d44229315e7781d30) --- .../hudi/table/ITTestHoodieDataSource.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 399829f94189e..148e14f30b2f7 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -1198,9 +1198,22 @@ void testLookupJoin(HoodieTableType tableType, String cacheType, boolean async) + " join t1/*+ OPTIONS('lookup.join.cache.ttl'='2 day', 'lookup.async'='" + async + "'," + " 'lookup.join.cache.type'='" + cacheType + "') */ " + " FOR SYSTEM_TIME AS OF o.proc_time AS b on o.uuid = b.uuid"; - execInsertSql(tableEnv, sql); - List result = CollectionUtil.iterableToList( - () -> tableEnv.sqlQuery("select * from t2").execute().collect()); + + // The lookup function loads the dimension table lazily on the first probe row, so a teardown / + // commit-visibility race can occasionally make the join emit no rows. Re-running the upsert into + // the uuid-keyed table t2 is idempotent, so retry until the expected rows materialize. + final int expectedNum = TestData.DATA_SET_SOURCE_INSERT.size(); + List result = Collections.emptyList(); + for (int attempt = 1; attempt <= MAX_STREAM_READ_ATTEMPTS; attempt++) { + execInsertSql(tableEnv, sql); + result = CollectionUtil.iterableToList( + () -> tableEnv.sqlQuery("select * from t2").execute().collect()); + if (result.size() >= expectedNum) { + break; + } + LOG.warn("testLookupJoin collected {} of {} rows on attempt {}/{}; a teardown race produced an " + + "empty lookup join. Retrying.", result.size(), expectedNum, attempt, MAX_STREAM_READ_ATTEMPTS); + } assertRowsEquals(result, TestData.DATA_SET_SOURCE_INSERT); } From d7c5a22bfe8614216f8dd65df52d6c6712276736 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Tue, 30 Jun 2026 20:17:03 +0800 Subject: [PATCH 112/255] fix(partition-ttl): Fix IllegalArgumentException in KeepByTimeStrategy when no candidate partitions exist (#19092) * fix(partition-ttl): Fix IllegalArgumentException in KeepByTimeStrategy when no candidate partitions exist (cherry picked from commit 3da0a9807dd11d038e73b6747de53ba57f979000) --- .../ttl/strategy/KeepByTimeStrategy.java | 4 ++ .../ttl/strategy/TestKeepByTimeStrategy.java | 62 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestKeepByTimeStrategy.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java index 329fee22cf60d..8968d69f70bde 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByTimeStrategy.java @@ -81,6 +81,10 @@ protected List getExpiredPartitionsForTimeStrategy(List partitio * @param partitionPaths Partitions to collect stats. */ private Map> getLastCommitTimeForPartitions(List partitionPaths) { + if (partitionPaths.isEmpty()) { + log.info("Candidate partition paths list is empty, skip TTL stats collection"); + return Collections.emptyMap(); + } int statsParallelism = Math.min(partitionPaths.size(), 200); return hoodieTable.getContext().map(partitionPaths, partitionPath -> { Option partitionLastModifiedTime = hoodieTable.getHoodieView() diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestKeepByTimeStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestKeepByTimeStrategy.java new file mode 100644 index 0000000000000..309eab2d6013f --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestKeepByTimeStrategy.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.ttl.strategy; + +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link KeepByTimeStrategy}. + */ +public class TestKeepByTimeStrategy { + + /** + * Regression test: when there are no candidate partitions to evaluate, + * the strategy must short-circuit and return an empty result instead of + * handing a parallelism of 0 to the engine, which would surface as: + * java.lang.IllegalArgumentException: Positive number of partitions required + * from ParallelCollectionRDD.slice on the Spark path. + */ + @Test + public void testGetExpiredPartitionsForTimeStrategy_emptyInput_returnsEmptyWithoutTouchingEngine() { + HoodieTable hoodieTable = mock(HoodieTable.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(hoodieTable.getConfig()).thenReturn(writeConfig); + when(writeConfig.getPartitionTTLStrategyDaysRetain()).thenReturn(10); + + KeepByTimeStrategy strategy = new KeepByTimeStrategy(hoodieTable, "20240101000000000"); + + List expired = strategy.getExpiredPartitionsForTimeStrategy(Collections.emptyList()); + + assertTrue(expired.isEmpty(), "Empty candidate list should yield no expired partitions"); + // Crucial: we must never reach the engine map call with parallelism=0. + verify(hoodieTable, never()).getContext(); + } +} From a132f0af254ee6f90e54c7f58c0bf9b7264aaadb Mon Sep 17 00:00:00 2001 From: Mahsood Ebrahim Date: Tue, 30 Jun 2026 22:55:30 -0700 Subject: [PATCH 113/255] feat(spark): add repair_orphan_files stored procedure (#19121) Add a `repair_orphan_files` Spark SQL stored procedure that finds and optionally removes orphan data files - files present on the filesystem but not referenced by any commit (active or archived). This makes orphan-file detection and cleanup accessible from any Spark SQL session, reusing the same detection logic (org.apache.hudi.table.repair.RepairUtils) that backs the HoodieRepairTool spark-submit utility. Highlights: - Dry-run (view) mode by default; cleanup mode moves orphans to backup_path. - Per-partition scoping via `partition =>` for very large tables; detection runs one Spark task per partition (listing + classification) so only orphan candidates are collected to the driver. - Optional archived_start_ts / archived_end_ts to scope the instants considered. - max_orphans cap (default 100000) that fails fast with an actionable message instead of risking driver OOM when collecting candidates. - Metadata-table safety cross-check: candidates still visible in the MDT are surfaced as SKIPPED_PRESENT_IN_MDT rows rather than removed. - Structured logging of BACKUP_FAILED root causes. Handles COW (base files) and MOR (base + log files) and all commit action types (COMMIT, DELTA_COMMIT, REPLACE_COMMIT). Registered in HoodieProcedures. Adds TestRepairOrphanFilesProcedure covering dry-run, cleanup + backup, MOR log-file detection, inflight-commit exclusion, backup_path validation, partition scoping, and the max_orphans cap. Co-authored-by: mahsoode Co-authored-by: Claude Opus 4.8 (1M context) (cherry picked from commit 63a1ab19254fe46f395b4b3ae81ae0c088499626) --- .../command/procedures/HoodieProcedures.scala | 1 + .../RepairOrphanFilesProcedure.scala | 336 ++++++++++++++++++ .../TestRepairOrphanFilesProcedure.scala | 256 +++++++++++++ 3 files changed, 593 insertions(+) create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOrphanFilesProcedure.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairOrphanFilesProcedure.scala diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala index fd084ec0bf6e8..fad6d39ed06c0 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala @@ -85,6 +85,7 @@ object HoodieProcedures { ,(RepairDeduplicateProcedure.NAME, RepairDeduplicateProcedure.builder) ,(RepairMigratePartitionMetaProcedure.NAME, RepairMigratePartitionMetaProcedure.builder) ,(RepairOverwriteHoodiePropsProcedure.NAME, RepairOverwriteHoodiePropsProcedure.builder) + ,(RepairOrphanFilesProcedure.NAME, RepairOrphanFilesProcedure.builder) ,(RunCleanProcedure.NAME, RunCleanProcedure.builder) ,(ValidateHoodieSyncProcedure.NAME, ValidateHoodieSyncProcedure.builder) ,(ShowInvalidParquetProcedure.NAME, ShowInvalidParquetProcedure.builder) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOrphanFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOrphanFilesProcedure.scala new file mode 100644 index 0000000000000..2d15215f0c66b --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOrphanFilesProcedure.scala @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.command.procedures + +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.engine.HoodieLocalEngineContext +import org.apache.hudi.common.fs.FSUtils +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.metadata.{FileSystemBackedTableMetadata, HoodieBackedTableMetadata} +import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath, StoragePathInfo} +import org.apache.hudi.table.repair.RepairUtils + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} + +import java.util.function.Supplier + +import scala.collection.JavaConverters._ + +/** + * Spark SQL stored procedure that finds and optionally removes orphan data files — files + * that exist on the filesystem but are not referenced by any commit (active or archived). + * + * Handles COW (base files) and MOR (base + log files), and all commit action types + * (COMMIT, DELTA_COMMIT, REPLACE_COMMIT). The detection reuses + * [[org.apache.hudi.table.repair.RepairUtils]], the same logic that backs the + * `HoodieRepairTool` spark-submit utility, so results are consistent with that tool. + * + * Usage: + * {{{ + * -- View mode (default): list orphan files without touching them + * CALL repair_orphan_files(table => 'my_table') + * + * -- Scoped to one partition + * CALL repair_orphan_files(table => 'my_table', partition => '2024/01/15') + * + * -- Cleanup: move orphan files to a backup location + * CALL repair_orphan_files( + * table => 'my_table', + * dry_run => false, + * backup_path => '/user/hudi/orphan_files_backup' + * ) + * }}} + * + * For very large tables, scope to one partition at a time using `partition =>` to avoid + * collecting all orphan paths to the driver at once. The `max_orphans` parameter (default + * 100,000) acts as a safety cap: if the detected count exceeds it the procedure fails with + * a clear error instead of silently causing a driver OOM. + */ +class RepairOrphanFilesProcedure extends BaseProcedure with ProcedureBuilder with Logging { + + private val PARAMETERS = Array[ProcedureParameter]( + ProcedureParameter.optional(0, "table", DataTypes.StringType, null), + ProcedureParameter.optional(1, "path", DataTypes.StringType, null), + ProcedureParameter.optional(2, "partition", DataTypes.StringType, ""), + ProcedureParameter.optional(3, "dry_run", DataTypes.BooleanType, true), + ProcedureParameter.optional(4, "backup_path", DataTypes.StringType, ""), + ProcedureParameter.optional(5, "archived_start_ts", DataTypes.StringType, ""), + ProcedureParameter.optional(6, "archived_end_ts", DataTypes.StringType, ""), + ProcedureParameter.optional(7, "max_orphans", DataTypes.IntegerType, 100000) + ) + + private val OUTPUT_TYPE = new StructType(Array[StructField]( + StructField("partition", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("file_name", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("instant_time", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("backup_path", DataTypes.StringType, nullable = true, Metadata.empty), + StructField("status", DataTypes.StringType, nullable = true, Metadata.empty) + )) + + def parameters: Array[ProcedureParameter] = PARAMETERS + + def outputType: StructType = OUTPUT_TYPE + + override def call(args: ProcedureArgs): Seq[Row] = { + super.checkArgs(PARAMETERS, args) + + val tableName = getArgValueOrDefault(args, PARAMETERS(0)) + val tablePathOpt = getArgValueOrDefault(args, PARAMETERS(1)) + val partition = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[String] + val dryRun = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Boolean] + val backupPath = getArgValueOrDefault(args, PARAMETERS(4)).get.asInstanceOf[String] + val archivedStartTs = getArgValueOrDefault(args, PARAMETERS(5)).get.asInstanceOf[String] + val archivedEndTs = getArgValueOrDefault(args, PARAMETERS(6)).get.asInstanceOf[String] + val maxOrphans = getArgValueOrDefault(args, PARAMETERS(7)).get.asInstanceOf[Int] + + if (!dryRun && backupPath.isEmpty) { + throw new IllegalArgumentException("backup_path is required when dry_run is false") + } + + // Phase 1: Partition listing (driver) + val basePath = getBasePath(tableName, tablePathOpt) + val metaClient = createMetaClient(jsc, basePath) + + val partitions: java.util.List[String] = + if (partition.nonEmpty) { + java.util.Collections.singletonList(partition) + } else { + // Use FileSystemBackedTableMetadata (filesystem listing, no MDT) for partition discovery. + // This avoids any reliance on the metadata table being present/consistent — the same + // approach HoodieRepairTool uses. + new FileSystemBackedTableMetadata( + new HoodieLocalEngineContext(metaClient.getStorageConf), + metaClient.getTableConfig, metaClient.getStorage, basePath).getAllPartitionPaths + } + + if (partitions.isEmpty) { + Seq.empty + } else { + doRepairOrphanFiles(basePath, metaClient, partitions, dryRun, backupPath, + archivedStartTs, archivedEndTs, maxOrphans) + } + } + + private def doRepairOrphanFiles( + basePath: String, + metaClient: HoodieTableMetaClient, + partitions: java.util.List[String], + dryRun: Boolean, + backupPath: String, + archivedStartTs: String, + archivedEndTs: String, + maxOrphans: Int): Seq[Row] = { + // Build the active and archived timelines once on the driver, loading completed-instant + // details into memory so executors can read commit metadata without further I/O. This is + // the same pattern as HoodieRepairTool: the loaded timelines are serializable and captured + // by the RDD closure below. The HoodieTableMetaClient itself is not captured (not needed + // on executors once details are loaded). + val activeTimeline = metaClient.getActiveTimeline + val archivedTimeline = + if (archivedStartTs.nonEmpty) metaClient.getArchivedTimeline(archivedStartTs) + else metaClient.getArchivedTimeline() + archivedTimeline.loadCompletedInstantDetailsInMemory() + + // StorageConfiguration is Serializable and is the only stateful value captured into the + // closure; storage handles are rebuilt per task from it. + val storageConf = metaClient.getStorageConf + val basePathStr = basePath + val archStartTs = archivedStartTs + val archEndTs = archivedEndTs + + // Phase 2: Parallel orphan file detection (Spark RDD, one task per partition). Each task + // lists its own partition and runs detection locally, so only the (small) set of orphan + // candidates is collected back to the driver rather than the full file listing. + val orphanRelPaths: List[String] = jsc.parallelize(partitions, partitions.size()) + .rdd + .flatMap { partitionStr => + val storage = HoodieStorageUtils.getStorage(basePathStr, storageConf) + val partPath = FSUtils.getAbsolutePartitionPath(new StoragePath(basePathStr), partitionStr) + // getAllDataFilesInPartition handles FileNotFoundException (partition deleted between + // listing and task execution) by returning an empty list rather than throwing. + val allStatuses = FSUtils.getAllDataFilesInPartition(storage, partPath) + val allPaths = allStatuses.asScala.map((info: StoragePathInfo) => info.getPath).asJava + + val instantToFilesMap = RepairUtils.tagInstantsOfBaseAndLogFiles(basePathStr, allPaths) + + if (instantToFilesMap.isEmpty) { + Iterator.empty + } else { + // Optionally scope detection to instants within [archived_start_ts, archived_end_ts]. + // Instants outside the range are left untouched (not reported as orphans). + val instants = instantToFilesMap.keySet.asScala.toSeq.sorted.filter { instant => + (archStartTs.isEmpty || instant >= archStartTs) && + (archEndTs.isEmpty || instant <= archEndTs) + } + + instants.flatMap { instant => + RepairUtils.findInstantFilesToRemove( + instant, + instantToFilesMap.get(instant), + activeTimeline, + archivedTimeline + ).asScala + }.iterator + } + } + .collect() + .toList + + if (orphanRelPaths.size > maxOrphans) { + throw new IllegalStateException( + s"Found ${orphanRelPaths.size} orphan candidates, which exceeds max_orphans=$maxOrphans. " + + s"Re-run with partition => '' to scope to one partition at a time, " + + s"or raise max_orphans if you are sure the driver has enough memory.") + } + + // Phase 3: Metadata table (MDT) safety check (driver). + // Files still visible in the MDT are not true orphans — surface them as + // SKIPPED_PRESENT_IN_MDT (per-file exclusion) so the operator sees which candidates the + // safety check held back, rather than silently dropping them. + val mdtConfig = HoodieMetadataConfig.newBuilder + .enable(true).ignoreSpuriousDeletes(true).build + val mdtReader = new HoodieBackedTableMetadata( + new HoodieLocalEngineContext(metaClient.getStorageConf), metaClient.getStorage, mdtConfig, basePath) + + val mdtUnsafePaths: Set[String] = + if (mdtReader.enabled) { + val byPartition = orphanRelPaths.groupBy(partitionOf) + val unsafe = byPartition.flatMap { case (partRel, paths) => + val mdtPartPath = FSUtils.getAbsolutePartitionPath(new StoragePath(basePath), partRel) + val mdtNames = mdtReader.getAllFilesInPartition(mdtPartPath).asScala + .map(_.getPath.getName).toSet + paths.filter(p => mdtNames.contains(new StoragePath(p).getName)) + }.toSet + if (unsafe.nonEmpty) { + logWarning(s"Found ${unsafe.size} orphan candidate(s) still visible in MDT — " + + s"emitting as SKIPPED_PRESENT_IN_MDT (per-file exclusion): $unsafe") + } + unsafe + } else { + logWarning("Metadata table not enabled; skipping MDT safety cross-check") + Set.empty[String] + } + + val safeOrphanPaths = orphanRelPaths.filterNot(mdtUnsafePaths.contains) + + // Phase 4: Build result rows. + val skippedRows: Seq[Row] = mdtUnsafePaths.toSeq.map { relPath => + val fileName = new StoragePath(relPath).getName + val partRel = partitionOf(relPath) + val instantTime = FSUtils.getCommitTime(fileName) + Row(partRel, fileName, instantTime, "", "SKIPPED_PRESENT_IN_MDT") + } + + val safeRows: Seq[Row] = if (dryRun) { + safeOrphanPaths.map { relPath => + val fileName = new StoragePath(relPath).getName + val partRel = partitionOf(relPath) + val instantTime = FSUtils.getCommitTime(fileName) + Row(partRel, fileName, instantTime, "", "IDENTIFIED") + } + } else { + val storage = metaClient.getStorage + val tableNm = metaClient.getTableConfig.getTableName // always from metaClient — the 'table' + // proc arg is null when invoked via path => + safeOrphanPaths.map { relPath => + val fileName = new StoragePath(relPath).getName + val partRel = partitionOf(relPath) + val instantTime = FSUtils.getCommitTime(fileName) + val srcPath = new StoragePath(basePath, relPath) + // Non-partitioned tables have partRel="" — back up directly under /. + val destDir = + if (partRel.isEmpty) new StoragePath(s"$backupPath/$tableNm") + else new StoragePath(s"$backupPath/$tableNm/$partRel") + val destPath = new StoragePath(destDir, fileName) + + // Each storage op is wrapped to capture exception class+message — a backup can fail for + // permissions, missing parent, RPC, or concurrent-delete reasons, and a status of + // BACKUP_FAILED with no log line leaves the operator with nothing to diagnose. Causes + // are accumulated and logged once at the end iff the final outcome is a failure. + val causes = scala.collection.mutable.ArrayBuffer.empty[String] + + val dirCreated: Boolean = + try { + val ok = storage.createDirectory(destDir) + if (!ok) causes += s"createDirectory($destDir)=false (likely permissions or destDir exists as a file)" + ok + } catch { + case t: Throwable => + causes += s"createDirectory($destDir) threw ${t.getClass.getSimpleName}: ${t.getMessage}" + false + } + + val moved: Boolean = + if (!dirCreated) { + false + } else { + try { + val ok = storage.rename(srcPath, destPath) + if (!ok) causes += s"rename returned false (srcPath missing, destPath exists, or cross-volume rename)" + ok + } catch { + case t: Throwable => + causes += s"rename threw ${t.getClass.getSimpleName}: ${t.getMessage}" + false + } + } + + // A concurrent cleaner may have already deleted srcPath — that is success, not failure. + val srcStillExists: Boolean = + try { + storage.exists(srcPath) + } catch { + case t: Throwable => + // Cannot confirm the concurrent-delete recovery path; treat as failure. + causes += s"exists($srcPath) threw ${t.getClass.getSimpleName}: ${t.getMessage}" + true + } + val succeeded = moved || !srcStillExists + + if (!succeeded) { + logWarning(s"BACKUP_FAILED for $srcPath -> $destPath; causes: ${causes.mkString("; ")}") + } + val status = if (succeeded) "BACKED_UP" else "BACKUP_FAILED" + val backupOut = if (succeeded) destPath.toString else "" + Row(partRel, fileName, instantTime, backupOut, status) + } + } + + safeRows ++ skippedRows + } + + // Extract the relative partition path from a relative file path using string ops. This avoids + // any path-normalization surprises and works uniformly for partitioned and non-partitioned + // tables (root files yield ""). + private def partitionOf(relPath: String): String = { + val s = relPath.lastIndexOf('/') + if (s < 0) "" else relPath.substring(0, s) + } + + override def build: Procedure = new RepairOrphanFilesProcedure() +} + +object RepairOrphanFilesProcedure { + val NAME = "repair_orphan_files" + + def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] { + override def get(): ProcedureBuilder = new RepairOrphanFilesProcedure() + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairOrphanFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairOrphanFilesProcedure.scala new file mode 100644 index 0000000000000..b77c3bd11e43f --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairOrphanFilesProcedure.scala @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.testutils.FileCreateUtils +import org.apache.hudi.hadoop.fs.HadoopFSUtils + +import org.apache.hadoop.fs.Path + +import java.util.UUID + +class TestRepairOrphanFilesProcedure extends HoodieSparkProcedureTestBase { + + private val ORPHAN_INSTANT = "20000101000000000" // Year 2000; never in any test timeline + + private def metaClientFor(tablePath: String): HoodieTableMetaClient = { + HoodieTableMetaClient.builder + .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath) + .build + } + + // Test 1 — dry run detects a base-file orphan in a non-partitioned COW table + test("Test Call repair_orphan_files dry run finds base file orphan") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + + // Create a non-partitioned COW table and write one real commit + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + // Inject orphan base file directly onto the filesystem with a stale instant timestamp + val orphanFileId = UUID.randomUUID().toString + FileCreateUtils.createBaseFile(metaClientFor(tablePath), "", ORPHAN_INSTANT, orphanFileId) + + // dry_run=true (default): should return exactly 1 row for the orphan, touch nothing + val result = spark.sql(s"call repair_orphan_files(table => '$tableName')").collect() + assertResult(1)(result.length) + + val row = result(0) + assertResult("")(row.getString(0)) // partition (root = "" for non-partitioned) + assert(row.getString(1).contains(ORPHAN_INSTANT), s"file_name should contain orphan instant: ${row.getString(1)}") + assertResult(ORPHAN_INSTANT)(row.getString(2)) // instant_time + assertResult("")(row.getString(3)) // backup_path is empty in dry run + assertResult("IDENTIFIED")(row.getString(4)) // status + } + } + + // Test 2 — cleanup mode backs up the orphan file and removes it from the table path + test("Test Call repair_orphan_files cleanup backs up base file orphan") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + val backupDir = s"${tmp.getCanonicalPath}/backup" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + val orphanFileId = UUID.randomUUID().toString + // FileCreateUtils.createBaseFile returns the absolute path of the created file + val orphanAbsPath = FileCreateUtils.createBaseFile(metaClientFor(tablePath), "", ORPHAN_INSTANT, orphanFileId) + val orphanFilePath = new Path(orphanAbsPath) + + val hadoopConf = spark.sparkContext.hadoopConfiguration + val fs = HadoopFSUtils.getFs(tablePath, hadoopConf) + assert(fs.exists(orphanFilePath), "Orphan file should exist before cleanup") + + // dry_run=false: orphan should be moved to backup + val result = spark.sql( + s"""call repair_orphan_files( + | table => '$tableName', + | dry_run => false, + | backup_path => '$backupDir' + |)""".stripMargin).collect() + + assertResult(1)(result.length) + assertResult("BACKED_UP")(result(0).getString(4)) + + // Orphan file must be gone from the table path + assert(!fs.exists(orphanFilePath), "Orphan file should no longer exist at original path") + + // Orphan file must exist at the backup path + val backedUpPath = new Path(result(0).getString(3)) + assert(fs.exists(backedUpPath), "Orphan file should exist at backup path") + + // Real data must still be readable + assertResult(1)(spark.sql(s"select id from $tableName").collect().length) + } + } + + // Test 3 — inflight commit files are skipped (RepairUtils skips non-completed instants) + test("Test Call repair_orphan_files skips inflight commit files") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + val inflightTs = "20010101000000000" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + val metaClient = metaClientFor(tablePath) + // Create the inflight marker in .hoodie so the active timeline sees it as non-completed + FileCreateUtils.createInflightCommit(metaClient, inflightTs) + + // Place a data file with the inflight instant timestamp on disk + FileCreateUtils.createBaseFile(metaClient, "", inflightTs, UUID.randomUUID().toString) + + // Procedure must return 0 rows — inflight instant is excluded by RepairUtils + val result = spark.sql(s"call repair_orphan_files(table => '$tableName')").collect() + assertResult(0)(result.length) + } + } + + // Test 4 — backup_path validation: error when dry_run=false and no backup_path given + test("Test Call repair_orphan_files requires backup_path when not dry run") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + + checkExceptionContain( + s"call repair_orphan_files(table => '$tableName', dry_run => false)" + )("backup_path is required") + } + } + + // Test 5 — partition filter scopes the scan to a specific partition + test("Test Call repair_orphan_files partition filter scopes scan") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + // Inject orphan at table root (partition = "") + FileCreateUtils.createBaseFile(metaClientFor(tablePath), "", ORPHAN_INSTANT, UUID.randomUUID().toString) + + // Filtering to a non-existent partition should find nothing + val filtered = spark.sql( + s"call repair_orphan_files(table => '$tableName', partition => 'nonexistent')").collect() + assertResult(0)(filtered.length) + + // Filtering to the root partition ("") should find the orphan + val root = spark.sql( + s"call repair_orphan_files(table => '$tableName', partition => '')").collect() + assertResult(1)(root.length) + assertResult("IDENTIFIED")(root(0).getString(4)) + } + } + + // Test 6 — log file orphan detected on a MOR table + test("Test Call repair_orphan_files detects log file orphan on MOR table") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts', type = 'mor') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + // Inject an orphan log file with a stale instant timestamp + val orphanFileId = UUID.randomUUID().toString + FileCreateUtils.createLogFile(metaClientFor(tablePath), "", ORPHAN_INSTANT, orphanFileId, 1) + + val result = spark.sql(s"call repair_orphan_files(table => '$tableName')").collect() + + // At least the orphan log file must appear in the result + val orphanRows = result.filter(_.getString(2) == ORPHAN_INSTANT) + assert(orphanRows.length >= 1, + s"Expected at least 1 orphan row with instant $ORPHAN_INSTANT, got: ${result.mkString(", ")}") + assert(orphanRows.forall(_.getString(4) == "IDENTIFIED")) + } + } + + // Note: the SKIPPED_PRESENT_IN_MDT branch (surfaces MDT-visible orphan candidates instead of + // silently dropping them) is verified by inspection rather than an end-to-end test. + // HoodieBackedTableMetadata.getAllFilesInPartition dynamically filters by the data table's + // timeline state — deleting a timeline file immediately removes the corresponding data file + // from the MDT's view as well. As a result, the (orphan ∧ in-MDT) state cannot be constructed + // by manipulating the timeline alone; only direct MDT writes could produce it, and the setup + // cost outweighs the value for what is a defense-in-depth path. + + // Test 7 — max_orphans cap prevents driver OOM: error when detected count exceeds the cap + test("Test Call repair_orphan_files max_orphans cap triggers error when exceeded") { + withTempDir { tmp => + val tableName = generateTableName + val tablePath = s"${tmp.getCanonicalPath}/$tableName" + + spark.sql( + s"""create table $tableName (id int, name string, price double, ts long) + |using hudi + |location '$tablePath' + |tblproperties (primaryKey = 'id', preCombineField = 'ts') + |""".stripMargin) + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + // Inject 3 orphan files, then cap at 2 — should throw + val metaClient = metaClientFor(tablePath) + FileCreateUtils.createBaseFile(metaClient, "", ORPHAN_INSTANT, UUID.randomUUID().toString) + FileCreateUtils.createBaseFile(metaClient, "", ORPHAN_INSTANT, UUID.randomUUID().toString) + FileCreateUtils.createBaseFile(metaClient, "", ORPHAN_INSTANT, UUID.randomUUID().toString) + + checkExceptionContain( + s"call repair_orphan_files(table => '$tableName', max_orphans => 2)" + )("exceeds max_orphans=2") + } + } +} From f17346a6574c386bcf18e47165c70a3dac03ead9 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 2 Jul 2026 16:24:25 +0800 Subject: [PATCH 114/255] fix: remove the dependency to flink-table-planner (#19131) (cherry picked from commit 727ded72b2522efc3e5bc1dc369bf82814d14d75) --- hudi-examples/hudi-examples-flink/pom.xml | 13 +- hudi-flink-datasource/hudi-flink/pom.xml | 13 +- .../AppendWriteFunctionWithBIMBufferSort.java | 6 +- ...AppendWriteFunctionWithContinuousSort.java | 6 +- ...dWriteFunctionWithDisruptorBufferSort.java | 6 +- .../hudi/sink/bulk/sort/SortOperatorGen.java | 433 +++++++++++++++++- .../sink/clustering/ClusteringOperator.java | 11 +- .../clustering/HoodieFlinkClusteringJob.java | 5 +- .../sink/utils/FlinkTransformationUtils.java | 41 ++ .../org/apache/hudi/sink/utils/Pipelines.java | 9 +- .../hudi/sink/v2/utils/PipelinesV2.java | 4 +- .../sink/bulk/sort/TestSortOperatorGen.java | 138 ++++++ .../cluster/ITTestHoodieFlinkClustering.java | 10 +- .../hudi/table/catalog/TestHoodieCatalog.java | 5 +- .../table/catalog/TestHoodieHiveCatalog.java | 6 +- .../hudi-flink1.18.x/pom.xml | 6 - .../hudi-flink1.19.x/pom.xml | 6 - .../hudi-flink1.20.x/pom.xml | 6 - hudi-flink-datasource/hudi-flink2.0.x/pom.xml | 6 - hudi-flink-datasource/hudi-flink2.1.x/pom.xml | 6 - pom.xml | 13 +- 21 files changed, 649 insertions(+), 100 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkTransformationUtils.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/sort/TestSortOperatorGen.java diff --git a/hudi-examples/hudi-examples-flink/pom.xml b/hudi-examples/hudi-examples-flink/pom.xml index 75a21c17053a0..da370499262c2 100644 --- a/hudi-examples/hudi-examples-flink/pom.xml +++ b/hudi-examples/hudi-examples-flink/pom.xml @@ -194,12 +194,6 @@ ${flink.version} provided - - org.apache.flink - ${flink.table.planner.artifactId} - ${flink.version} - provided - org.apache.flink ${flink.statebackend.rocksdb.artifactId} @@ -371,6 +365,13 @@ test test-jar + + + org.apache.flink + ${flink.table.planner.artifactId} + ${flink.version} + test + org.apache.flink flink-csv diff --git a/hudi-flink-datasource/hudi-flink/pom.xml b/hudi-flink-datasource/hudi-flink/pom.xml index 1c8f323d08006..1923534f475b3 100644 --- a/hudi-flink-datasource/hudi-flink/pom.xml +++ b/hudi-flink-datasource/hudi-flink/pom.xml @@ -234,12 +234,6 @@ ${flink.version} provided - - org.apache.flink - ${flink.table.planner.artifactId} - ${flink.version} - provided - org.apache.flink ${flink.statebackend.rocksdb.artifactId} @@ -452,6 +446,13 @@ test test-jar + + + org.apache.flink + ${flink.table.planner.artifactId} + ${flink.version} + test + org.apache.flink flink-csv diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithBIMBufferSort.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithBIMBufferSort.java index 091f018c3c5ff..11017d9e4fc26 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithBIMBufferSort.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithBIMBufferSort.java @@ -33,7 +33,6 @@ import org.apache.flink.runtime.operators.sort.QuickSort; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.binary.BinaryRowData; -import org.apache.flink.table.planner.codegen.sort.SortCodeGenerator; import org.apache.flink.table.runtime.generated.GeneratedNormalizedKeyComputer; import org.apache.flink.table.runtime.generated.GeneratedRecordComparator; import org.apache.flink.table.runtime.operators.sort.BinaryInMemorySortBuffer; @@ -89,9 +88,8 @@ public void open(Configuration parameters) throws Exception { // Resolve sort keys (defaults to record key if not specified) List sortKeyList = AppendWriteFunctions.resolveSortKeys(config); SortOperatorGen sortOperatorGen = new SortOperatorGen(rowType, sortKeyList.toArray(new String[0])); - SortCodeGenerator codeGenerator = sortOperatorGen.createSortCodeGenerator(); - GeneratedNormalizedKeyComputer keyComputer = codeGenerator.generateNormalizedKeyComputer("SortComputer"); - GeneratedRecordComparator recordComparator = codeGenerator.generateRecordComparator("SortComparator"); + GeneratedNormalizedKeyComputer keyComputer = sortOperatorGen.generateNormalizedKeyComputer("SortComputer"); + GeneratedRecordComparator recordComparator = sortOperatorGen.generateRecordComparator("SortComparator"); this.memorySegmentPools = this.memorySegmentPoolFactory.createMemorySegmentPools(config, 2, OptionsResolver.getWriteBufferSizeInBytes(config)); this.activeBuffer = BufferUtils.createBuffer(rowType, diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithContinuousSort.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithContinuousSort.java index caffecbb34ba3..e1f03fdfb2a80 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithContinuousSort.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithContinuousSort.java @@ -31,7 +31,6 @@ import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.planner.codegen.sort.SortCodeGenerator; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.runtime.generated.GeneratedNormalizedKeyComputer; import org.apache.flink.table.runtime.generated.GeneratedRecordComparator; @@ -127,9 +126,8 @@ public void open(Configuration parameters) throws Exception { // Create sort code generator for normalized key computation and record comparison SortOperatorGen sortOperatorGen = new SortOperatorGen(rowType, sortKeyList.toArray(new String[0])); - SortCodeGenerator codeGenerator = sortOperatorGen.createSortCodeGenerator(); - GeneratedNormalizedKeyComputer generatedKeyComputer = codeGenerator.generateNormalizedKeyComputer("ContinuousSortKeyComputer"); - GeneratedRecordComparator generatedComparator = codeGenerator.generateRecordComparator("ContinuousSortComparator"); + GeneratedNormalizedKeyComputer generatedKeyComputer = sortOperatorGen.generateNormalizedKeyComputer("ContinuousSortKeyComputer"); + GeneratedRecordComparator generatedComparator = sortOperatorGen.generateRecordComparator("ContinuousSortComparator"); // Instantiate code-generated components ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithDisruptorBufferSort.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithDisruptorBufferSort.java index df99a71b9bbe0..347ad81da22d4 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithDisruptorBufferSort.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithDisruptorBufferSort.java @@ -35,7 +35,6 @@ import org.apache.flink.runtime.operators.sort.QuickSort; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.binary.BinaryRowData; -import org.apache.flink.table.planner.codegen.sort.SortCodeGenerator; import org.apache.flink.table.runtime.generated.GeneratedNormalizedKeyComputer; import org.apache.flink.table.runtime.generated.GeneratedRecordComparator; import org.apache.flink.table.runtime.operators.sort.BinaryInMemorySortBuffer; @@ -95,9 +94,8 @@ public void open(Configuration parameters) throws Exception { // Create Flink-native sort components SortOperatorGen sortOperatorGen = new SortOperatorGen(rowType, sortKeyList.toArray(new String[0])); - SortCodeGenerator codeGenerator = sortOperatorGen.createSortCodeGenerator(); - this.keyComputer = codeGenerator.generateNormalizedKeyComputer("SortComputer"); - this.recordComparator = codeGenerator.generateRecordComparator("SortComparator"); + this.keyComputer = sortOperatorGen.generateNormalizedKeyComputer("SortComputer"); + this.recordComparator = sortOperatorGen.generateRecordComparator("SortComparator"); this.memorySegmentPool = this.memorySegmentPoolFactory.createMemorySegmentPool(config, OptionsResolver.getWriteBufferSizeInBytes(config)); initDisruptorBuffer(); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperatorGen.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperatorGen.java index 068f7540f9538..2db08f9764562 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperatorGen.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperatorGen.java @@ -20,40 +20,449 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; -import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.planner.codegen.sort.SortCodeGenerator; -import org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec; +import org.apache.flink.table.runtime.generated.GeneratedNormalizedKeyComputer; +import org.apache.flink.table.runtime.generated.GeneratedRecordComparator; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.ZonedTimestampType; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * Tools to generate the sort operator. */ public class SortOperatorGen { + private static final int MAX_NORMALIZED_KEY_BYTES = 16; + private static final int VARIABLE_LENGTH_NORMALIZED_KEY_BYTES = 8; + private final int[] sortIndices; private final RowType rowType; - private final TableConfig tableConfig = TableConfig.getDefault(); + private final RowData.FieldGetter[] fieldGetters; public SortOperatorGen(RowType rowType, String[] sortFields) { - this.sortIndices = Arrays.stream(sortFields).mapToInt(rowType::getFieldIndex).toArray(); this.rowType = rowType; + this.sortIndices = Arrays.stream(sortFields).mapToInt(field -> { + int index = rowType.getFieldIndex(field); + if (index < 0) { + throw new IllegalArgumentException("Can not find sort field '" + field + "' in row type " + rowType); + } + return index; + }).toArray(); + this.fieldGetters = Arrays.stream(sortIndices) + .mapToObj(index -> RowData.createFieldGetter(rowType.getTypeAt(index), index)) + .toArray(RowData.FieldGetter[]::new); } public OneInputStreamOperator createSortOperator(Configuration conf) { - SortCodeGenerator codeGen = createSortCodeGenerator(); return new SortOperator( - codeGen.generateNormalizedKeyComputer("SortComputer"), - codeGen.generateRecordComparator("SortComparator"), + generateNormalizedKeyComputer("SortComputer"), + generateRecordComparator("SortComparator"), conf); } - public SortCodeGenerator createSortCodeGenerator() { - SortSpec.SortSpecBuilder builder = SortSpec.builder(); + public GeneratedNormalizedKeyComputer generateNormalizedKeyComputer(String name) { + String className = generatedClassName(name); + return new GeneratedNormalizedKeyComputer(className, generateNormalizedKeyComputerCode(className)); + } + + public GeneratedRecordComparator generateRecordComparator(String name) { + String className = generatedClassName(name); + return new GeneratedRecordComparator(className, generateRecordComparatorCode(className), fieldGetters); + } + + private String generatedClassName(String name) { + String normalizedName = name.replaceAll("[^A-Za-z0-9_$]", "_"); + if (normalizedName.isEmpty() || !Character.isJavaIdentifierStart(normalizedName.charAt(0))) { + normalizedName = "_" + normalizedName; + } + int hash = 31 * Arrays.hashCode(sortIndices) + rowType.asSerializableString().hashCode(); + return normalizedName + "_" + Integer.toUnsignedString(hash); + } + + private String generateRecordComparatorCode(String className) { + StringBuilder code = new StringBuilder(); + code.append("public final class ").append(className) + .append(" implements org.apache.flink.table.runtime.generated.RecordComparator {\n") + .append(" private final Object[] references;\n") + .append(" public ").append(className).append("(Object[] references) {\n") + .append(" this.references = references;\n") + .append(" }\n") + .append(" @Override\n") + .append(" public int compare(org.apache.flink.table.data.RowData row1, ") + .append("org.apache.flink.table.data.RowData row2) {\n"); + for (int i = 0; i < sortIndices.length; i++) { + int sortIndex = sortIndices[i]; + code.append(" boolean isNull1_").append(i).append(" = row1.isNullAt(").append(sortIndex).append(");\n") + .append(" boolean isNull2_").append(i).append(" = row2.isNullAt(").append(sortIndex).append(");\n") + .append(" if (isNull1_").append(i).append(" || isNull2_").append(i).append(") {\n") + .append(" if (isNull1_").append(i).append(" && isNull2_").append(i).append(") {\n") + .append(" } else {\n") + .append(" return isNull1_").append(i).append(" ? 1 : -1;\n") + .append(" }\n") + .append(" } else {\n") + .append(" int cmp_").append(i).append(" = ").append(compareExpression(i, sortIndex)).append(";\n") + .append(" if (cmp_").append(i).append(" != 0) {\n") + .append(" return cmp_").append(i).append(";\n") + .append(" }\n") + .append(" }\n"); + } + code.append(" return 0;\n") + .append(" }\n") + .append(" private int compareFallback(org.apache.flink.table.data.RowData row1, ") + .append("org.apache.flink.table.data.RowData row2, int referenceIndex) {\n") + .append(" Object value1 = ((org.apache.flink.table.data.RowData.FieldGetter) references[referenceIndex])") + .append(".getFieldOrNull(row1);\n") + .append(" Object value2 = ((org.apache.flink.table.data.RowData.FieldGetter) references[referenceIndex])") + .append(".getFieldOrNull(row2);\n") + .append(" return compareValues(value1, value2);\n") + .append(" }\n") + .append(" private static int compareValues(Object value1, Object value2) {\n") + .append(" if (value1 == value2) {\n") + .append(" return 0;\n") + .append(" }\n") + .append(" if (value1 == null) {\n") + .append(" return 1;\n") + .append(" }\n") + .append(" if (value2 == null) {\n") + .append(" return -1;\n") + .append(" }\n") + .append(" if (value1 instanceof byte[] && value2 instanceof byte[]) {\n") + .append(" return compareUnsignedBytes((byte[]) value1, (byte[]) value2);\n") + .append(" }\n") + .append(" if (value1 instanceof java.lang.Comparable && value2 instanceof java.lang.Comparable) {\n") + .append(" return ((java.lang.Comparable) value1).compareTo(value2);\n") + .append(" }\n") + .append(" throw new IllegalArgumentException(\"Unsupported sort field value type: \" ") + .append("+ value1.getClass().getName());\n") + .append(" }\n") + .append(" private static int compareUnsignedBytes(byte[] bytes1, byte[] bytes2) {\n") + .append(" int len = java.lang.Math.min(bytes1.length, bytes2.length);\n") + .append(" for (int i = 0; i < len; i++) {\n") + .append(" int result = java.lang.Byte.toUnsignedInt(bytes1[i]) ") + .append("- java.lang.Byte.toUnsignedInt(bytes2[i]);\n") + .append(" if (result != 0) {\n") + .append(" return result;\n") + .append(" }\n") + .append(" }\n") + .append(" return bytes1.length - bytes2.length;\n") + .append(" }\n") + .append("}\n"); + return code.toString(); + } + + private String compareExpression(int referenceIndex, int sortIndex) { + LogicalType logicalType = rowType.getTypeAt(sortIndex); + switch (logicalType.getTypeRoot()) { + case BOOLEAN: + return "java.lang.Boolean.compare(row1.getBoolean(" + sortIndex + "), row2.getBoolean(" + sortIndex + "))"; + case TINYINT: + return "java.lang.Byte.compare(row1.getByte(" + sortIndex + "), row2.getByte(" + sortIndex + "))"; + case SMALLINT: + return "java.lang.Short.compare(row1.getShort(" + sortIndex + "), row2.getShort(" + sortIndex + "))"; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTERVAL_YEAR_MONTH: + return "java.lang.Integer.compare(row1.getInt(" + sortIndex + "), row2.getInt(" + sortIndex + "))"; + case BIGINT: + case INTERVAL_DAY_TIME: + return "java.lang.Long.compare(row1.getLong(" + sortIndex + "), row2.getLong(" + sortIndex + "))"; + case FLOAT: + return "java.lang.Float.compare(row1.getFloat(" + sortIndex + "), row2.getFloat(" + sortIndex + "))"; + case DOUBLE: + return "java.lang.Double.compare(row1.getDouble(" + sortIndex + "), row2.getDouble(" + sortIndex + "))"; + case CHAR: + case VARCHAR: + return "row1.getString(" + sortIndex + ").compareTo(row2.getString(" + sortIndex + "))"; + case BINARY: + case VARBINARY: + return "compareUnsignedBytes(row1.getBinary(" + sortIndex + "), row2.getBinary(" + sortIndex + "))"; + case DECIMAL: + DecimalType decimalType = (DecimalType) logicalType; + return "row1.getDecimal(" + sortIndex + ", " + decimalType.getPrecision() + ", " + + decimalType.getScale() + ").compareTo(row2.getDecimal(" + sortIndex + ", " + + decimalType.getPrecision() + ", " + decimalType.getScale() + "))"; + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampCompareExpression(sortIndex, ((TimestampType) logicalType).getPrecision()); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampCompareExpression(sortIndex, ((LocalZonedTimestampType) logicalType).getPrecision()); + case TIMESTAMP_WITH_TIME_ZONE: + return timestampCompareExpression(sortIndex, ((ZonedTimestampType) logicalType).getPrecision()); + default: + return "compareFallback(row1, row2, " + referenceIndex + ")"; + } + } + + private String timestampCompareExpression(int sortIndex, int precision) { + return "row1.getTimestamp(" + sortIndex + ", " + precision + ").compareTo(row2.getTimestamp(" + + sortIndex + ", " + precision + "))"; + } + + private String generateNormalizedKeyComputerCode(String className) { + List normalizedKeyFields = normalizedKeyFields(); + int numKeyBytes = normalizedKeyFields.stream().mapToInt(NormalizedKeyField::totalBytes).sum(); + boolean fullyDetermines = normalizedKeyFields.size() == sortIndices.length + && normalizedKeyFields.stream().allMatch(NormalizedKeyField::fullyDetermines); + + StringBuilder code = new StringBuilder(); + code.append("public final class ").append(className) + .append(" implements org.apache.flink.table.runtime.generated.NormalizedKeyComputer {\n") + .append(" public ").append(className).append("(Object[] references) {\n") + .append(" }\n") + .append(" @Override\n") + .append(" public void putKey(org.apache.flink.table.data.RowData rowData, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset) {\n"); + int keyOffset = 0; + for (NormalizedKeyField normalizedKeyField : normalizedKeyFields) { + int valueOffset = keyOffset + 1; + int sortIndex = normalizedKeyField.sortIndex; + int valueBytes = normalizedKeyField.valueBytes; + code.append(" if (rowData.isNullAt(").append(sortIndex).append(")) {\n") + .append(" target.put(offset + ").append(keyOffset).append(", (byte) 1);\n") + .append(" zeroBytes(target, offset + ").append(valueOffset).append(", ") + .append(valueBytes).append(");\n") + .append(" } else {\n") + .append(" target.put(offset + ").append(keyOffset).append(", (byte) 0);\n") + .append(" ").append(normalizedKeyExpression(sortIndex, valueOffset, valueBytes)).append("\n") + .append(" }\n"); + keyOffset += normalizedKeyField.totalBytes(); + } + code.append(" }\n") + .append(" @Override\n") + .append(" public int compareKey(org.apache.flink.core.memory.MemorySegment memorySegment, int i, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset) {\n") + .append(" for (int j = 0; j < ").append(numKeyBytes).append("; j++) {\n") + .append(" int cmp = java.lang.Byte.toUnsignedInt(memorySegment.get(i + j))\n") + .append(" - java.lang.Byte.toUnsignedInt(target.get(offset + j));\n") + .append(" if (cmp != 0) {\n") + .append(" return cmp;\n") + .append(" }\n") + .append(" }\n") + .append(" return 0;\n") + .append(" }\n") + .append(" @Override\n") + .append(" public void swapKey(org.apache.flink.core.memory.MemorySegment seg1, int index1, ") + .append("org.apache.flink.core.memory.MemorySegment seg2, int index2) {\n") + .append(" for (int j = 0; j < ").append(numKeyBytes).append("; j++) {\n") + .append(" byte tmp = seg1.get(index1 + j);\n") + .append(" seg1.put(index1 + j, seg2.get(index2 + j));\n") + .append(" seg2.put(index2 + j, tmp);\n") + .append(" }\n") + .append(" }\n") + .append(" @Override\n") + .append(" public int getNumKeyBytes() {\n") + .append(" return ").append(numKeyBytes).append(";\n") + .append(" }\n") + .append(" @Override\n") + .append(" public boolean isKeyFullyDetermines() {\n") + .append(" return ").append(fullyDetermines).append(";\n") + .append(" }\n") + .append(" @Override\n") + .append(" public boolean invertKey() {\n") + .append(" return false;\n") + .append(" }\n") + .append(" private static void zeroBytes(org.apache.flink.core.memory.MemorySegment target, ") + .append("int offset, int numBytes) {\n") + .append(" for (int i = 0; i < numBytes; i++) {\n") + .append(" target.put(offset + i, (byte) 0);\n") + .append(" }\n") + .append(" }\n") + .append(" private static void putBytesNormalizedKey(byte[] bytes, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset, int numBytes) {\n") + .append(" int len = java.lang.Math.min(bytes.length, numBytes);\n") + .append(" for (int i = 0; i < len; i++) {\n") + .append(" target.put(offset + i, bytes[i]);\n") + .append(" }\n") + .append(" zeroBytes(target, offset + len, numBytes - len);\n") + .append(" }\n") + .append(" private static void putFloatNormalizedKey(float value, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset, int numBytes) {\n") + .append(" int bits = java.lang.Float.floatToIntBits(value);\n") + .append(" int normalized = bits >= 0 ? bits ^ java.lang.Integer.MIN_VALUE : ~bits;\n") + .append(" org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil") + .append(".putUnsignedIntegerNormalizedKey(normalized, target, offset, numBytes);\n") + .append(" }\n") + .append(" private static void putDoubleNormalizedKey(double value, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset, int numBytes) {\n") + .append(" long bits = java.lang.Double.doubleToLongBits(value);\n") + .append(" long normalized = bits >= 0 ? bits ^ java.lang.Long.MIN_VALUE : ~bits;\n") + .append(" org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil") + .append(".putUnsignedLongNormalizedKey(normalized, target, offset, numBytes);\n") + .append(" }\n") + .append(" private static void putTimestampNormalizedKey(org.apache.flink.table.data.TimestampData timestamp, ") + .append("org.apache.flink.core.memory.MemorySegment target, int offset, int numBytes) {\n") + .append(" org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil") + .append(".putLongNormalizedKey(timestamp.getMillisecond(), target, offset, java.lang.Math.min(numBytes, 8));\n") + .append(" if (numBytes > 8) {\n") + .append(" org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil") + .append(".putIntNormalizedKey(timestamp.getNanoOfMillisecond(), target, offset + 8, numBytes - 8);\n") + .append(" }\n") + .append(" }\n") + .append("}\n"); + return code.toString(); + } + + private List normalizedKeyFields() { + List normalizedKeyFields = new ArrayList<>(); + int remainingBytes = MAX_NORMALIZED_KEY_BYTES; for (int sortIndex : sortIndices) { - builder.addField(sortIndex, true, true); + LogicalType logicalType = rowType.getTypeAt(sortIndex); + int maxValueBytes = maxNormalizedKeyValueBytes(logicalType); + if (maxValueBytes <= 0 || remainingBytes <= 1) { + break; + } + + int valueBytes = Math.min(maxValueBytes, remainingBytes - 1); + boolean fullyDetermines = isFixedLengthNormalizedKey(logicalType) && valueBytes == maxValueBytes; + normalizedKeyFields.add(new NormalizedKeyField(sortIndex, valueBytes, fullyDetermines)); + remainingBytes -= valueBytes + 1; + if (!fullyDetermines) { + break; + } + } + return normalizedKeyFields; + } + + private int maxNormalizedKeyValueBytes(LogicalType logicalType) { + switch (logicalType.getTypeRoot()) { + case BOOLEAN: + case TINYINT: + return 1; + case SMALLINT: + return 2; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTERVAL_YEAR_MONTH: + case FLOAT: + return 4; + case BIGINT: + case INTERVAL_DAY_TIME: + case DOUBLE: + return 8; + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return VARIABLE_LENGTH_NORMALIZED_KEY_BYTES; + case DECIMAL: + return org.apache.flink.table.data.DecimalData.isCompact(((DecimalType) logicalType).getPrecision()) ? 8 : 0; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return 12; + default: + return 0; + } + } + + private boolean isFixedLengthNormalizedKey(LogicalType logicalType) { + switch (logicalType.getTypeRoot()) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTERVAL_YEAR_MONTH: + case FLOAT: + case BIGINT: + case INTERVAL_DAY_TIME: + case DOUBLE: + case DECIMAL: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return true; + default: + return false; + } + } + + private String normalizedKeyExpression(int sortIndex, int valueOffset, int valueBytes) { + LogicalType logicalType = rowType.getTypeAt(sortIndex); + String offset = "offset + " + valueOffset; + switch (logicalType.getTypeRoot()) { + case BOOLEAN: + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putBooleanNormalizedKey(" + + "rowData.getBoolean(" + sortIndex + "), target, " + offset + ", " + valueBytes + ");"; + case TINYINT: + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putByteNormalizedKey(" + + "rowData.getByte(" + sortIndex + "), target, " + offset + ", " + valueBytes + ");"; + case SMALLINT: + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putShortNormalizedKey(" + + "rowData.getShort(" + sortIndex + "), target, " + offset + ", " + valueBytes + ");"; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTERVAL_YEAR_MONTH: + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putIntNormalizedKey(" + + "rowData.getInt(" + sortIndex + "), target, " + offset + ", " + valueBytes + ");"; + case BIGINT: + case INTERVAL_DAY_TIME: + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putLongNormalizedKey(" + + "rowData.getLong(" + sortIndex + "), target, " + offset + ", " + valueBytes + ");"; + case FLOAT: + return "putFloatNormalizedKey(rowData.getFloat(" + sortIndex + "), target, " + offset + ", " + + valueBytes + ");"; + case DOUBLE: + return "putDoubleNormalizedKey(rowData.getDouble(" + sortIndex + "), target, " + offset + ", " + + valueBytes + ");"; + case CHAR: + case VARCHAR: + return "putBytesNormalizedKey(rowData.getString(" + sortIndex + ").toBytes(), target, " + offset + ", " + + valueBytes + ");"; + case BINARY: + case VARBINARY: + return "putBytesNormalizedKey(rowData.getBinary(" + sortIndex + "), target, " + offset + ", " + + valueBytes + ");"; + case DECIMAL: + DecimalType decimalType = (DecimalType) logicalType; + return "org.apache.flink.api.common.typeutils.base.NormalizedKeyUtil.putLongNormalizedKey(" + + "rowData.getDecimal(" + sortIndex + ", " + decimalType.getPrecision() + ", " + + decimalType.getScale() + ").toUnscaledLong(), target, " + offset + ", " + valueBytes + ");"; + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampNormalizedKeyExpression(sortIndex, ((TimestampType) logicalType).getPrecision(), offset, + valueBytes); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampNormalizedKeyExpression(sortIndex, ((LocalZonedTimestampType) logicalType).getPrecision(), + offset, valueBytes); + case TIMESTAMP_WITH_TIME_ZONE: + return timestampNormalizedKeyExpression(sortIndex, ((ZonedTimestampType) logicalType).getPrecision(), offset, + valueBytes); + default: + throw new IllegalArgumentException("Unsupported normalized key field type: " + logicalType); + } + } + + private String timestampNormalizedKeyExpression(int sortIndex, int precision, String offset, int valueBytes) { + return "putTimestampNormalizedKey(rowData.getTimestamp(" + sortIndex + ", " + precision + "), target, " + + offset + ", " + valueBytes + ");"; + } + + private static class NormalizedKeyField { + private final int sortIndex; + private final int valueBytes; + private final boolean fullyDetermines; + + private NormalizedKeyField(int sortIndex, int valueBytes, boolean fullyDetermines) { + this.sortIndex = sortIndex; + this.valueBytes = valueBytes; + this.fullyDetermines = fullyDetermines; + } + + private int totalBytes() { + return valueBytes + 1; + } + + private boolean fullyDetermines() { + return fullyDetermines; } - return new SortCodeGenerator(tableConfig, Thread.currentThread().getContextClassLoader(), rowType, builder.build()); } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java index f67be66b3c3e6..6e66957988253 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java @@ -74,7 +74,6 @@ import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.binary.BinaryRowData; -import org.apache.flink.table.planner.codegen.sort.SortCodeGenerator; import org.apache.flink.table.runtime.generated.NormalizedKeyComputer; import org.apache.flink.table.runtime.generated.RecordComparator; import org.apache.flink.table.runtime.operators.TableStreamOperator; @@ -322,8 +321,9 @@ private Iterator readRecordsForGroupBaseFiles(List private BinaryExternalSorter initSorter() { ClassLoader cl = getContainingTask().getUserCodeClassLoader(); - NormalizedKeyComputer computer = createSortCodeGenerator().generateNormalizedKeyComputer("SortComputer").newInstance(cl); - RecordComparator comparator = createSortCodeGenerator().generateRecordComparator("SortComparator").newInstance(cl); + SortOperatorGen sortOperatorGen = createSortOperatorGen(); + NormalizedKeyComputer computer = sortOperatorGen.generateNormalizedKeyComputer("SortComputer").newInstance(cl); + RecordComparator comparator = sortOperatorGen.generateRecordComparator("SortComparator").newInstance(cl); MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager(); BinaryExternalSorter sorter = Utils.getBinaryExternalSorter( @@ -345,10 +345,9 @@ private BinaryExternalSorter initSorter() { return sorter; } - private SortCodeGenerator createSortCodeGenerator() { - SortOperatorGen sortOperatorGen = new SortOperatorGen(rowType, + private SortOperatorGen createSortOperatorGen() { + return new SortOperatorGen(rowType, conf.get(FlinkOptions.CLUSTERING_SORT_COLUMNS).split(",")); - return sortOperatorGen.createSortCodeGenerator(); } private String getFileIds(List clusteringOperations) { diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java index 9a2b07a7b81b1..d2f6419515ac2 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java @@ -49,7 +49,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; import org.slf4j.Logger; @@ -60,6 +59,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static org.apache.hudi.sink.utils.FlinkTransformationUtils.setManagedMemoryWeight; + /** * Flink hudi clustering program that can be executed manually. */ @@ -398,7 +399,7 @@ private void cluster() throws Exception { .setParallelism(clusteringParallelism); if (OptionsResolver.sortClusteringEnabled(conf)) { - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkTransformationUtils.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkTransformationUtils.java new file mode 100644 index 0000000000000..c19a9220f71b0 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkTransformationUtils.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.utils; + +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.core.memory.ManagedMemoryUseCase; + +/** + * Utilities for Flink transformations. + */ +public final class FlinkTransformationUtils { + private FlinkTransformationUtils() { + } + + public static void setManagedMemoryWeight(Transformation transformation, long memoryBytes) { + if (memoryBytes <= 0) { + return; + } + int weight = Math.max(1, (int) (memoryBytes >> 20)); // bytes to MiB + transformation.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, weight) + .ifPresent(previousWeight -> { + throw new IllegalStateException("Managed memory weight has been set, this should not happen."); + }); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java index 036f724b00d56..8d39fae75092d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java @@ -79,7 +79,6 @@ import org.apache.flink.streaming.api.operators.ProcessOperator; import org.apache.flink.streaming.api.transformations.OneInputTransformation; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.apache.flink.table.types.logical.RowType; @@ -154,7 +153,7 @@ public static DataStream bulkInsert(Configuration conf, RowType rowType SortOperatorGen sortOperatorGen = BucketBulkInsertWriterHelper.getFileIdSorterGen(rowTypeWithFileId); dataStream = dataStream.transform("file_sorter", typeInfo, sortOperatorGen.createSortOperator(conf)) .setParallelism(PARALLELISM_VALUE); - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + FlinkTransformationUtils.setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); } } else if (!FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.PARTITION_PATH_FIELD)) { @@ -184,7 +183,7 @@ public static DataStream bulkInsert(Configuration conf, RowType rowType .transform(isNeededSortInput ? "sorter:(partition_key, record_key)" : "sorter:(partition_key)", InternalTypeInfo.of(rowType), sortOperatorGen.createSortOperator(conf)) .setParallelism(PARALLELISM_VALUE); - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + FlinkTransformationUtils.setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); } } @@ -563,7 +562,7 @@ public static DataStreamSink cluster(Configuration conf, new ClusteringOperator(conf, rowType)) .setParallelism(conf.get(FlinkOptions.CLUSTERING_TASKS)); if (OptionsResolver.sortClusteringEnabled(conf)) { - ExecNodeUtil.setManagedMemoryWeight(clusteringStream.getTransformation(), + FlinkTransformationUtils.setManagedMemoryWeight(clusteringStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); } DataStreamSink clusteringCommitEventDataStream = clusteringStream.addSink(new ClusteringCommitSink(conf)) @@ -606,7 +605,7 @@ public static String getTablePath(Configuration conf) { public static void declareManagedMemoryIfNecessary(Configuration conf, DataStream dataStream, Supplier bufferSizeSupplier) { if (OptionsResolver.isManagedMemoryBufferEnabled(conf)) { - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), bufferSizeSupplier.get()); + FlinkTransformationUtils.setManagedMemoryWeight(dataStream.getTransformation(), bufferSizeSupplier.get()); } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java index 107e8673c42b7..c1b76db677ab6 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java @@ -44,9 +44,9 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.operators.ProcessOperator; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.types.logical.RowType; +import static org.apache.hudi.sink.utils.FlinkTransformationUtils.setManagedMemoryWeight; import static org.apache.hudi.sink.utils.Pipelines.opUID; /** @@ -227,7 +227,7 @@ public static DataStream clusterV2(Configuration conf, RowType rowType, new ClusteringOperator(conf, rowType)) .setParallelism(conf.get(FlinkOptions.CLUSTERING_TASKS)); if (OptionsResolver.sortClusteringEnabled(conf)) { - ExecNodeUtil.setManagedMemoryWeight(clusteringStream.getTransformation(), + setManagedMemoryWeight(clusteringStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); } return clusteringStream.transform( diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/sort/TestSortOperatorGen.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/sort/TestSortOperatorGen.java new file mode 100644 index 0000000000000..8a6c7169e73b9 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/sort/TestSortOperatorGen.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.bulk.sort; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.runtime.generated.NormalizedKeyComputer; +import org.apache.flink.table.runtime.generated.RecordComparator; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarBinaryType; +import org.apache.flink.table.types.logical.VarCharType; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test cases for {@link SortOperatorGen}. + */ +class TestSortOperatorGen { + + @Test + void testGeneratedRecordComparator() { + RowType rowType = RowType.of( + new LogicalType[] { + new IntType(), + new VarCharType(), + new VarBinaryType(), + new DecimalType(10, 2), + new TimestampType(3) + }, + new String[] {"id", "name", "bytes", "amount", "ts"}); + + SortOperatorGen sortOperatorGen = + new SortOperatorGen(rowType, new String[] {"id", "name", "bytes", "amount", "ts"}); + RecordComparator comparator = sortOperatorGen.generateRecordComparator("TestSortComparator") + .newInstance(Thread.currentThread().getContextClassLoader()); + + GenericRowData row1 = row(1, "a", new byte[] {1, 2}, "1.00", 1000); + GenericRowData row2 = row(1, "a", new byte[] {1, 3}, "1.00", 1000); + GenericRowData row3 = row(1, "a", new byte[] {1, 3}, "2.00", 1000); + GenericRowData row4 = row(1, "a", new byte[] {1, 3}, "2.00", 2000); + GenericRowData nullRow = row(null, "a", new byte[] {1, 3}, "2.00", 2000); + + assertTrue(comparator.compare(row1, row2) < 0); + assertTrue(comparator.compare(row2, row3) < 0); + assertTrue(comparator.compare(row3, row4) < 0); + assertTrue(comparator.compare(nullRow, row1) > 0); + assertTrue(comparator.compare(row1, nullRow) < 0); + assertEquals(0, comparator.compare(row4, row4)); + } + + @Test + void testGeneratedNormalizedKeyComputer() { + RowType rowType = RowType.of(new LogicalType[] {new IntType()}, new String[] {"id"}); + NormalizedKeyComputer computer = new SortOperatorGen(rowType, new String[] {"id"}) + .generateNormalizedKeyComputer("TestSortComputer") + .newInstance(Thread.currentThread().getContextClassLoader()); + MemorySegment segment1 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + MemorySegment segment2 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + + computer.putKey(GenericRowData.of(1), segment1, 0); + computer.putKey(GenericRowData.of(2), segment2, 0); + + assertTrue(computer.getNumKeyBytes() > 1); + assertTrue(computer.isKeyFullyDetermines()); + assertTrue(computer.compareKey(segment1, 0, segment2, 0) < 0); + } + + @Test + void testGeneratedNormalizedKeyComputerWithNullsLast() { + RowType rowType = RowType.of(new LogicalType[] {new IntType()}, new String[] {"id"}); + NormalizedKeyComputer computer = new SortOperatorGen(rowType, new String[] {"id"}) + .generateNormalizedKeyComputer("TestSortComputer") + .newInstance(Thread.currentThread().getContextClassLoader()); + MemorySegment segment1 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + MemorySegment segment2 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + + computer.putKey(GenericRowData.of(1), segment1, 0); + computer.putKey(GenericRowData.of((Object) null), segment2, 0); + + assertTrue(computer.compareKey(segment1, 0, segment2, 0) < 0); + } + + @Test + void testGeneratedNormalizedKeyComputerStopsAfterVariableLengthPrefix() { + RowType rowType = RowType.of( + new LogicalType[] {new VarCharType(), new IntType()}, + new String[] {"name", "id"}); + NormalizedKeyComputer computer = new SortOperatorGen(rowType, new String[] {"name", "id"}) + .generateNormalizedKeyComputer("TestSortComputer") + .newInstance(Thread.currentThread().getContextClassLoader()); + MemorySegment segment1 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + MemorySegment segment2 = MemorySegmentFactory.wrap(new byte[computer.getNumKeyBytes()]); + + computer.putKey(GenericRowData.of(StringData.fromString("abcdefghx"), 2), segment1, 0); + computer.putKey(GenericRowData.of(StringData.fromString("abcdefghy"), 1), segment2, 0); + + assertFalse(computer.isKeyFullyDetermines()); + assertEquals(0, computer.compareKey(segment1, 0, segment2, 0)); + } + + private static GenericRowData row(Integer id, String name, byte[] bytes, String amount, long timestampMillis) { + return GenericRowData.of( + id, + StringData.fromString(name), + bytes, + DecimalData.fromBigDecimal(new BigDecimal(amount), 10, 2), + TimestampData.fromEpochMillis(timestampMillis)); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java index 4fbf90e2e421a..c7816afc38ae9 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java @@ -65,7 +65,6 @@ import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.table.api.internal.TableEnvironmentImpl; -import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.Row; @@ -84,6 +83,7 @@ import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; +import static org.apache.hudi.sink.utils.FlinkTransformationUtils.setManagedMemoryWeight; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -193,7 +193,7 @@ public void testHoodieFlinkClustering() throws Exception { new ClusteringOperator(conf, rowType)) .setParallelism(clusteringPlan.getInputGroups().size()); - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); dataStream @@ -398,7 +398,7 @@ public void testHoodieFlinkClusteringScheduleAfterArchive() throws Exception { new ClusteringOperator(conf, rowType)) .setParallelism(clusteringPlan.getInputGroups().size()); - ExecNodeUtil.setManagedMemoryWeight( + setManagedMemoryWeight( dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); @@ -661,7 +661,7 @@ private void runCluster(RowType rowType) throws Exception { new ClusteringOperator(conf, rowType)) .setParallelism(clusteringPlan.getInputGroups().size()); - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); dataStream @@ -765,7 +765,7 @@ private void runOfflineCluster(TableEnvironment tableEnv, Configuration conf) th new ClusteringOperator(conf, rowType)) .setParallelism(clusteringPlan.getInputGroups().size()); - ExecNodeUtil.setManagedMemoryWeight(dataStream.getTransformation(), + setManagedMemoryWeight(dataStream.getTransformation(), conf.get(FlinkOptions.WRITE_SORT_MEMORY) * 1024L * 1024L); dataStream diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java index 379a769ec8153..2edc2f1255a59 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java @@ -46,7 +46,6 @@ import org.apache.hudi.utils.TestConfigurations; import org.apache.hudi.utils.TestData; -import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; import org.apache.flink.table.api.DataTypes; @@ -283,7 +282,7 @@ public void testCreateTable() throws Exception { final ResolvedCatalogTable singleKeyMultiplePartitionTable = new ResolvedCatalogTable( CatalogUtils.createCatalogTable( Schema.newBuilder().fromResolvedSchema(CREATE_TABLE_SCHEMA).build(), - Lists.newArrayList("par1", "par2"), + Arrays.asList("par1", "par2"), EXPECTED_OPTIONS, "test"), CREATE_TABLE_SCHEMA @@ -301,7 +300,7 @@ public void testCreateTable() throws Exception { final ResolvedCatalogTable multipleKeySinglePartitionTable = new ResolvedCatalogTable( CatalogUtils.createCatalogTable( Schema.newBuilder().fromResolvedSchema(CREATE_MULTI_KEY_TABLE_SCHEMA).build(), - Lists.newArrayList("par1"), + Collections.singletonList("par1"), EXPECTED_OPTIONS, "test"), CREATE_TABLE_SCHEMA diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java index 7f67a0281e5cd..6f778b9bc2974 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Schema; import org.apache.flink.table.catalog.AbstractCatalog; @@ -71,6 +70,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -126,7 +126,7 @@ public class TestHoodieHiveCatalog extends BaseTestHoodieCatalog { .column("par2", DataTypes.STRING()) .primaryKey("uuid") .build(); - List multiPartitions = Lists.newArrayList("par1", "par2"); + List multiPartitions = Arrays.asList("par1", "par2"); private static HoodieHiveCatalog hoodieCatalog; private final ObjectPath tablePath = new ObjectPath(TEST_DEFAULT_DATABASE, "test"); @@ -276,7 +276,7 @@ public void testCreateAndGetHoodieTable(HoodieTableType tableType) throws Except assertEquals(keyGeneratorClassName, NonpartitionedAvroKeyGenerator.class.getName()); // validate the order of partition fields in the multi-partition table - List multiPartitions = Lists.newArrayList("par2", "par1"); + List multiPartitions = Arrays.asList("par2", "par1"); ObjectPath multiPartitionsTablePath = new ObjectPath("default", "tb_mp_" + System.currentTimeMillis()); CatalogTable multiPartitionsTable = CatalogUtils.createCatalogTable(singleKeyMultiPartitionTableSchema, multiPartitions, options, "multi-partition hudi table"); diff --git a/hudi-flink-datasource/hudi-flink1.18.x/pom.xml b/hudi-flink-datasource/hudi-flink1.18.x/pom.xml index 4df05d62af7a0..72adebafa4b1b 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.18.x/pom.xml @@ -127,12 +127,6 @@ ${flink1.18.version} provided - - org.apache.flink - flink-table-planner_2.12 - ${flink1.18.version} - provided - diff --git a/hudi-flink-datasource/hudi-flink1.19.x/pom.xml b/hudi-flink-datasource/hudi-flink1.19.x/pom.xml index 5c29d042577e0..983dde06bd55a 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.19.x/pom.xml @@ -127,12 +127,6 @@ ${flink1.19.version} provided - - org.apache.flink - flink-table-planner_2.12 - ${flink1.19.version} - provided - diff --git a/hudi-flink-datasource/hudi-flink1.20.x/pom.xml b/hudi-flink-datasource/hudi-flink1.20.x/pom.xml index fee65624c6857..70838282e5ac1 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.20.x/pom.xml @@ -127,12 +127,6 @@ ${flink1.20.version} provided - - org.apache.flink - flink-table-planner_2.12 - ${flink1.20.version} - provided - diff --git a/hudi-flink-datasource/hudi-flink2.0.x/pom.xml b/hudi-flink-datasource/hudi-flink2.0.x/pom.xml index 90c40c1f8959e..322779f7c1197 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink2.0.x/pom.xml @@ -127,12 +127,6 @@ ${flink2.0.version} provided - - org.apache.flink - flink-table-planner_2.12 - ${flink2.0.version} - provided - diff --git a/hudi-flink-datasource/hudi-flink2.1.x/pom.xml b/hudi-flink-datasource/hudi-flink2.1.x/pom.xml index f6665d3c42b35..f8d07669b2a8e 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink2.1.x/pom.xml @@ -121,12 +121,6 @@ ${flink2.1.version} provided - - org.apache.flink - flink-table-planner_2.12 - ${flink2.1.version} - provided - diff --git a/pom.xml b/pom.xml index 35ac14994c179..1d5d6e17ebfda 100644 --- a/pom.xml +++ b/pom.xml @@ -2514,14 +2514,11 @@ *:*_2.12 org.apache.flink:*_2.12 From 605d2e4bff54fd675fa322d51851617f773707e8 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Fri, 3 Jul 2026 09:51:35 +0800 Subject: [PATCH 115/255] chore(glue-sync): Ignore EntityNotFoundException when dropping Glue partitions (#19142) (cherry picked from commit 667626cb9b37719d222b7840cdb5ab8f0a41110d) --- .../aws/sync/AWSGlueCatalogSyncClient.java | 20 ++++++- .../hudi/aws/sync/TestAWSGlueSyncClient.java | 56 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index 884b5c53a25d1..10f0b5a1592d2 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -71,6 +71,7 @@ import software.amazon.awssdk.services.glue.model.GetPartitionsResponse; import software.amazon.awssdk.services.glue.model.GetTableRequest; import software.amazon.awssdk.services.glue.model.KeySchemaElement; +import software.amazon.awssdk.services.glue.model.PartitionError; import software.amazon.awssdk.services.glue.model.PartitionIndex; import software.amazon.awssdk.services.glue.model.PartitionIndexDescriptor; import software.amazon.awssdk.services.glue.model.PartitionInput; @@ -137,6 +138,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient { private static final int MAX_PARTITIONS_PER_CHANGE_REQUEST = 100; private static final int MAX_PARTITIONS_PER_READ_REQUEST = 1000; private static final int MAX_DELETE_PARTITIONS_PER_REQUEST = 25; + private static final String ENTITY_NOT_FOUND_ERROR_CODE = "EntityNotFoundException"; protected final GlueAsyncClient awsGlue; private static final String GLUE_PARTITION_INDEX_ENABLE = "partition_filtering.enabled"; private static final int PARTITION_INDEX_MAX_NUMBER = 3; @@ -437,8 +439,22 @@ private void dropPartitionsInternal(String tableName, List partitionsToD BatchDeletePartitionResponse response = future.get(); if (CollectionUtils.nonEmpty(response.errors())) { - throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName) - + " with error(s): " + response.errors()); + // Dropping a partition that no longer exists is a no-op for an idempotent cleanup, so + // ignore EntityNotFoundException errors and only fail on other (e.g. permission/throttling) errors. + Map> errorsByIgnorable = response.errors().stream() + .collect(Collectors.partitioningBy( + error -> ENTITY_NOT_FOUND_ERROR_CODE.equals(error.errorDetail().errorCode()))); + List ignorableErrors = errorsByIgnorable.get(true); + if (!ignorableErrors.isEmpty()) { + log.info("Ignored dropping {} non-existent partition(s) from table {}: {}", ignorableErrors.size(), + tableId(databaseName, tableName), + ignorableErrors.stream().map(PartitionError::partitionValues).collect(Collectors.toList())); + } + List realErrors = errorsByIgnorable.get(false); + if (!realErrors.isEmpty()) { + throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName) + + " with error(s): " + realErrors); + } } } catch (Exception e) { throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName), e); diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java index f4822e32f05de..8c3b24c0b6776 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java @@ -647,6 +647,62 @@ void testDropPartitions_ErrorResponses() { assertTrue(ex.getCause().getCause().getMessage().contains("Fail to drop partitions")); } + @Test + void testDropPartitions_IgnoresEntityNotFound() { + String tableName = "tbl"; + List toDrop = List.of("2025/05/19"); + + // Glue reports EntityNotFoundException for a partition that no longer exists; it should be ignored. + ErrorDetail detail = ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build(); + PartitionError pe = PartitionError.builder().partitionValues(Arrays.asList("2025", "05", "19")).errorDetail(detail).build(); + BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder() + .errors(Collections.singletonList(pe)) + .build(); + when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class))) + .thenReturn(CompletableFuture.completedFuture(resp)); + + // should swallow the EntityNotFound error and not throw + awsGlueSyncClient.dropPartitions(tableName, toDrop); + + verify(mockAwsGlue).batchDeletePartition(any(BatchDeletePartitionRequest.class)); + } + + @Test + void testDropPartitions_MixedErrorsStillThrow() { + String tableName = "tbl"; + List toDrop = Arrays.asList("2025/05/19", "2025/05/18"); + + // One ignorable EntityNotFound error and one real error -> should still throw for the real one. + PartitionError ignorable = PartitionError.builder() + .partitionValues(Arrays.asList("2025", "05", "19")) + .errorDetail(ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build()) + .build(); + PartitionError real = PartitionError.builder() + .partitionValues(Arrays.asList("2025", "05", "18")) + .errorDetail(ErrorDetail.builder().errorCode("InternalServiceException").build()) + .build(); + BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder() + .errors(Arrays.asList(ignorable, real)) + .build(); + when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class))) + .thenReturn(CompletableFuture.completedFuture(resp)); + + HoodieGlueSyncException ex = assertThrows( + HoodieGlueSyncException.class, + () -> awsGlueSyncClient.dropPartitions(tableName, toDrop) + ); + // Walk the full cause chain: the error list is nested a few wrappers deep. + StringBuilder chain = new StringBuilder(); + for (Throwable t = ex; t != null; t = t.getCause()) { + chain.append(t.getMessage()).append('\n'); + } + String messages = chain.toString(); + assertTrue(messages.contains("Fail to drop partitions")); + // Only the real error should be surfaced, not the ignored EntityNotFound one. + assertTrue(messages.contains("InternalServiceException")); + assertFalse(messages.contains(EntityNotFoundException.class.getSimpleName())); + } + @Disabled("Integration test – requires real AWS environment") @Test void testIntegrationTableExists_RealGlueEnvironment() { From 6efa2525845f9d2a5c5d800385a921a40a0c22e0 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 2 Jul 2026 22:43:52 -0700 Subject: [PATCH 116/255] fix(storage-format): return all records when scanning multi-block native HFiles (#19146) (cherry picked from commit 6f2fbaaf6e4bd58e8fb2c7eff9c604afc5ff25c3) --- .../apache/hudi/io/hfile/HFileDataBlock.java | 6 +- .../io/hfile/TestHFileMultiBlockScan.java | 94 +++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java index cbf3e719f6a02..551cfd2c961b6 100644 --- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java +++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java @@ -53,7 +53,9 @@ public class HFileDataBlock extends HFileBlock { // so the latest timestamp is used. private static final long LATEST_TIMESTAMP = Long.MAX_VALUE; - // End offset of content in the block, relative to the start of the start of the block + // End offset of content in the block, relative to the start of the block. The key-values + // occupy exactly uncompressedSizeWithoutHeader bytes after the header; the checksum trails + // the content and is not part of it, so it must not be subtracted here. protected final int uncompressedContentEndRelativeOffset; private final List entriesToWrite = new ArrayList<>(); @@ -64,7 +66,7 @@ protected HFileDataBlock(HFileContext context, super(context, HFileBlockType.DATA, byteBuff, startOffsetInBuff); this.uncompressedContentEndRelativeOffset = - this.uncompressedEndOffset - this.sizeCheckSum - this.startOffsetInBuff; + this.uncompressedEndOffset - this.startOffsetInBuff; } // For write purpose. diff --git a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java new file mode 100644 index 0000000000000..e36410f856b56 --- /dev/null +++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.hfile; + +import org.apache.hudi.io.ByteArraySeekableDataInputStream; +import org.apache.hudi.io.ByteBufferBackedInputStream; +import org.apache.hudi.io.compress.CompressionCodec; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.ByteArrayOutputStream; +import java.util.Locale; + +import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reproduces a full-scan record drop on native-writer HFiles that span many data blocks. A large + * block size packs many records per block; the trailer entry count is correct, but a + * {@code seekTo()} then {@code next()} forward scan stops short of a block's content end because + * the content-end bound subtracts the trailing checksum size. The drop only surfaces once the + * checksum region exceeds the last record's size, which is why small-block tests never caught it. + */ +public class TestHFileMultiBlockScan { + + @ParameterizedTest + @CsvSource({ + "NONE, 64, 5000", "GZIP, 64, 5000", + "NONE, 1048576, 200000", "GZIP, 1048576, 200000", + "NONE, 65536, 200000", "GZIP, 65536, 200000" + }) + public void fullScanReturnsAllRecords(String codec, int blockSize, int numRecords) throws Exception { + HFileContext context = new HFileContext.Builder() + .blockSize(blockSize) + .compressionCodec(CompressionCodec.valueOf(codec)) + .build(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (HFileWriter writer = new HFileWriterImpl(context, baos)) { + for (int i = 0; i < numRecords; i++) { + writer.append(key(i), getUTF8Bytes(value(i))); + } + } + byte[] fileBytes = baos.toByteArray(); + + try (HFileReader reader = new HFileReaderImpl( + new ByteArraySeekableDataInputStream(new ByteBufferBackedInputStream(fileBytes)), + fileBytes.length)) { + reader.initializeMetadata(); + assertEquals(numRecords, reader.getNumKeyValueEntries(), + "trailer entry count for codec=" + codec + " blockSize=" + blockSize); + + String scenario = "codec=" + codec + " blockSize=" + blockSize + " numRecords=" + numRecords; + int scanned = 0; + int firstSkipAt = -1; + boolean hasNext = reader.seekTo(); + while (hasNext) { + KeyValue kv = reader.getKeyValue().get(); + if (firstSkipAt < 0 && !key(scanned).equals(kv.getKey().getContentInString())) { + firstSkipAt = scanned; + } + scanned++; + hasNext = reader.next(); + } + assertEquals(numRecords, scanned, + "forward scan returned fewer records than written; " + scenario + + " firstSkipAtPosition=" + firstSkipAt); + } + } + + private static String key(int i) { + return String.format(Locale.ROOT, "%010d", i); + } + + private static String value(int i) { + return String.format(Locale.ROOT, "value-%010d", i); + } +} From 2acbe6514ee0fa16d0123ce705dd92bab3606c1f Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 00:19:44 +0800 Subject: [PATCH 117/255] fix(build): restore imports dropped by earlier cherry-picks Two earlier picks onto release-1.2.1 dropped imports that are still referenced, breaking the branch build: - HoodieJsonPayload: 4a2c490dfdef (#18967) collapsed both the HoodieSchema and FileIOUtils imports into a single HoodieAvroSchemaCache import. Dropping HoodieSchema was correct, but FileIOUtils is still used by unCompressData, so hudi-common failed to compile with "cannot find symbol: variable FileIOUtils". - TestGlobalRecordLevelIndexBackend: e3e003b6d1dd (#18894) removed the last assertThrows call but left the static import, tripping the checkstyle UnusedImports rule in hudi-flink. Both are release-branch-only regressions; master carries the correct imports and is unaffected. --- .../src/main/java/org/apache/hudi/common/HoodieJsonPayload.java | 1 + .../partitioner/index/TestGlobalRecordLevelIndexBackend.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java index e89b10cccf283..463f773b82150 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.schema.HoodieAvroSchemaCache; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.io.util.FileIOUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java index 37ac44d6ae06f..1241b720af703 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java @@ -44,7 +44,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.apache.hudi.common.model.HoodieTableType.COPY_ON_WRITE; import static org.mockito.Mockito.mock; From f0fa7a198506f5ebbd713b71ebb6ee4a6ffbaf70 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 2 Jul 2026 22:47:56 -0700 Subject: [PATCH 118/255] refactor(spark): consolidate the vendored AvroUtils fork into hudi-spark-common (#19147) The vendored spark-avro AvroUtils.scala is duplicated across all six Spark version modules (hudi-spark3.3.x through hudi-spark4.2.x). The six copies are identical except for a stray space inside one import line of the 3.3.x copy. Move a single copy to hudi-spark-common, which every version module already depends on, and delete the six duplicates. The consolidated copy omits the upstream RowReader trait and the three imports only it used: the trait references the version-specific vendored AvroDeserializer, which stays in the version modules and is not visible to hudi-spark-common, and it has no usages anywhere in the repo. The members the version modules actually use (toFieldStr, AvroMatchedField, AvroSchemaHelper) are unchanged, and IncompatibleSchemaException still resolves from hudi-spark-client, same as before. Consolidating also gives the file a unique repo path, so it re-enters the coverage denominator: the six identically-pathed copies were dropped by report-path resolution and invisible to Codecov. (cherry picked from commit 9953687a2c9351c42da313548f86e471d1dd810d) --- .../org/apache/spark/sql/avro/AvroUtils.scala | 45 +--- .../org/apache/spark/sql/avro/AvroUtils.scala | 227 ------------------ .../org/apache/spark/sql/avro/AvroUtils.scala | 227 ------------------ .../org/apache/spark/sql/avro/AvroUtils.scala | 227 ------------------ .../org/apache/spark/sql/avro/AvroUtils.scala | 227 ------------------ 5 files changed, 3 insertions(+), 950 deletions(-) rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark-common}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (85%) delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala similarity index 85% rename from hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala rename to hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala index 028ebebf0bf59..b11987a40c675 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala @@ -18,10 +18,7 @@ package org.apache.spark.sql.avro import org.apache.avro.Schema -import org.apache.avro.file.FileReader -import org.apache.avro.generic.GenericRecord import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -34,6 +31,9 @@ import scala.collection.JavaConverters._ * This code is borrowed, so that we can better control compatibility w/in Spark minor * branches (3.2.x, 3.1.x, etc) * + * This copy omits the upstream `RowReader` trait: it is unused in Hudi and references + * the version-specific vendored `AvroDeserializer`, which is not visible to this module. + * * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY */ private[sql] object AvroUtils extends Logging { @@ -55,45 +55,6 @@ private[sql] object AvroUtils extends Logging { case _ => false } - // The trait provides iterator-like interface for reading records from an Avro file, - // deserializing and returning them as internal rows. - trait RowReader { - protected val fileReader: FileReader[GenericRecord] - protected val deserializer: AvroDeserializer - protected val stopPosition: Long - - private[this] var completed = false - private[this] var currentRow: Option[InternalRow] = None - - def hasNextRow: Boolean = { - while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { - fileReader.close() - completed = true - currentRow = None - } else { - val record = fileReader.next() - // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize - // potentially filters rows - currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] - } - } - currentRow.isDefined - } - - def nextRow: InternalRow = { - if (currentRow.isEmpty) { - hasNextRow - } - val returnRow = currentRow - currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted - returnRow.getOrElse { - throw new NoSuchElementException("next on empty iterator") - } - } - } - /** Wrapper for a pair of matched fields, one Catalyst and one corresponding Avro field. */ private[sql] case class AvroMatchedField( catalystField: StructField, diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala deleted file mode 100644 index 8aae6b442f8a1..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.avro.Schema -import org.apache.avro.file. FileReader -import org.apache.avro.generic.GenericRecord -import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.util.Locale - -import scala.collection.JavaConverters._ - -/** - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] object AvroUtils extends Logging { - - def supportsDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall { f => supportsDataType(f.dataType) } - - case ArrayType(elementType, _) => supportsDataType(elementType) - - case MapType(keyType, valueType, _) => - supportsDataType(keyType) && supportsDataType(valueType) - - case udt: UserDefinedType[_] => supportsDataType(udt.sqlType) - - case _: NullType => true - - case _ => false - } - - // The trait provides iterator-like interface for reading records from an Avro file, - // deserializing and returning them as internal rows. - trait RowReader { - protected val fileReader: FileReader[GenericRecord] - protected val deserializer: AvroDeserializer - protected val stopPosition: Long - - private[this] var completed = false - private[this] var currentRow: Option[InternalRow] = None - - def hasNextRow: Boolean = { - while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { - fileReader.close() - completed = true - currentRow = None - } else { - val record = fileReader.next() - // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize - // potentially filters rows - currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] - } - } - currentRow.isDefined - } - - def nextRow: InternalRow = { - if (currentRow.isEmpty) { - hasNextRow - } - val returnRow = currentRow - currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted - returnRow.getOrElse { - throw new NoSuchElementException("next on empty iterator") - } - } - } - - /** Wrapper for a pair of matched fields, one Catalyst and one corresponding Avro field. */ - private[sql] case class AvroMatchedField( - catalystField: StructField, - catalystPosition: Int, - avroField: Schema.Field) - - /** - * Helper class to perform field lookup/matching on Avro schemas. - * - * This will match `avroSchema` against `catalystSchema`, attempting to find a matching field in - * the Avro schema for each field in the Catalyst schema and vice-versa, respecting settings for - * case sensitivity. The match results can be accessed using the getter methods. - * - * @param avroSchema The schema in which to search for fields. Must be of type RECORD. - * @param catalystSchema The Catalyst schema to use for matching. - * @param avroPath The seq of parent field names leading to `avroSchema`. - * @param catalystPath The seq of parent field names leading to `catalystSchema`. - * @param positionalFieldMatch If true, perform field matching in a positional fashion - * (structural comparison between schemas, ignoring names); - * otherwise, perform field matching using field names. - */ - class AvroSchemaHelper( - avroSchema: Schema, - catalystSchema: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - positionalFieldMatch: Boolean) { - if (avroSchema.getType != Schema.Type.RECORD) { - throw new IncompatibleSchemaException( - s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}") - } - - private[this] val avroFieldArray = avroSchema.getFields.asScala.toArray - private[this] val fieldMap = avroSchema.getFields.asScala - .groupBy(_.name.toLowerCase(Locale.ROOT)) - .mapValues(_.toSeq) // toSeq needed for scala 2.13 - - /** The fields which have matching equivalents in both Avro and Catalyst schemas. */ - val matchedFields: Seq[AvroMatchedField] = catalystSchema.zipWithIndex.flatMap { - case (sqlField, sqlPos) => - getAvroField(sqlField.name, sqlPos).map(AvroMatchedField(sqlField, sqlPos, _)) - } - - /** - * Validate that there are no Catalyst fields which don't have a matching Avro field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. If `ignoreNullable` is false, - * consider nullable Catalyst fields to be eligible to be an extra field; otherwise, - * ignore nullable Catalyst fields when checking for extras. - */ - def validateNoExtraCatalystFields(ignoreNullable: Boolean): Unit = - catalystSchema.zipWithIndex.foreach { case (sqlField, sqlPos) => - if (getAvroField(sqlField.name, sqlPos).isEmpty && - (!ignoreNullable || !sqlField.nullable)) { - if (positionalFieldMatch) { - throw new IncompatibleSchemaException("Cannot find field at position " + - s"$sqlPos of ${toFieldStr(avroPath)} from Avro schema (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Cannot find ${toFieldStr(catalystPath :+ sqlField.name)} in Avro schema") - } - } - } - - /** - * Validate that there are no Avro fields which don't have a matching Catalyst field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. Only required (non-nullable) - * fields are checked; nullable fields are ignored. - */ - def validateNoExtraRequiredAvroFields(): Unit = { - val extraFields = avroFieldArray.toSet -- matchedFields.map(_.avroField) - extraFields.filterNot(isNullable).foreach { extraField => - if (positionalFieldMatch) { - throw new IncompatibleSchemaException(s"Found field '${extraField.name()}' at position " + - s"${extraField.pos()} of ${toFieldStr(avroPath)} from Avro schema but there is no " + - s"match in the SQL schema at ${toFieldStr(catalystPath)} (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Found ${toFieldStr(avroPath :+ extraField.name())} in Avro schema but there is no " + - "match in the SQL schema") - } - } - } - - /** - * Extract a single field from the contained avro schema which has the desired field name, - * performing the matching with proper case sensitivity according to SQLConf.resolver. - * - * @param name The name of the field to search for. - * @return `Some(match)` if a matching Avro field is found, otherwise `None`. - */ - private[avro] def getFieldByName(name: String): Option[Schema.Field] = { - - // get candidates, ignoring case of field name - val candidates = fieldMap.getOrElse(name.toLowerCase(Locale.ROOT), Seq.empty) - - // search candidates, taking into account case sensitivity settings - candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match { - case Seq(avroField) => Some(avroField) - case Seq() => None - case matches => throw new IncompatibleSchemaException(s"Searching for '$name' in Avro " + - s"schema at ${toFieldStr(avroPath)} gave ${matches.size} matches. Candidates: " + - matches.map(_.name()).mkString("[", ", ", "]") - ) - } - } - - /** Get the Avro field corresponding to the provided Catalyst field name/position, if any. */ - def getAvroField(fieldName: String, catalystPos: Int): Option[Schema.Field] = { - if (positionalFieldMatch) { - avroFieldArray.lift(catalystPos) - } else { - getFieldByName(fieldName) - } - } - } - - /** - * Convert a sequence of hierarchical field names (like `Seq(foo, bar)`) into a human-readable - * string representing the field, like "field 'foo.bar'". If `names` is empty, the string - * "top-level record" is returned. - */ - private[avro] def toFieldStr(names: Seq[String]): String = names match { - case Seq() => "top-level record" - case n => s"field '${n.mkString(".")}'" - } - - /** Return true iff `avroField` is nullable, i.e. `UNION` type and has `NULL` as an option. */ - private[avro] def isNullable(avroField: Schema.Field): Boolean = - avroField.schema().getType == Schema.Type.UNION && - avroField.schema().getTypes.asScala.exists(_.getType == Schema.Type.NULL) -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala deleted file mode 100644 index 8aae6b442f8a1..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.avro.Schema -import org.apache.avro.file. FileReader -import org.apache.avro.generic.GenericRecord -import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.util.Locale - -import scala.collection.JavaConverters._ - -/** - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] object AvroUtils extends Logging { - - def supportsDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall { f => supportsDataType(f.dataType) } - - case ArrayType(elementType, _) => supportsDataType(elementType) - - case MapType(keyType, valueType, _) => - supportsDataType(keyType) && supportsDataType(valueType) - - case udt: UserDefinedType[_] => supportsDataType(udt.sqlType) - - case _: NullType => true - - case _ => false - } - - // The trait provides iterator-like interface for reading records from an Avro file, - // deserializing and returning them as internal rows. - trait RowReader { - protected val fileReader: FileReader[GenericRecord] - protected val deserializer: AvroDeserializer - protected val stopPosition: Long - - private[this] var completed = false - private[this] var currentRow: Option[InternalRow] = None - - def hasNextRow: Boolean = { - while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { - fileReader.close() - completed = true - currentRow = None - } else { - val record = fileReader.next() - // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize - // potentially filters rows - currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] - } - } - currentRow.isDefined - } - - def nextRow: InternalRow = { - if (currentRow.isEmpty) { - hasNextRow - } - val returnRow = currentRow - currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted - returnRow.getOrElse { - throw new NoSuchElementException("next on empty iterator") - } - } - } - - /** Wrapper for a pair of matched fields, one Catalyst and one corresponding Avro field. */ - private[sql] case class AvroMatchedField( - catalystField: StructField, - catalystPosition: Int, - avroField: Schema.Field) - - /** - * Helper class to perform field lookup/matching on Avro schemas. - * - * This will match `avroSchema` against `catalystSchema`, attempting to find a matching field in - * the Avro schema for each field in the Catalyst schema and vice-versa, respecting settings for - * case sensitivity. The match results can be accessed using the getter methods. - * - * @param avroSchema The schema in which to search for fields. Must be of type RECORD. - * @param catalystSchema The Catalyst schema to use for matching. - * @param avroPath The seq of parent field names leading to `avroSchema`. - * @param catalystPath The seq of parent field names leading to `catalystSchema`. - * @param positionalFieldMatch If true, perform field matching in a positional fashion - * (structural comparison between schemas, ignoring names); - * otherwise, perform field matching using field names. - */ - class AvroSchemaHelper( - avroSchema: Schema, - catalystSchema: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - positionalFieldMatch: Boolean) { - if (avroSchema.getType != Schema.Type.RECORD) { - throw new IncompatibleSchemaException( - s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}") - } - - private[this] val avroFieldArray = avroSchema.getFields.asScala.toArray - private[this] val fieldMap = avroSchema.getFields.asScala - .groupBy(_.name.toLowerCase(Locale.ROOT)) - .mapValues(_.toSeq) // toSeq needed for scala 2.13 - - /** The fields which have matching equivalents in both Avro and Catalyst schemas. */ - val matchedFields: Seq[AvroMatchedField] = catalystSchema.zipWithIndex.flatMap { - case (sqlField, sqlPos) => - getAvroField(sqlField.name, sqlPos).map(AvroMatchedField(sqlField, sqlPos, _)) - } - - /** - * Validate that there are no Catalyst fields which don't have a matching Avro field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. If `ignoreNullable` is false, - * consider nullable Catalyst fields to be eligible to be an extra field; otherwise, - * ignore nullable Catalyst fields when checking for extras. - */ - def validateNoExtraCatalystFields(ignoreNullable: Boolean): Unit = - catalystSchema.zipWithIndex.foreach { case (sqlField, sqlPos) => - if (getAvroField(sqlField.name, sqlPos).isEmpty && - (!ignoreNullable || !sqlField.nullable)) { - if (positionalFieldMatch) { - throw new IncompatibleSchemaException("Cannot find field at position " + - s"$sqlPos of ${toFieldStr(avroPath)} from Avro schema (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Cannot find ${toFieldStr(catalystPath :+ sqlField.name)} in Avro schema") - } - } - } - - /** - * Validate that there are no Avro fields which don't have a matching Catalyst field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. Only required (non-nullable) - * fields are checked; nullable fields are ignored. - */ - def validateNoExtraRequiredAvroFields(): Unit = { - val extraFields = avroFieldArray.toSet -- matchedFields.map(_.avroField) - extraFields.filterNot(isNullable).foreach { extraField => - if (positionalFieldMatch) { - throw new IncompatibleSchemaException(s"Found field '${extraField.name()}' at position " + - s"${extraField.pos()} of ${toFieldStr(avroPath)} from Avro schema but there is no " + - s"match in the SQL schema at ${toFieldStr(catalystPath)} (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Found ${toFieldStr(avroPath :+ extraField.name())} in Avro schema but there is no " + - "match in the SQL schema") - } - } - } - - /** - * Extract a single field from the contained avro schema which has the desired field name, - * performing the matching with proper case sensitivity according to SQLConf.resolver. - * - * @param name The name of the field to search for. - * @return `Some(match)` if a matching Avro field is found, otherwise `None`. - */ - private[avro] def getFieldByName(name: String): Option[Schema.Field] = { - - // get candidates, ignoring case of field name - val candidates = fieldMap.getOrElse(name.toLowerCase(Locale.ROOT), Seq.empty) - - // search candidates, taking into account case sensitivity settings - candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match { - case Seq(avroField) => Some(avroField) - case Seq() => None - case matches => throw new IncompatibleSchemaException(s"Searching for '$name' in Avro " + - s"schema at ${toFieldStr(avroPath)} gave ${matches.size} matches. Candidates: " + - matches.map(_.name()).mkString("[", ", ", "]") - ) - } - } - - /** Get the Avro field corresponding to the provided Catalyst field name/position, if any. */ - def getAvroField(fieldName: String, catalystPos: Int): Option[Schema.Field] = { - if (positionalFieldMatch) { - avroFieldArray.lift(catalystPos) - } else { - getFieldByName(fieldName) - } - } - } - - /** - * Convert a sequence of hierarchical field names (like `Seq(foo, bar)`) into a human-readable - * string representing the field, like "field 'foo.bar'". If `names` is empty, the string - * "top-level record" is returned. - */ - private[avro] def toFieldStr(names: Seq[String]): String = names match { - case Seq() => "top-level record" - case n => s"field '${n.mkString(".")}'" - } - - /** Return true iff `avroField` is nullable, i.e. `UNION` type and has `NULL` as an option. */ - private[avro] def isNullable(avroField: Schema.Field): Boolean = - avroField.schema().getType == Schema.Type.UNION && - avroField.schema().getTypes.asScala.exists(_.getType == Schema.Type.NULL) -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala deleted file mode 100644 index 8aae6b442f8a1..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.avro.Schema -import org.apache.avro.file. FileReader -import org.apache.avro.generic.GenericRecord -import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.util.Locale - -import scala.collection.JavaConverters._ - -/** - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] object AvroUtils extends Logging { - - def supportsDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall { f => supportsDataType(f.dataType) } - - case ArrayType(elementType, _) => supportsDataType(elementType) - - case MapType(keyType, valueType, _) => - supportsDataType(keyType) && supportsDataType(valueType) - - case udt: UserDefinedType[_] => supportsDataType(udt.sqlType) - - case _: NullType => true - - case _ => false - } - - // The trait provides iterator-like interface for reading records from an Avro file, - // deserializing and returning them as internal rows. - trait RowReader { - protected val fileReader: FileReader[GenericRecord] - protected val deserializer: AvroDeserializer - protected val stopPosition: Long - - private[this] var completed = false - private[this] var currentRow: Option[InternalRow] = None - - def hasNextRow: Boolean = { - while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { - fileReader.close() - completed = true - currentRow = None - } else { - val record = fileReader.next() - // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize - // potentially filters rows - currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] - } - } - currentRow.isDefined - } - - def nextRow: InternalRow = { - if (currentRow.isEmpty) { - hasNextRow - } - val returnRow = currentRow - currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted - returnRow.getOrElse { - throw new NoSuchElementException("next on empty iterator") - } - } - } - - /** Wrapper for a pair of matched fields, one Catalyst and one corresponding Avro field. */ - private[sql] case class AvroMatchedField( - catalystField: StructField, - catalystPosition: Int, - avroField: Schema.Field) - - /** - * Helper class to perform field lookup/matching on Avro schemas. - * - * This will match `avroSchema` against `catalystSchema`, attempting to find a matching field in - * the Avro schema for each field in the Catalyst schema and vice-versa, respecting settings for - * case sensitivity. The match results can be accessed using the getter methods. - * - * @param avroSchema The schema in which to search for fields. Must be of type RECORD. - * @param catalystSchema The Catalyst schema to use for matching. - * @param avroPath The seq of parent field names leading to `avroSchema`. - * @param catalystPath The seq of parent field names leading to `catalystSchema`. - * @param positionalFieldMatch If true, perform field matching in a positional fashion - * (structural comparison between schemas, ignoring names); - * otherwise, perform field matching using field names. - */ - class AvroSchemaHelper( - avroSchema: Schema, - catalystSchema: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - positionalFieldMatch: Boolean) { - if (avroSchema.getType != Schema.Type.RECORD) { - throw new IncompatibleSchemaException( - s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}") - } - - private[this] val avroFieldArray = avroSchema.getFields.asScala.toArray - private[this] val fieldMap = avroSchema.getFields.asScala - .groupBy(_.name.toLowerCase(Locale.ROOT)) - .mapValues(_.toSeq) // toSeq needed for scala 2.13 - - /** The fields which have matching equivalents in both Avro and Catalyst schemas. */ - val matchedFields: Seq[AvroMatchedField] = catalystSchema.zipWithIndex.flatMap { - case (sqlField, sqlPos) => - getAvroField(sqlField.name, sqlPos).map(AvroMatchedField(sqlField, sqlPos, _)) - } - - /** - * Validate that there are no Catalyst fields which don't have a matching Avro field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. If `ignoreNullable` is false, - * consider nullable Catalyst fields to be eligible to be an extra field; otherwise, - * ignore nullable Catalyst fields when checking for extras. - */ - def validateNoExtraCatalystFields(ignoreNullable: Boolean): Unit = - catalystSchema.zipWithIndex.foreach { case (sqlField, sqlPos) => - if (getAvroField(sqlField.name, sqlPos).isEmpty && - (!ignoreNullable || !sqlField.nullable)) { - if (positionalFieldMatch) { - throw new IncompatibleSchemaException("Cannot find field at position " + - s"$sqlPos of ${toFieldStr(avroPath)} from Avro schema (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Cannot find ${toFieldStr(catalystPath :+ sqlField.name)} in Avro schema") - } - } - } - - /** - * Validate that there are no Avro fields which don't have a matching Catalyst field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. Only required (non-nullable) - * fields are checked; nullable fields are ignored. - */ - def validateNoExtraRequiredAvroFields(): Unit = { - val extraFields = avroFieldArray.toSet -- matchedFields.map(_.avroField) - extraFields.filterNot(isNullable).foreach { extraField => - if (positionalFieldMatch) { - throw new IncompatibleSchemaException(s"Found field '${extraField.name()}' at position " + - s"${extraField.pos()} of ${toFieldStr(avroPath)} from Avro schema but there is no " + - s"match in the SQL schema at ${toFieldStr(catalystPath)} (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Found ${toFieldStr(avroPath :+ extraField.name())} in Avro schema but there is no " + - "match in the SQL schema") - } - } - } - - /** - * Extract a single field from the contained avro schema which has the desired field name, - * performing the matching with proper case sensitivity according to SQLConf.resolver. - * - * @param name The name of the field to search for. - * @return `Some(match)` if a matching Avro field is found, otherwise `None`. - */ - private[avro] def getFieldByName(name: String): Option[Schema.Field] = { - - // get candidates, ignoring case of field name - val candidates = fieldMap.getOrElse(name.toLowerCase(Locale.ROOT), Seq.empty) - - // search candidates, taking into account case sensitivity settings - candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match { - case Seq(avroField) => Some(avroField) - case Seq() => None - case matches => throw new IncompatibleSchemaException(s"Searching for '$name' in Avro " + - s"schema at ${toFieldStr(avroPath)} gave ${matches.size} matches. Candidates: " + - matches.map(_.name()).mkString("[", ", ", "]") - ) - } - } - - /** Get the Avro field corresponding to the provided Catalyst field name/position, if any. */ - def getAvroField(fieldName: String, catalystPos: Int): Option[Schema.Field] = { - if (positionalFieldMatch) { - avroFieldArray.lift(catalystPos) - } else { - getFieldByName(fieldName) - } - } - } - - /** - * Convert a sequence of hierarchical field names (like `Seq(foo, bar)`) into a human-readable - * string representing the field, like "field 'foo.bar'". If `names` is empty, the string - * "top-level record" is returned. - */ - private[avro] def toFieldStr(names: Seq[String]): String = names match { - case Seq() => "top-level record" - case n => s"field '${n.mkString(".")}'" - } - - /** Return true iff `avroField` is nullable, i.e. `UNION` type and has `NULL` as an option. */ - private[avro] def isNullable(avroField: Schema.Field): Boolean = - avroField.schema().getType == Schema.Type.UNION && - avroField.schema().getTypes.asScala.exists(_.getType == Schema.Type.NULL) -} diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala deleted file mode 100644 index 8aae6b442f8a1..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.avro.Schema -import org.apache.avro.file. FileReader -import org.apache.avro.generic.GenericRecord -import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.util.Locale - -import scala.collection.JavaConverters._ - -/** - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] object AvroUtils extends Logging { - - def supportsDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall { f => supportsDataType(f.dataType) } - - case ArrayType(elementType, _) => supportsDataType(elementType) - - case MapType(keyType, valueType, _) => - supportsDataType(keyType) && supportsDataType(valueType) - - case udt: UserDefinedType[_] => supportsDataType(udt.sqlType) - - case _: NullType => true - - case _ => false - } - - // The trait provides iterator-like interface for reading records from an Avro file, - // deserializing and returning them as internal rows. - trait RowReader { - protected val fileReader: FileReader[GenericRecord] - protected val deserializer: AvroDeserializer - protected val stopPosition: Long - - private[this] var completed = false - private[this] var currentRow: Option[InternalRow] = None - - def hasNextRow: Boolean = { - while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { - fileReader.close() - completed = true - currentRow = None - } else { - val record = fileReader.next() - // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize - // potentially filters rows - currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] - } - } - currentRow.isDefined - } - - def nextRow: InternalRow = { - if (currentRow.isEmpty) { - hasNextRow - } - val returnRow = currentRow - currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted - returnRow.getOrElse { - throw new NoSuchElementException("next on empty iterator") - } - } - } - - /** Wrapper for a pair of matched fields, one Catalyst and one corresponding Avro field. */ - private[sql] case class AvroMatchedField( - catalystField: StructField, - catalystPosition: Int, - avroField: Schema.Field) - - /** - * Helper class to perform field lookup/matching on Avro schemas. - * - * This will match `avroSchema` against `catalystSchema`, attempting to find a matching field in - * the Avro schema for each field in the Catalyst schema and vice-versa, respecting settings for - * case sensitivity. The match results can be accessed using the getter methods. - * - * @param avroSchema The schema in which to search for fields. Must be of type RECORD. - * @param catalystSchema The Catalyst schema to use for matching. - * @param avroPath The seq of parent field names leading to `avroSchema`. - * @param catalystPath The seq of parent field names leading to `catalystSchema`. - * @param positionalFieldMatch If true, perform field matching in a positional fashion - * (structural comparison between schemas, ignoring names); - * otherwise, perform field matching using field names. - */ - class AvroSchemaHelper( - avroSchema: Schema, - catalystSchema: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - positionalFieldMatch: Boolean) { - if (avroSchema.getType != Schema.Type.RECORD) { - throw new IncompatibleSchemaException( - s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}") - } - - private[this] val avroFieldArray = avroSchema.getFields.asScala.toArray - private[this] val fieldMap = avroSchema.getFields.asScala - .groupBy(_.name.toLowerCase(Locale.ROOT)) - .mapValues(_.toSeq) // toSeq needed for scala 2.13 - - /** The fields which have matching equivalents in both Avro and Catalyst schemas. */ - val matchedFields: Seq[AvroMatchedField] = catalystSchema.zipWithIndex.flatMap { - case (sqlField, sqlPos) => - getAvroField(sqlField.name, sqlPos).map(AvroMatchedField(sqlField, sqlPos, _)) - } - - /** - * Validate that there are no Catalyst fields which don't have a matching Avro field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. If `ignoreNullable` is false, - * consider nullable Catalyst fields to be eligible to be an extra field; otherwise, - * ignore nullable Catalyst fields when checking for extras. - */ - def validateNoExtraCatalystFields(ignoreNullable: Boolean): Unit = - catalystSchema.zipWithIndex.foreach { case (sqlField, sqlPos) => - if (getAvroField(sqlField.name, sqlPos).isEmpty && - (!ignoreNullable || !sqlField.nullable)) { - if (positionalFieldMatch) { - throw new IncompatibleSchemaException("Cannot find field at position " + - s"$sqlPos of ${toFieldStr(avroPath)} from Avro schema (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Cannot find ${toFieldStr(catalystPath :+ sqlField.name)} in Avro schema") - } - } - } - - /** - * Validate that there are no Avro fields which don't have a matching Catalyst field, throwing - * [[IncompatibleSchemaException]] if such extra fields are found. Only required (non-nullable) - * fields are checked; nullable fields are ignored. - */ - def validateNoExtraRequiredAvroFields(): Unit = { - val extraFields = avroFieldArray.toSet -- matchedFields.map(_.avroField) - extraFields.filterNot(isNullable).foreach { extraField => - if (positionalFieldMatch) { - throw new IncompatibleSchemaException(s"Found field '${extraField.name()}' at position " + - s"${extraField.pos()} of ${toFieldStr(avroPath)} from Avro schema but there is no " + - s"match in the SQL schema at ${toFieldStr(catalystPath)} (using positional matching)") - } else { - throw new IncompatibleSchemaException( - s"Found ${toFieldStr(avroPath :+ extraField.name())} in Avro schema but there is no " + - "match in the SQL schema") - } - } - } - - /** - * Extract a single field from the contained avro schema which has the desired field name, - * performing the matching with proper case sensitivity according to SQLConf.resolver. - * - * @param name The name of the field to search for. - * @return `Some(match)` if a matching Avro field is found, otherwise `None`. - */ - private[avro] def getFieldByName(name: String): Option[Schema.Field] = { - - // get candidates, ignoring case of field name - val candidates = fieldMap.getOrElse(name.toLowerCase(Locale.ROOT), Seq.empty) - - // search candidates, taking into account case sensitivity settings - candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match { - case Seq(avroField) => Some(avroField) - case Seq() => None - case matches => throw new IncompatibleSchemaException(s"Searching for '$name' in Avro " + - s"schema at ${toFieldStr(avroPath)} gave ${matches.size} matches. Candidates: " + - matches.map(_.name()).mkString("[", ", ", "]") - ) - } - } - - /** Get the Avro field corresponding to the provided Catalyst field name/position, if any. */ - def getAvroField(fieldName: String, catalystPos: Int): Option[Schema.Field] = { - if (positionalFieldMatch) { - avroFieldArray.lift(catalystPos) - } else { - getFieldByName(fieldName) - } - } - } - - /** - * Convert a sequence of hierarchical field names (like `Seq(foo, bar)`) into a human-readable - * string representing the field, like "field 'foo.bar'". If `names` is empty, the string - * "top-level record" is returned. - */ - private[avro] def toFieldStr(names: Seq[String]): String = names match { - case Seq() => "top-level record" - case n => s"field '${n.mkString(".")}'" - } - - /** Return true iff `avroField` is nullable, i.e. `UNION` type and has `NULL` as an option. */ - private[avro] def isNullable(avroField: Schema.Field): Boolean = - avroField.schema().getType == Schema.Type.UNION && - avroField.schema().getTypes.asScala.exists(_.getType == Schema.Type.NULL) -} From 8327a728d8784d73901cdb7ef94ced9f2b90cf3c Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 2 Jul 2026 22:48:50 -0700 Subject: [PATCH 119/255] refactor(spark): share Spark 4.x partition-values and mapping classes in hudi-spark4-common (#19148) * refactor(spark): extract shared base classes for HoodiePartitionValues Move the InternalRow delegation methods shared by all Spark versions from Spark3HoodiePartitionValues and the per-version Spark 4.x copies into BaseHoodiePartitionValues in hudi-spark-common, and the getVariant delegation shared by all Spark 4.x versions into an abstract Spark4HoodiePartitionValues in hudi-spark4-common. The Spark 4.0/4.1/4.2 classes now only carry copy() plus, on 4.1/4.2, the geography/geometry getters introduced by Spark 4.1. * refactor(spark): share Spark 4.x mapping and internal-row impls in hudi-spark4-common Extract the bodies duplicated across the Spark 4.0/4.1/4.2 copies of HoodiePartitionFileSliceMapping, HoodiePartitionCDCFileGroupMapping, and HoodieInternalRow into shared traits/abstract classes in hudi-spark4-common, mirroring how hudi-spark3-common already shares them for the 3.x family. Version classes keep only genuine deltas: the Spark 4.1/4.2 geography and geometry getters, and the concrete type returned by copy() via a newInternalRow factory hook. (cherry picked from commit 39aa022c8f2686c0aa0e9745da1ed203ca35e581) --- .../hudi/BaseHoodiePartitionValues.scala | 108 ++++++++++++++++++ .../hudi/Spark3HoodiePartitionValues.scala | 81 +------------ ...k4HoodiePartitionCDCFileGroupMapping.scala | 35 ++++++ ...park4HoodiePartitionFileSliceMapping.scala | 41 +++++++ .../hudi/Spark4HoodiePartitionValues.scala | 37 ++++++ .../model/Spark4HoodieInternalRow.scala | 55 +++++++++ ...40HoodiePartitionCDCFileGroupMapping.scala | 11 +- ...ark40HoodiePartitionFileSliceMapping.scala | 11 +- .../hudi/Spark40HoodiePartitionValues.scala | 85 +------------- .../model/Spark40HoodieInternalRow.scala | 25 ++-- ...41HoodiePartitionCDCFileGroupMapping.scala | 9 +- ...ark41HoodiePartitionFileSliceMapping.scala | 11 +- .../hudi/Spark41HoodiePartitionValues.scala | 86 +------------- .../model/Spark41HoodieInternalRow.scala | 19 +-- 14 files changed, 307 insertions(+), 307 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseHoodiePartitionValues.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/client/model/Spark4HoodieInternalRow.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseHoodiePartitionValues.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseHoodiePartitionValues.scala new file mode 100644 index 0000000000000..a6fd7e238f69c --- /dev/null +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseHoodiePartitionValues.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} +import org.apache.spark.sql.types.{DataType, Decimal} +import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} + +/** + * Abstract base class for HoodiePartitionValues implementations. + * Contains all common delegation logic to the underlying InternalRow. + */ +abstract class BaseHoodiePartitionValues(val values: InternalRow) extends HoodiePartitionValues { + + override def numFields: Int = { + values.numFields + } + + override def setNullAt(i: Int): Unit = { + values.setNullAt(i) + } + + override def update(i: Int, value: Any): Unit = { + values.update(i, value) + } + + override def isNullAt(ordinal: Int): Boolean = { + values.isNullAt(ordinal) + } + + override def getBoolean(ordinal: Int): Boolean = { + values.getBoolean(ordinal) + } + + override def getByte(ordinal: Int): Byte = { + values.getByte(ordinal) + } + + override def getShort(ordinal: Int): Short = { + values.getShort(ordinal) + } + + override def getInt(ordinal: Int): Int = { + values.getInt(ordinal) + } + + override def getLong(ordinal: Int): Long = { + values.getLong(ordinal) + } + + override def getFloat(ordinal: Int): Float = { + values.getFloat(ordinal) + } + + override def getDouble(ordinal: Int): Double = { + values.getDouble(ordinal) + } + + override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = { + values.getDecimal(ordinal, precision, scale) + } + + override def getUTF8String(ordinal: Int): UTF8String = { + values.getUTF8String(ordinal) + } + + override def getBinary(ordinal: Int): Array[Byte] = { + values.getBinary(ordinal) + } + + override def getInterval(ordinal: Int): CalendarInterval = { + values.getInterval(ordinal) + } + + override def getStruct(ordinal: Int, numFields: Int): InternalRow = { + values.getStruct(ordinal, numFields) + } + + override def getArray(ordinal: Int): ArrayData = { + values.getArray(ordinal) + } + + override def getMap(ordinal: Int): MapData = { + values.getMap(ordinal) + } + + override def get(ordinal: Int, dataType: DataType): AnyRef = { + values.get(ordinal, dataType) + } +} diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3HoodiePartitionValues.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3HoodiePartitionValues.scala index bd50e3ebfca21..acdc175be2859 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3HoodiePartitionValues.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3HoodiePartitionValues.scala @@ -20,88 +20,11 @@ package org.apache.hudi import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.types.{DataType, Decimal} -import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} -case class Spark3HoodiePartitionValues(values: InternalRow) extends HoodiePartitionValues { - override def numFields: Int = { - values.numFields - } - - override def setNullAt(i: Int): Unit = { - values.setNullAt(i) - } - - override def update(i: Int, value: Any): Unit = { - values.update(i, value) - } +case class Spark3HoodiePartitionValues(override val values: InternalRow) + extends BaseHoodiePartitionValues(values) { override def copy(): InternalRow = { Spark3HoodiePartitionValues(values.copy()) } - - override def isNullAt(ordinal: Int): Boolean = { - values.isNullAt(ordinal) - } - - override def getBoolean(ordinal: Int): Boolean = { - values.getBoolean(ordinal) - } - - override def getByte(ordinal: Int): Byte = { - values.getByte(ordinal) - } - - override def getShort(ordinal: Int): Short = { - values.getShort(ordinal) - } - - override def getInt(ordinal: Int): Int = { - values.getInt(ordinal) - } - - override def getLong(ordinal: Int): Long = { - values.getLong(ordinal) - } - - override def getFloat(ordinal: Int): Float = { - values.getFloat(ordinal) - } - - override def getDouble(ordinal: Int): Double = { - values.getDouble(ordinal) - } - - override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = { - values.getDecimal(ordinal, precision, scale) - } - - override def getUTF8String(ordinal: Int): UTF8String = { - values.getUTF8String(ordinal) - } - - override def getBinary(ordinal: Int): Array[Byte] = { - values.getBinary(ordinal) - } - - override def getInterval(ordinal: Int): CalendarInterval = { - values.getInterval(ordinal) - } - - override def getStruct(ordinal: Int, numFields: Int): InternalRow = { - values.getStruct(ordinal, numFields) - } - - override def getArray(ordinal: Int): ArrayData = { - values.getArray(ordinal) - } - - override def getMap(ordinal: Int): MapData = { - values.getMap(ordinal) - } - - override def get(ordinal: Int, dataType: DataType): AnyRef = { - values.get(ordinal, dataType) - } } diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala new file mode 100644 index 0000000000000..428ad6a141124 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi + +import org.apache.hudi.common.table.cdc.HoodieCDCFileSplit + +/** + * Implementation of [[HoodiePartitionCDCFileGroupMapping]] shared by all Spark 4.x + * versions, mixed into the version-specific partition values classes. + */ +trait Spark4HoodiePartitionCDCFileGroupMapping extends HoodiePartitionCDCFileGroupMapping { + + protected def fileSplits: List[HoodieCDCFileSplit] + + override def getFileSplits(): List[HoodieCDCFileSplit] = { + fileSplits + } +} diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala new file mode 100644 index 0000000000000..3046a0ddbf2e5 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi + +import org.apache.hudi.common.model.FileSlice + +import org.apache.spark.sql.catalyst.InternalRow + +/** + * Implementation of [[HoodiePartitionFileSliceMapping]] shared by all Spark 4.x + * versions, mixed into the version-specific partition values classes. + */ +trait Spark4HoodiePartitionFileSliceMapping extends HoodiePartitionFileSliceMapping { + + def values: InternalRow + + protected def slices: Map[String, FileSlice] + + override def getSlice(fileId: String): Option[FileSlice] = { + slices.get(fileId) + } + + override def getPartitionValues: InternalRow = values +} diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala new file mode 100644 index 0000000000000..a729c6a0574a8 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.unsafe.types.VariantVal + +/** + * Base class for Spark 4.x HoodiePartitionValues implementations. + * Adds the delegation logic available in all Spark 4.x versions on top of + * [[BaseHoodiePartitionValues]]. Version-specific subclasses only implement + * `copy()` and the getters introduced by a newer Spark version. + */ +abstract class Spark4HoodiePartitionValues(values: InternalRow) + extends BaseHoodiePartitionValues(values) { + + override def getVariant(ordinal: Int): VariantVal = { + values.getVariant(ordinal) + } +} diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/client/model/Spark4HoodieInternalRow.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/client/model/Spark4HoodieInternalRow.scala new file mode 100644 index 0000000000000..dd3b7cd0e3bc6 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/client/model/Spark4HoodieInternalRow.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.model + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.unsafe.types.{UTF8String, VariantVal} + +/** + * Base class for Spark 4.x HoodieInternalRow implementations. + * Adds the delegation logic available in all Spark 4.x versions on top of + * [[HoodieInternalRow]]. Version-specific subclasses only implement + * [[newInternalRow]] and the getters introduced by a newer Spark version. + */ +abstract class Spark4HoodieInternalRow( + metaFields: Array[UTF8String], + sourceRow: InternalRow, + sourceContainsMetaFields: Boolean) + extends HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) { + + override def getVariant(ordinal: Int): VariantVal = { + ruleOutMetaFieldsAccess(ordinal, classOf[VariantVal]) + sourceRow.getVariant(rebaseOrdinal(ordinal)) + } + + override def copy(): InternalRow = { + val copyMetaFields = metaFields.map(f => if (f != null) f.copy() else null) + newInternalRow( + copyMetaFields, + if (sourceRow == null) null else sourceRow.copy(), + sourceContainsMetaFields) + } + + /** + * Creates a new instance of the version-specific row type, used by [[copy]]. + */ + protected def newInternalRow(metaFields: Array[UTF8String], + sourceRow: InternalRow, + sourceContainsMetaFields: Boolean): Spark4HoodieInternalRow +} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionCDCFileGroupMapping.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionCDCFileGroupMapping.scala index 58eaac6324632..28fcfaa080b2b 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionCDCFileGroupMapping.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionCDCFileGroupMapping.scala @@ -24,11 +24,6 @@ import org.apache.hudi.common.table.cdc.HoodieCDCFileSplit import org.apache.spark.sql.catalyst.InternalRow class Spark40HoodiePartitionCDCFileGroupMapping(partitionValues: InternalRow, - fileSplits: List[HoodieCDCFileSplit]) - extends Spark40HoodiePartitionValues(partitionValues) - with HoodiePartitionCDCFileGroupMapping { - - override def getFileSplits(): List[HoodieCDCFileSplit] = { - fileSplits - } -} + protected val fileSplits: List[HoodieCDCFileSplit]) + extends Spark40HoodiePartitionValues(partitionValues) + with Spark4HoodiePartitionCDCFileGroupMapping diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionFileSliceMapping.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionFileSliceMapping.scala index 0f769f5bd7ea2..7de6dbf71f95c 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionFileSliceMapping.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionFileSliceMapping.scala @@ -24,13 +24,6 @@ import org.apache.hudi.common.model.FileSlice import org.apache.spark.sql.catalyst.InternalRow class Spark40HoodiePartitionFileSliceMapping(values: InternalRow, - slices: Map[String, FileSlice]) + protected val slices: Map[String, FileSlice]) extends Spark40HoodiePartitionValues(values) - with HoodiePartitionFileSliceMapping { - - override def getSlice(fileId: String): Option[FileSlice] = { - slices.get(fileId) - } - - override def getPartitionValues: InternalRow = values -} + with Spark4HoodiePartitionFileSliceMapping diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionValues.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionValues.scala index db6e3f10341ac..360a6aa8996ea 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionValues.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionValues.scala @@ -20,92 +20,11 @@ package org.apache.hudi import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.types.{DataType, Decimal} -import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String, VariantVal} -case class Spark40HoodiePartitionValues(values: InternalRow) extends HoodiePartitionValues { - override def numFields: Int = { - values.numFields - } - - override def setNullAt(i: Int): Unit = { - values.setNullAt(i) - } - - override def update(i: Int, value: Any): Unit = { - values.update(i, value) - } +case class Spark40HoodiePartitionValues(override val values: InternalRow) + extends Spark4HoodiePartitionValues(values) { override def copy(): InternalRow = { Spark40HoodiePartitionValues(values.copy()) } - - override def isNullAt(ordinal: Int): Boolean = { - values.isNullAt(ordinal) - } - - override def getBoolean(ordinal: Int): Boolean = { - values.getBoolean(ordinal) - } - - override def getByte(ordinal: Int): Byte = { - values.getByte(ordinal) - } - - override def getShort(ordinal: Int): Short = { - values.getShort(ordinal) - } - - override def getInt(ordinal: Int): Int = { - values.getInt(ordinal) - } - - override def getLong(ordinal: Int): Long = { - values.getLong(ordinal) - } - - override def getFloat(ordinal: Int): Float = { - values.getFloat(ordinal) - } - - override def getDouble(ordinal: Int): Double = { - values.getDouble(ordinal) - } - - override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = { - values.getDecimal(ordinal, precision, scale) - } - - override def getUTF8String(ordinal: Int): UTF8String = { - values.getUTF8String(ordinal) - } - - override def getBinary(ordinal: Int): Array[Byte] = { - values.getBinary(ordinal) - } - - override def getInterval(ordinal: Int): CalendarInterval = { - values.getInterval(ordinal) - } - - override def getVariant(ordinal: Int): VariantVal = { - values.getVariant(ordinal) - } - - override def getStruct(ordinal: Int, numFields: Int): InternalRow = { - values.getStruct(ordinal, numFields) - } - - override def getArray(ordinal: Int): ArrayData = { - values.getArray(ordinal) - } - - override def getMap(ordinal: Int): MapData = { - values.getMap(ordinal) - } - - override def get(ordinal: Int, dataType: DataType): AnyRef = { - values.get(ordinal, dataType) - } } diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/client/model/Spark40HoodieInternalRow.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/client/model/Spark40HoodieInternalRow.scala index 1da628d03a511..6b0ce70edd456 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/client/model/Spark40HoodieInternalRow.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/client/model/Spark40HoodieInternalRow.scala @@ -19,24 +19,17 @@ package org.apache.hudi.client.model import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.unsafe.types.{UTF8String, VariantVal} +import org.apache.spark.unsafe.types.UTF8String class Spark40HoodieInternalRow( - metaFields: Array[UTF8String], - sourceRow: InternalRow, - sourceContainsMetaFields: Boolean) - extends HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) { + metaFields: Array[UTF8String], + sourceRow: InternalRow, + sourceContainsMetaFields: Boolean) + extends Spark4HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) { - override def getVariant(ordinal: Int): VariantVal = { - ruleOutMetaFieldsAccess(ordinal, classOf[VariantVal]) - sourceRow.getVariant(rebaseOrdinal(ordinal)) - } - - override def copy(): InternalRow = { - val copyMetaFields = metaFields.map(f => if (f != null) f.copy() else null) - new Spark40HoodieInternalRow( - copyMetaFields, - if (sourceRow == null) null else sourceRow.copy(), - sourceContainsMetaFields) + override protected def newInternalRow(metaFields: Array[UTF8String], + sourceRow: InternalRow, + sourceContainsMetaFields: Boolean): Spark4HoodieInternalRow = { + new Spark40HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) } } diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala index 005c17eb4128f..96f53d886ba50 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala @@ -24,11 +24,6 @@ import org.apache.hudi.common.table.cdc.HoodieCDCFileSplit import org.apache.spark.sql.catalyst.InternalRow class Spark41HoodiePartitionCDCFileGroupMapping(partitionValues: InternalRow, - fileSplits: List[HoodieCDCFileSplit]) + protected val fileSplits: List[HoodieCDCFileSplit]) extends Spark41HoodiePartitionValues(partitionValues) - with HoodiePartitionCDCFileGroupMapping { - - override def getFileSplits(): List[HoodieCDCFileSplit] = { - fileSplits - } -} + with Spark4HoodiePartitionCDCFileGroupMapping diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala index 07e199e4a9b1e..fbd66d8fc8a7c 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala @@ -24,13 +24,6 @@ import org.apache.hudi.common.model.FileSlice import org.apache.spark.sql.catalyst.InternalRow class Spark41HoodiePartitionFileSliceMapping(values: InternalRow, - slices: Map[String, FileSlice]) + protected val slices: Map[String, FileSlice]) extends Spark41HoodiePartitionValues(values) - with HoodiePartitionFileSliceMapping { - - override def getSlice(fileId: String): Option[FileSlice] = { - slices.get(fileId) - } - - override def getPartitionValues: InternalRow = values -} + with Spark4HoodiePartitionFileSliceMapping diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala index 7d2c71717d573..3964352f3cd85 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala @@ -20,71 +20,15 @@ package org.apache.hudi import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.types.{DataType, Decimal} -import org.apache.spark.unsafe.types.{CalendarInterval, GeographyVal, GeometryVal, UTF8String, VariantVal} +import org.apache.spark.unsafe.types.{GeographyVal, GeometryVal} -case class Spark41HoodiePartitionValues(values: InternalRow) extends HoodiePartitionValues { - override def numFields: Int = { - values.numFields - } - - override def setNullAt(i: Int): Unit = { - values.setNullAt(i) - } - - override def update(i: Int, value: Any): Unit = { - values.update(i, value) - } +case class Spark41HoodiePartitionValues(override val values: InternalRow) + extends Spark4HoodiePartitionValues(values) { override def copy(): InternalRow = { Spark41HoodiePartitionValues(values.copy()) } - override def isNullAt(ordinal: Int): Boolean = { - values.isNullAt(ordinal) - } - - override def getBoolean(ordinal: Int): Boolean = { - values.getBoolean(ordinal) - } - - override def getByte(ordinal: Int): Byte = { - values.getByte(ordinal) - } - - override def getShort(ordinal: Int): Short = { - values.getShort(ordinal) - } - - override def getInt(ordinal: Int): Int = { - values.getInt(ordinal) - } - - override def getLong(ordinal: Int): Long = { - values.getLong(ordinal) - } - - override def getFloat(ordinal: Int): Float = { - values.getFloat(ordinal) - } - - override def getDouble(ordinal: Int): Double = { - values.getDouble(ordinal) - } - - override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = { - values.getDecimal(ordinal, precision, scale) - } - - override def getUTF8String(ordinal: Int): UTF8String = { - values.getUTF8String(ordinal) - } - - override def getBinary(ordinal: Int): Array[Byte] = { - values.getBinary(ordinal) - } - override def getGeography(ordinal: Int): GeographyVal = { values.getGeography(ordinal) } @@ -92,28 +36,4 @@ case class Spark41HoodiePartitionValues(values: InternalRow) extends HoodieParti override def getGeometry(ordinal: Int): GeometryVal = { values.getGeometry(ordinal) } - - override def getInterval(ordinal: Int): CalendarInterval = { - values.getInterval(ordinal) - } - - override def getVariant(ordinal: Int): VariantVal = { - values.getVariant(ordinal) - } - - override def getStruct(ordinal: Int, numFields: Int): InternalRow = { - values.getStruct(ordinal, numFields) - } - - override def getArray(ordinal: Int): ArrayData = { - values.getArray(ordinal) - } - - override def getMap(ordinal: Int): MapData = { - values.getMap(ordinal) - } - - override def get(ordinal: Int, dataType: DataType): AnyRef = { - values.get(ordinal, dataType) - } } diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala index a56368eee1e4a..308424396a752 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala @@ -19,18 +19,13 @@ package org.apache.hudi.client.model import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.unsafe.types.{GeographyVal, GeometryVal, UTF8String, VariantVal} +import org.apache.spark.unsafe.types.{GeographyVal, GeometryVal, UTF8String} class Spark41HoodieInternalRow( metaFields: Array[UTF8String], sourceRow: InternalRow, sourceContainsMetaFields: Boolean) - extends HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) { - - override def getVariant(ordinal: Int): VariantVal = { - ruleOutMetaFieldsAccess(ordinal, classOf[VariantVal]) - sourceRow.getVariant(rebaseOrdinal(ordinal)) - } + extends Spark4HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) { override def getGeography(ordinal: Int): GeographyVal = { ruleOutMetaFieldsAccess(ordinal, classOf[GeographyVal]) @@ -42,11 +37,9 @@ class Spark41HoodieInternalRow( sourceRow.getGeometry(rebaseOrdinal(ordinal)) } - override def copy(): InternalRow = { - val copyMetaFields = metaFields.map(f => if (f != null) f.copy() else null) - new Spark41HoodieInternalRow( - copyMetaFields, - if (sourceRow == null) null else sourceRow.copy(), - sourceContainsMetaFields) + override protected def newInternalRow(metaFields: Array[UTF8String], + sourceRow: InternalRow, + sourceContainsMetaFields: Boolean): Spark4HoodieInternalRow = { + new Spark41HoodieInternalRow(metaFields, sourceRow, sourceContainsMetaFields) } } From 0ba508d0573776610c0f033054b2075bc01bc2c7 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 2 Jul 2026 22:49:33 -0700 Subject: [PATCH 120/255] refactor(spark): dedup catalyst utils and analysis rules across Spark version modules (#19149) * refactor(spark): pull shared HoodieCatalystExpressionUtils bodies into a common base class Introduce BaseHoodieCatalystExpressionUtils in hudi-spark-common carrying the method bodies that are byte-identical across Spark 3.3-4.2 (normalizeExprs, extractPredicatesWithinOutputSet, canUpCast, and the order-preserving transformation matcher). The ParseToDate/ParseToTimestamp patterns, whose case-class shapes differ across Spark versions, are kept per version behind the unapplyOrderPreservingDateParsing hook. HoodieSpark4CatalystExpressionUtils in hudi-spark4-common now carries the bodies uniform within the Spark 4.x family (getEncoder, matchCast, unapplyCastExpression, date-parsing hook), turning the Spark 4.x objects into empty declarations. The Spark 3.x objects keep only genuine per-version deltas (RowEncoder vs ExpressionEncoder, AnsiCast, EvalMode). The empty HoodieSpark3CatalystExpressionUtils shim is removed; adapter-facing object names are unchanged. * refactor(spark): implement shared HoodieCatalystPlansUtils bodies in base and family classes Move the method bodies that are byte-identical across all six Spark version modules into the BaseHoodieCatalystPlanUtils trait in hudi-spark-common (unapplyResolvedTable, projectOverSchema, isRepairTable, getRepairTableChildren, the four index-plan matchers, unapplyInsertIntoStatement, and createProjectForByNameQuery, replacing the dead 'None' default that every version overrode identically). Add family-level bases for bodies identical only within a Spark major version: HoodieSpark3CatalystPlanUtils in hudi-spark3-common (unapplyUpdateAction with the 2-field UpdateAction, extractJsonFromSerializedOffset with the pre-4.1 package) and HoodieSpark4CatalystPlanUtils in hudi-spark4-common (7-field MergeIntoTable, 5-field LogicalRelation scan, error-class based failAnalysisForMIT and failTableNotFound). Per-version objects keep only genuine Spark API deltas: pattern arities of MergeIntoTable/ScanOperation/LogicalRelation/UpdateAction, the SerializedOffset package move in Spark 4.1, pre-error-class failure messages on 3.3, the _LEGACY_ERROR_TEMP_2309 error class on 3.4, and the Spark 3.4-only default-columns workaround in unapplyInsertIntoStatement. Adapter-facing object names are unchanged. * refactor(spark): pull shared HoodieSchemaUtils bodies into family base classes Nothing in this family is identical across all six Spark versions, so the bodies are shared at the Spark-major-version level: HoodieSpark3SchemaUtils in hudi-spark3-common carries getSchema (pre-4.0 JdbcUtils.getSchema signature without the Connection argument), and HoodieSpark4SchemaUtils in hudi-spark4-common carries all three methods, which are uniform across 4.0-4.2, turning those objects into empty declarations. Spark 3.x objects keep the per-version deltas: the 3-arg SchemaUtils.checkColumnNameDuplication on 3.3 (colType parameter removed in Spark 3.4) and StructType#toAttributes on 3.3/3.4 (moved to DataTypeUtils in Spark 3.5, SPARK-44353). * refactor(spark): share ResolveColumnsForInsertInto preprocessing across Spark 4.x The three-level preprocess logic (partition-spec normalization, user-specified column projection, output-column resolution with default-value support, and the arity-mismatch error rewrite) is byte-identical across Spark 4.0-4.2, so it moves into the abstract HoodieSpark4ResolveColumnsForInsertInto base class in hudi-spark4-common. The per-version case classes keep only apply(), whose InsertIntoStatement pattern arity differs in Spark 4.2 (two fields added). The DataSourceV2ToV1Fallback rules stay fully per-version: every adjacent pair differs in InsertIntoStatement and/or DataSourceV2Relation case-class shapes, so no clean seam exists within the family. Spark 3.5 keeps its full copy of ResolveColumnsForInsertInto since hudi-spark3-common also compiles against Spark 3.3/3.4, which lack the required APIs. Rule class names are unchanged (HoodieAnalysis instantiates them reflectively by name). (cherry picked from commit b933fab87559614c3548ee6909dfc1999b3b6b80) --- .../BaseHoodieCatalystExpressionUtils.scala | 114 ++++++++++++++ .../sql/BaseHoodieCatalystPlanUtils.scala | 82 +++++++++- .../HoodieSpark3CatalystExpressionUtils.scala | 37 ----- .../sql/HoodieSpark3CatalystPlanUtils.scala | 44 ++++++ .../spark/sql/HoodieSpark3SchemaUtils.scala | 39 +++++ ...HoodieSpark33CatalystExpressionUtils.scala | 81 +--------- .../sql/HoodieSpark33CatalystPlanUtils.scala | 103 +------------ .../spark/sql/HoodieSpark33SchemaUtils.scala | 14 +- ...HoodieSpark34CatalystExpressionUtils.scala | 81 +--------- .../sql/HoodieSpark34CatalystPlanUtils.scala | 94 +----------- .../spark/sql/HoodieSpark34SchemaUtils.scala | 14 +- ...HoodieSpark35CatalystExpressionUtils.scala | 81 +--------- .../sql/HoodieSpark35CatalystPlanUtils.scala | 103 +------------ .../spark/sql/HoodieSpark35SchemaUtils.scala | 16 +- .../HoodieSpark4CatalystExpressionUtils.scala | 45 ++++-- .../sql/HoodieSpark4CatalystPlanUtils.scala | 66 ++++++++ .../spark/sql/HoodieSpark4SchemaUtils.scala | 52 +++++++ .../hudi/analysis/HoodieSpark4Analysis.scala | 143 ++++++++++++++++++ ...HoodieSpark40CatalystExpressionUtils.scala | 99 +----------- .../sql/HoodieSpark40CatalystPlanUtils.scala | 123 +-------------- .../spark/sql/HoodieSpark40SchemaUtils.scala | 31 +--- .../hudi/analysis/HoodieSpark40Analysis.scala | 122 +-------------- ...HoodieSpark41CatalystExpressionUtils.scala | 99 +----------- .../sql/HoodieSpark41CatalystPlanUtils.scala | 123 +-------------- .../spark/sql/HoodieSpark41SchemaUtils.scala | 31 +--- .../hudi/analysis/HoodieSpark41Analysis.scala | 121 +-------------- 26 files changed, 627 insertions(+), 1331 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3SchemaUtils.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystPlanUtils.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4SchemaUtils.scala create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark4Analysis.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala new file mode 100644 index 0000000000000..821eebcce2300 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering +import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} +import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.types.DataType + +/** + * Base implementation of [[HoodieCatalystExpressionUtils]] carrying the method bodies that are + * identical across all supported Spark versions. Methods relying on Spark APIs that changed + * across versions are implemented in the per-version `HoodieSparkXXCatalystExpressionUtils` + * objects (or in [[HoodieSpark4CatalystExpressionUtils]] when shared within a Spark major version). + */ +abstract class BaseHoodieCatalystExpressionUtils extends HoodieCatalystExpressionUtils with PredicateHelper { + + override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = { + DataSourceStrategy.normalizeExprs(exprs, attributes) + } + + override def extractPredicatesWithinOutputSet(condition: Expression, + outputSet: AttributeSet): Option[Expression] = { + super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) + } + + override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { + expr match { + case OrderPreservingTransformation(attrRef) => Some(attrRef) + case _ => None + } + } + + def canUpCast(fromType: DataType, toType: DataType): Boolean = + Cast.canUpCast(fromType, toType) + + /** + * Matches order-preserving date/time parsing expressions whose case-class shapes differ across + * Spark versions (currently [[org.apache.spark.sql.catalyst.expressions.ParseToDate]] and + * [[org.apache.spark.sql.catalyst.expressions.ParseToTimestamp]]), returning the source child + * expression that the order-preserving transformation matching should recurse into + */ + protected def unapplyOrderPreservingDateParsing(expr: Expression): Option[Expression] + + private object OrderPreservingTransformation { + def unapply(expr: Expression): Option[AttributeReference] = { + expr match { + // Date/Time Expressions + case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) + case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) + case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) + case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) + case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + + // String Expressions + case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) + // Left API change: Improve RuntimeReplaceable + // https://issues.apache.org/jira/browse/SPARK-38240 + case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + + // Math Expressions + // Binary + case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) + case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) + case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) + case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) + // Unary + case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) + case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) + + // Other + case cast @ Cast(OrderPreservingTransformation(attrRef), _, _, _) + if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) + + // Identity transformation + case attrRef: AttributeReference => Some(attrRef) + // Date/time parsing expressions whose shapes are Spark-version-specific + case _ => unapplyOrderPreservingDateParsing(expr) match { + case Some(child) => unapply(child) + case None => None + } + } + } + } +} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystPlanUtils.scala index 498f3f276dc01..c8ceb713a6a3a 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystPlanUtils.scala @@ -20,14 +20,14 @@ package org.apache.spark.sql import org.apache.hudi.SparkAdapterSupport import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.TableOutputResolver +import org.apache.spark.sql.catalyst.analysis.{ResolvedTable, TableOutputResolver} import org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, Join, JoinHint, LogicalPlan} +import org.apache.spark.sql.catalyst.plans.logical.{CreateIndex, DropIndex, HoodieShowIndexes, InsertIntoStatement, Join, JoinHint, LogicalPlan, RefreshIndex} import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} import org.apache.spark.sql.execution.{ExtendedMode, SimpleMode} -import org.apache.spark.sql.execution.command.{CreateTableLikeCommand, ExplainCommand} +import org.apache.spark.sql.execution.command.{CreateTableLikeCommand, ExplainCommand, RepairTableCommand} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType @@ -37,12 +37,17 @@ trait BaseHoodieCatalystPlanUtils extends HoodieCatalystPlansUtils { /** * Instantiates [[ProjectionOverSchema]] utility */ - def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema + def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = + ProjectionOverSchema(schema, output) /** * Un-applies [[ResolvedTable]] that had its signature changed in Spark 3.2 */ - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] + def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = + plan match { + case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) + case _ => None + } def resolveOutputColumns(tableName: String, expected: Seq[Attribute], @@ -77,7 +82,72 @@ trait BaseHoodieCatalystPlanUtils extends HoodieCatalystPlansUtils { a.sameOutput(b) } - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = None + override def isRepairTable(plan: LogicalPlan): Boolean = { + plan.isInstanceOf[RepairTableCommand] + } + + override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { + plan match { + case rtc: RepairTableCommand => + Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) + case _ => + None + } + } + + override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { + plan match { + case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => + Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) + case _ => + None + } + } + + override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { + plan match { + case ci@DropIndex(table, indexName, ignoreIfNotExists) => + Some((table, indexName, ignoreIfNotExists)) + case _ => + None + } + } + + override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { + plan match { + case ci@HoodieShowIndexes(table, output) => + Some((table, output)) + case _ => + None + } + } + + override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { + plan match { + case ci@RefreshIndex(table, indexName) => + Some((table, indexName)) + case _ => + None + } + } + + override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { + plan match { + case insert: InsertIntoStatement => + Some((insert.table, insert.userSpecifiedCols, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists)) + case _ => + None + } + } + + override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { + plan match { + case insert: InsertIntoStatement => + Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) + case _ => + None + } + } } object BaseHoodieCatalystPlanUtils extends SparkAdapterSupport { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala deleted file mode 100644 index cf63383ef35a7..0000000000000 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression} -abstract class HoodieSpark3CatalystExpressionUtils extends HoodieCatalystExpressionUtils { - - /** - * The attribute name may differ from the one in the schema if the query analyzer - * is case insensitive. We should change attribute names to match the ones in the schema, - * so we do not need to worry about case sensitivity anymore - */ - def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] - - /** - * Returns a filter that its reference is a subset of `outputSet` and it contains the maximum - * constraints from `condition`. This is used for predicate push-down - * When there is no such filter, `None` is returned. - */ - def extractPredicatesWithinOutputSet(condition: Expression, - outputSet: AttributeSet): Option[Expression] -} diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala new file mode 100644 index 0000000000000..eb22674d9de7d --- /dev/null +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.{Assignment, UpdateAction} +import org.apache.spark.sql.execution.streaming.SerializedOffset + +/** + * Implementation of [[HoodieCatalystPlansUtils]] carrying the method bodies shared by all + * supported Spark 3.x versions + */ +abstract class HoodieSpark3CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { + + override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { + mergeAction match { + case UpdateAction(condition, assignments) => Some((condition, assignments)) + case _ => None + } + } + + override def extractJsonFromSerializedOffset(offset: Any): Option[String] = { + offset match { + case SerializedOffset(json) => Some(json) + case _ => None + } + } +} diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3SchemaUtils.scala new file mode 100644 index 0000000000000..6adad02c1d918 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3SchemaUtils.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils +import org.apache.spark.sql.jdbc.JdbcDialect +import org.apache.spark.sql.types.StructType + +import java.sql.{Connection, ResultSet} + +/** + * Utils on schema shared by all supported Spark 3.x versions. + */ +abstract class HoodieSpark3SchemaUtils extends HoodieSchemaUtils { + override def getSchema(conn: Connection, + resultSet: ResultSet, + dialect: JdbcDialect, + alwaysNullable: Boolean = false, + isTimestampNTZ: Boolean = false): StructType = { + JdbcUtils.getSchema(resultSet, dialect, alwaysNullable) + } +} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala index e083313742408..8ebfce7785fc8 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala @@ -17,26 +17,16 @@ package org.apache.spark.sql -import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Add, AnsiCast, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.catalyst.expressions.{AnsiCast, Cast, Expression, ParseToDate, ParseToTimestamp} import org.apache.spark.sql.types.{DataType, StructType} -object HoodieSpark33CatalystExpressionUtils extends HoodieSpark3CatalystExpressionUtils with PredicateHelper { +object HoodieSpark33CatalystExpressionUtils extends BaseHoodieCatalystExpressionUtils { override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { RowEncoder.apply(schema).resolveAndBind() } - override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = - DataSourceStrategy.normalizeExprs(exprs, attributes) - - override def extractPredicatesWithinOutputSet(condition: Expression, - outputSet: AttributeSet): Option[Expression] = { - super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) - } - override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = expr match { case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) @@ -44,16 +34,6 @@ object HoodieSpark33CatalystExpressionUtils extends HoodieSpark3CatalystExpressi case _ => None } - override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { - expr match { - case OrderPreservingTransformation(attrRef) => Some(attrRef) - case _ => None - } - } - - def canUpCast(fromType: DataType, toType: DataType): Boolean = - Cast.canUpCast(fromType, toType) - override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = expr match { case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => @@ -63,57 +43,10 @@ object HoodieSpark33CatalystExpressionUtils extends HoodieSpark3CatalystExpressi case _ => None } - private object OrderPreservingTransformation { - def unapply(expr: Expression): Option[AttributeReference] = { - expr match { - // Date/Time Expressions - case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ParseToDate(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // String Expressions - case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Left API change: Improve RuntimeReplaceable - // https://issues.apache.org/jira/browse/SPARK-38240 - case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Math Expressions - // Binary - case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Unary - case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Other - case cast @ Cast(OrderPreservingTransformation(attrRef), _, _, _) - if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) - - // Identity transformation - case attrRef: AttributeReference => Some(attrRef) - // No match - case _ => None - } + override protected def unapplyOrderPreservingDateParsing(expr: Expression): Option[Expression] = + expr match { + case ParseToDate(child, _, _) => Some(child) + case ParseToTimestamp(child, _, _, _) => Some(child) + case _ => None } - } } diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala index 114bc958c1722..1a81ab0468665 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala @@ -18,25 +18,14 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} +import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.RepairTableCommand +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MergeIntoTable} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} -import org.apache.spark.sql.execution.streaming.SerializedOffset -import org.apache.spark.sql.types.StructType -object HoodieSpark33CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { - - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = - plan match { - case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) - case _ => None - } +object HoodieSpark33CatalystPlanUtils extends HoodieSpark3CatalystPlanUtils { override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { plan match { @@ -57,22 +46,6 @@ object HoodieSpark33CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { } } - override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = - ProjectionOverSchema(schema, output) - - override def isRepairTable(plan: LogicalPlan): Boolean = { - plan.isInstanceOf[RepairTableCommand] - } - - override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { - plan match { - case rtc: RepairTableCommand => - Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) - case _ => - None - } - } - override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { a.failAnalysis(s"cannot resolve ${a.sql} in MERGE command given columns [$cols]") } @@ -80,72 +53,4 @@ object HoodieSpark33CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { override def failTableNotFound(tableName: String): Unit = { throw new AnalysisException(s"Table or view not found: $tableName") } - - override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { - plan match { - case ci @ CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => - Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) - case _ => - None - } - } - - override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { - plan match { - case ci @ DropIndex(table, indexName, ignoreIfNotExists) => - Some((table, indexName, ignoreIfNotExists)) - case _ => - None - } - } - - override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { - plan match { - case ci @ HoodieShowIndexes(table, output) => - Some((table, output)) - case _ => - None - } - } - - override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { - plan match { - case ci @ RefreshIndex(table, indexName) => - Some((table, indexName)) - case _ => - None - } - } - - override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { - plan match { - case insert: InsertIntoStatement => - Some((insert.table, insert.userSpecifiedCols, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists)) - case _ => - None - } - } - - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { - plan match { - case insert: InsertIntoStatement => - Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) - case _ => - None - } - } - - override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { - mergeAction match { - case UpdateAction(condition, assignments) => Some((condition, assignments)) - case _ => None - } - } - - override def extractJsonFromSerializedOffset(offset: Any): Option[String] = { - offset match { - case SerializedOffset(json) => Some(json) - case _ => None - } - } } diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala index 41748ac155535..7b968bde80648 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala @@ -20,17 +20,13 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils -import org.apache.spark.sql.jdbc.JdbcDialect import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.SchemaUtils -import java.sql.{Connection, ResultSet} - /** * Utils on schema for Spark 3.3. */ -object HoodieSpark33SchemaUtils extends HoodieSchemaUtils { +object HoodieSpark33SchemaUtils extends HoodieSpark3SchemaUtils { override def checkColumnNameDuplication(columnNames: Seq[String], colType: String, caseSensitiveAnalysis: Boolean): Unit = { @@ -40,12 +36,4 @@ object HoodieSpark33SchemaUtils extends HoodieSchemaUtils { override def toAttributes(struct: StructType): Seq[Attribute] = { struct.toAttributes } - - override def getSchema(conn: Connection, - resultSet: ResultSet, - dialect: JdbcDialect, - alwaysNullable: Boolean = false, - isTimestampNTZ: Boolean = false): StructType = { - JdbcUtils.getSchema(resultSet, dialect, alwaysNullable) - } } diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala index 03c7d0412f8a2..f5767eb82a76b 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala @@ -17,26 +17,16 @@ package org.apache.spark.sql -import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, EvalMode, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.catalyst.expressions.{Cast, EvalMode, Expression, ParseToDate, ParseToTimestamp} import org.apache.spark.sql.types.{DataType, StructType} -object HoodieSpark34CatalystExpressionUtils extends HoodieSpark3CatalystExpressionUtils with PredicateHelper { +object HoodieSpark34CatalystExpressionUtils extends BaseHoodieCatalystExpressionUtils { override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { RowEncoder.apply(schema).resolveAndBind() } - override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = { - DataSourceStrategy.normalizeExprs(exprs, attributes) - } - - override def extractPredicatesWithinOutputSet(condition: Expression, outputSet: AttributeSet): Option[Expression] = { - super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) - } - override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = { expr match { case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) @@ -44,16 +34,6 @@ object HoodieSpark34CatalystExpressionUtils extends HoodieSpark3CatalystExpressi } } - override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { - expr match { - case OrderPreservingTransformation(attrRef) => Some(attrRef) - case _ => None - } - } - - def canUpCast(fromType: DataType, toType: DataType): Boolean = - Cast.canUpCast(fromType, toType) - override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = expr match { case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => @@ -61,57 +41,10 @@ object HoodieSpark34CatalystExpressionUtils extends HoodieSpark3CatalystExpressi case _ => None } - private object OrderPreservingTransformation { - def unapply(expr: Expression): Option[AttributeReference] = { - expr match { - // Date/Time Expressions - case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ParseToDate(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _, _, _) => Some(attrRef) - case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // String Expressions - case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Left API change: Improve RuntimeReplaceable - // https://issues.apache.org/jira/browse/SPARK-38240 - case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Math Expressions - // Binary - case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Unary - case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Other - case cast @ Cast(OrderPreservingTransformation(attrRef), _, _, _) - if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) - - // Identity transformation - case attrRef: AttributeReference => Some(attrRef) - // No match - case _ => None - } + override protected def unapplyOrderPreservingDateParsing(expr: Expression): Option[Expression] = + expr match { + case ParseToDate(child, _, _) => Some(child) + case ParseToTimestamp(child, _, _, _, _) => Some(child) + case _ => None } - } } diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala index 78c918f325a13..97b922ba4cb61 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala @@ -18,26 +18,15 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} +import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.RepairTableCommand +import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, MergeIntoTable} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} -import org.apache.spark.sql.execution.streaming.SerializedOffset import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType -object HoodieSpark34CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { - - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = - plan match { - case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) - case _ => None - } +object HoodieSpark34CatalystPlanUtils extends HoodieSpark3CatalystPlanUtils { override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { plan match { @@ -58,22 +47,6 @@ object HoodieSpark34CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { } } - override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = - ProjectionOverSchema(schema, output) - - override def isRepairTable(plan: LogicalPlan): Boolean = { - plan.isInstanceOf[RepairTableCommand] - } - - override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { - plan match { - case rtc: RepairTableCommand => - Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) - case _ => - None - } - } - override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { a.failAnalysis( errorClass = "_LEGACY_ERROR_TEMP_2309", @@ -88,42 +61,6 @@ object HoodieSpark34CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { messageParameters = Map("relationName" -> s"`$tableName`")) } - override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { - plan match { - case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => - Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) - case _ => - None - } - } - - override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { - plan match { - case ci@DropIndex(table, indexName, ignoreIfNotExists) => - Some((table, indexName, ignoreIfNotExists)) - case _ => - None - } - } - - override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { - plan match { - case ci@HoodieShowIndexes(table, output) => - Some((table, output)) - case _ => - None - } - } - - override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { - plan match { - case ci@RefreshIndex(table, indexName) => - Some((table, indexName)) - case _ => - None - } - } - override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { plan match { case insert: InsertIntoStatement => @@ -145,27 +82,4 @@ object HoodieSpark34CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { None } } - - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { - plan match { - case insert: InsertIntoStatement => - Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) - case _ => - None - } - } - - override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { - mergeAction match { - case UpdateAction(condition, assignments) => Some((condition, assignments)) - case _ => None - } - } - - override def extractJsonFromSerializedOffset(offset: Any): Option[String] = { - offset match { - case SerializedOffset(json) => Some(json) - case _ => None - } - } } diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala index c5fb1beded17d..c272208f7b8a4 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala @@ -20,17 +20,13 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils -import org.apache.spark.sql.jdbc.JdbcDialect import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.SchemaUtils -import java.sql.{Connection, ResultSet} - /** * Utils on schema for Spark 3.4. */ -object HoodieSpark34SchemaUtils extends HoodieSchemaUtils { +object HoodieSpark34SchemaUtils extends HoodieSpark3SchemaUtils { override def checkColumnNameDuplication(columnNames: Seq[String], colType: String, caseSensitiveAnalysis: Boolean): Unit = { @@ -40,12 +36,4 @@ object HoodieSpark34SchemaUtils extends HoodieSchemaUtils { override def toAttributes(struct: StructType): Seq[Attribute] = { struct.toAttributes } - - override def getSchema(conn: Connection, - resultSet: ResultSet, - dialect: JdbcDialect, - alwaysNullable: Boolean = false, - isTimestampNTZ: Boolean = false): StructType = { - JdbcUtils.getSchema(resultSet, dialect, alwaysNullable) - } } diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala index 6f1456972a52a..8039e6b968155 100644 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala @@ -17,26 +17,16 @@ package org.apache.spark.sql -import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, EvalMode, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.catalyst.expressions.{Cast, EvalMode, Expression, ParseToDate, ParseToTimestamp} import org.apache.spark.sql.types.{DataType, StructType} -object HoodieSpark35CatalystExpressionUtils extends HoodieSpark3CatalystExpressionUtils with PredicateHelper { +object HoodieSpark35CatalystExpressionUtils extends BaseHoodieCatalystExpressionUtils { override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { ExpressionEncoder.apply(schema).resolveAndBind() } - override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = { - DataSourceStrategy.normalizeExprs(exprs, attributes) - } - - override def extractPredicatesWithinOutputSet(condition: Expression, outputSet: AttributeSet): Option[Expression] = { - super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) - } - override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = { expr match { case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) @@ -44,16 +34,6 @@ object HoodieSpark35CatalystExpressionUtils extends HoodieSpark3CatalystExpressi } } - override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { - expr match { - case OrderPreservingTransformation(attrRef) => Some(attrRef) - case _ => None - } - } - - def canUpCast(fromType: DataType, toType: DataType): Boolean = - Cast.canUpCast(fromType, toType) - override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = expr match { case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => @@ -61,57 +41,10 @@ object HoodieSpark35CatalystExpressionUtils extends HoodieSpark3CatalystExpressi case _ => None } - private object OrderPreservingTransformation { - def unapply(expr: Expression): Option[AttributeReference] = { - expr match { - // Date/Time Expressions - case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ParseToDate(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _, _, _) => Some(attrRef) - case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // String Expressions - case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Left API change: Improve RuntimeReplaceable - // https://issues.apache.org/jira/browse/SPARK-38240 - case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Math Expressions - // Binary - case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Unary - case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Other - case cast @ Cast(OrderPreservingTransformation(attrRef), _, _, _) - if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) - - // Identity transformation - case attrRef: AttributeReference => Some(attrRef) - // No match - case _ => None - } + override protected def unapplyOrderPreservingDateParsing(expr: Expression): Option[Expression] = + expr match { + case ParseToDate(child, _, _, _) => Some(child) + case ParseToTimestamp(child, _, _, _, _) => Some(child) + case _ => None } - } } diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala index db480f4fb42e0..534d8696b2c41 100644 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala @@ -18,25 +18,14 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} +import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.RepairTableCommand +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MergeIntoTable} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} -import org.apache.spark.sql.execution.streaming.SerializedOffset -import org.apache.spark.sql.types.StructType -object HoodieSpark35CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { - - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = - plan match { - case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) - case _ => None - } +object HoodieSpark35CatalystPlanUtils extends HoodieSpark3CatalystPlanUtils { override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { plan match { @@ -57,22 +46,6 @@ object HoodieSpark35CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { } } - override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = - ProjectionOverSchema(schema, output) - - override def isRepairTable(plan: LogicalPlan): Boolean = { - plan.isInstanceOf[RepairTableCommand] - } - - override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { - plan match { - case rtc: RepairTableCommand => - Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) - case _ => - None - } - } - override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { a.failAnalysis( errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", @@ -86,72 +59,4 @@ object HoodieSpark35CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { errorClass = "TABLE_OR_VIEW_NOT_FOUND", messageParameters = Map("relationName" -> s"`$tableName`")) } - - override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { - plan match { - case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => - Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) - case _ => - None - } - } - - override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { - plan match { - case ci@DropIndex(table, indexName, ignoreIfNotExists) => - Some((table, indexName, ignoreIfNotExists)) - case _ => - None - } - } - - override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { - plan match { - case ci@HoodieShowIndexes(table, output) => - Some((table, output)) - case _ => - None - } - } - - override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { - plan match { - case ci@RefreshIndex(table, indexName) => - Some((table, indexName)) - case _ => - None - } - } - - override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { - plan match { - case insert: InsertIntoStatement => - Some((insert.table, insert.userSpecifiedCols, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists)) - case _ => - None - } - } - - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { - plan match { - case insert: InsertIntoStatement => - Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) - case _ => - None - } - } - - override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { - mergeAction match { - case UpdateAction(condition, assignments) => Some((condition, assignments)) - case _ => None - } - } - - override def extractJsonFromSerializedOffset(offset: Any): Option[String] = { - offset match { - case SerializedOffset(json) => Some(json) - case _ => None - } - } } diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala index 708abd69446bb..5193450599605 100644 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala @@ -21,17 +21,13 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils -import org.apache.spark.sql.jdbc.JdbcDialect import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.SchemaUtils -import java.sql.{Connection, ResultSet} - /** - * Utils on schema for Spark 3.4+. + * Utils on schema for Spark 3.5. */ -object HoodieSpark35SchemaUtils extends HoodieSchemaUtils { +object HoodieSpark35SchemaUtils extends HoodieSpark3SchemaUtils { override def checkColumnNameDuplication(columnNames: Seq[String], colType: String, caseSensitiveAnalysis: Boolean): Unit = { @@ -41,12 +37,4 @@ object HoodieSpark35SchemaUtils extends HoodieSchemaUtils { override def toAttributes(struct: StructType): Seq[Attribute] = { DataTypeUtils.toAttributes(struct) } - - override def getSchema(conn: Connection, - resultSet: ResultSet, - dialect: JdbcDialect, - alwaysNullable: Boolean = false, - isTimestampNTZ: Boolean = false): StructType = { - JdbcUtils.getSchema(resultSet, dialect, alwaysNullable) - } } diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystExpressionUtils.scala index 3da62104db734..22272b2b031c4 100644 --- a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystExpressionUtils.scala @@ -17,22 +17,37 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression} +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.catalyst.expressions.{Cast, EvalMode, Expression, ParseToDate, ParseToTimestamp} +import org.apache.spark.sql.types.{DataType, StructType} -abstract class HoodieSpark4CatalystExpressionUtils extends HoodieCatalystExpressionUtils { +/** + * Implementation of [[HoodieCatalystExpressionUtils]] shared by all supported Spark 4.x versions + */ +abstract class HoodieSpark4CatalystExpressionUtils extends BaseHoodieCatalystExpressionUtils { + + override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { + ExpressionEncoder.apply(schema).resolveAndBind() + } + + override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = { + expr match { + case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) + case _ => None + } + } - /** - * The attribute name may differ from the one in the schema if the query analyzer - * is case insensitive. We should change attribute names to match the ones in the schema, - * so we do not need to worry about case sensitivity anymore - */ - def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] + override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = + expr match { + case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => + Some((castedExpr, dataType, timeZoneId, if (ansiEnabled == EvalMode.ANSI) true else false)) + case _ => None + } - /** - * Returns a filter that its reference is a subset of `outputSet` and it contains the maximum - * constraints from `condition`. This is used for predicate push-down - * When there is no such filter, `None` is returned. - */ - def extractPredicatesWithinOutputSet(condition: Expression, - outputSet: AttributeSet): Option[Expression] + override protected def unapplyOrderPreservingDateParsing(expr: Expression): Option[Expression] = + expr match { + case ParseToDate(child, _, _, _) => Some(child) + case ParseToTimestamp(child, _, _, _, _) => Some(child) + case _ => None + } } diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystPlanUtils.scala new file mode 100644 index 0000000000000..7af2089ae2d23 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4CatalystPlanUtils.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.planning.ScanOperation +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MergeIntoTable} +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} +import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} + +/** + * Implementation of [[HoodieCatalystPlansUtils]] carrying the method bodies shared by all + * supported Spark 4.x versions + */ +abstract class HoodieSpark4CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { + + override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { + plan match { + case MergeIntoTable(targetTable, sourceTable, mergeCondition, _, _, _, _) => + Some((targetTable, sourceTable, mergeCondition)) + case _ => None + } + } + + override def maybeApplyForNewFileFormat(plan: LogicalPlan): LogicalPlan = { + plan match { + case s@ScanOperation(_, _, _, + l@LogicalRelation(fs: HadoopFsRelation, _, _, _, _)) + if fs.fileFormat.isInstanceOf[ParquetFileFormat with HoodieFormatTrait] + && !fs.fileFormat.asInstanceOf[ParquetFileFormat with HoodieFormatTrait].isProjected => + FileFormatUtilsForFileGroupReader.applyNewFileFormatChanges(s, l, fs) + case _ => plan + } + } + + override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { + a.failAnalysis( + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + messageParameters = Map( + "objectName" -> a.sql, + "proposal" -> cols)) + } + + override def failTableNotFound(tableName: String): Unit = { + throw new AnalysisException( + errorClass = "TABLE_OR_VIEW_NOT_FOUND", + messageParameters = Map("relationName" -> s"`$tableName`")) + } +} diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4SchemaUtils.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4SchemaUtils.scala new file mode 100644 index 0000000000000..433394d332d1f --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/HoodieSpark4SchemaUtils.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.types.DataTypeUtils +import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils +import org.apache.spark.sql.jdbc.JdbcDialect +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.SchemaUtils + +import java.sql.{Connection, ResultSet} + +/** + * Utils on schema shared by all supported Spark 4.x versions. + */ +abstract class HoodieSpark4SchemaUtils extends HoodieSchemaUtils { + override def checkColumnNameDuplication(columnNames: Seq[String], + colType: String, + caseSensitiveAnalysis: Boolean): Unit = { + SchemaUtils.checkColumnNameDuplication(columnNames, caseSensitiveAnalysis) + } + + override def toAttributes(struct: StructType): Seq[Attribute] = { + DataTypeUtils.toAttributes(struct) + } + + override def getSchema(conn: Connection, + resultSet: ResultSet, + dialect: JdbcDialect, + alwaysNullable: Boolean = false, + isTimestampNTZ: Boolean = false): StructType = { + JdbcUtils.getSchema(conn, resultSet, dialect, alwaysNullable) + } +} diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark4Analysis.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark4Analysis.scala new file mode 100644 index 0000000000000..8d42660111838 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark4Analysis.scala @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.analysis + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.analysis.{ResolveInsertionBase, TableOutputResolver} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement +import org.apache.spark.sql.errors.DataTypeErrors.toSQLId +import org.apache.spark.sql.errors.QueryCompilationErrors +import org.apache.spark.sql.execution.datasources.PreprocessTableInsertion +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.PartitioningUtils.normalizePartitionSpec + +/** + * In Spark 3.5, the following Resolution rules are removed, + * [[ResolveUserSpecifiedColumns]] and [[ResolveDefaultColumns]] + * (see code changes in [[org.apache.spark.sql.catalyst.analysis.Analyzer]] + * from https://github.com/apache/spark/pull/41262). + * The same logic of resolving the user specified columns and default values, + * which are required for a subset of columns as user specified compared to the table + * schema to work properly, are deferred to [[PreprocessTableInsertion]] for v1 INSERT. + * + * Note that [[HoodieAnalysis]] intercepts the [[InsertIntoStatement]] after Spark's built-in + * Resolution rules are applies, the logic of resolving the user specified columns and default + * values may no longer be applied. To make INSERT with a subset of columns specified by user + * to work, the custom resolution rules `HoodieSpark4XResolveColumnsForInsertInto` extending + * this base class are added to achieve the same, before converting [[InsertIntoStatement]] + * into [[InsertIntoHoodieTableCommand]]. + * + * The implementation is copied and adapted from [[PreprocessTableInsertion]] + * https://github.com/apache/spark/blob/d061aadf25fd258d2d3e7332a489c9c24a2b5530/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L373 + * + * Also note that, the project logic in [[ResolveImplementationsEarly]] for INSERT is still + * needed in the case of INSERT with all columns in a different ordering. + * + * This base class carries the preprocessing logic shared by all supported Spark 4.x versions; + * the per-version subclasses keep only the plan matching that depends on version-specific + * case-class shapes of [[InsertIntoStatement]]. + */ +abstract class HoodieSpark4ResolveColumnsForInsertInto extends ResolveInsertionBase { + + protected def preprocess(insert: InsertIntoStatement, + catalogTable: Option[CatalogTable]): InsertIntoStatement = { + preprocess(insert, catalogTable, catalogTable.map(_.partitionSchema).getOrElse(new StructType())) + } + + protected def preprocess(insert: InsertIntoStatement, + catalogTable: Option[CatalogTable], + partitionSchema: StructType): InsertIntoStatement = { + val tblName = catalogTable.map(_.identifier.quotedString).getOrElse("unknown") + preprocess(insert, tblName, partitionSchema, catalogTable) + } + + // NOTE: this is copied from [[PreprocessTableInsertion]] with additional logic + // to unset user-specified columns at the end + protected def preprocess(insert: InsertIntoStatement, + tblName: String, + partColNames: StructType, + catalogTable: Option[CatalogTable]): InsertIntoStatement = { + + val normalizedPartSpec = normalizePartitionSpec( + insert.partitionSpec, partColNames, tblName, conf.resolver) + + val staticPartCols = normalizedPartSpec.filter(_._2.isDefined).keySet + val expectedColumns = insert.table.output.filterNot(a => staticPartCols.contains(a.name)) + + val partitionsTrackedByCatalog = catalogTable.isDefined && + catalogTable.get.partitionColumnNames.nonEmpty && + catalogTable.get.tracksPartitionsInCatalog + if (partitionsTrackedByCatalog && normalizedPartSpec.nonEmpty) { + // empty partition column value + if (normalizedPartSpec.values.flatten.exists(v => v != null && v.isEmpty)) { + val spec = normalizedPartSpec.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") + throw QueryCompilationErrors.invalidPartitionSpecError( + s"The spec ($spec) contains an empty partition column value") + } + } + + // Create a project if this INSERT has a user-specified column list. + val hasColumnList = insert.userSpecifiedCols.nonEmpty + val query = if (hasColumnList) { + createProjectForByNameQuery(tblName, insert) + } else { + insert.query + } + val newQuery = try { + TableOutputResolver.resolveOutputColumns( + tblName, + expectedColumns, + query, + byName = hasColumnList || insert.byName, + conf, + supportColDefaultValue = true) + } catch { + case e: AnalysisException if staticPartCols.nonEmpty && + (e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS" || + e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") => + val newException = e.copy( + errorClass = Some("INSERT_PARTITION_COLUMN_ARITY_MISMATCH"), + messageParameters = e.messageParameters ++ Map( + "tableColumns" -> insert.table.output.map(c => toSQLId(c.name)).mkString(", "), + "staticPartCols" -> staticPartCols.toSeq.sorted.map(c => toSQLId(c)).mkString(", ") + )) + newException.setStackTrace(e.getStackTrace) + throw newException + } + if (normalizedPartSpec.nonEmpty) { + if (normalizedPartSpec.size != partColNames.length) { + throw QueryCompilationErrors.requestedPartitionsMismatchTablePartitionsError( + tblName, normalizedPartSpec, partColNames) + } + + // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] + // and the user specified is no longer need after resolution + // (`userSpecifiedCols = Seq()`) + insert.copy(query = newQuery, partitionSpec = normalizedPartSpec, userSpecifiedCols = Seq()) + } else { + // All partition columns are dynamic because the InsertIntoTable command does + // not explicitly specify partitioning columns. + // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] + // and the user specified is no longer need after resolution + // (`userSpecifiedCols = Seq()`) + insert.copy(query = newQuery, partitionSpec = partColNames.map(_.name).map(_ -> None).toMap, + userSpecifiedCols = Seq()) + } + } +} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystExpressionUtils.scala index a183f754483e8..67c6aea40be5a 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystExpressionUtils.scala @@ -17,101 +17,4 @@ package org.apache.spark.sql -import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, EvalMode, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy -import org.apache.spark.sql.types.{DataType, StructType} - -object HoodieSpark40CatalystExpressionUtils extends HoodieSpark4CatalystExpressionUtils with PredicateHelper { - - override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { - ExpressionEncoder.apply(schema).resolveAndBind() - } - - override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = { - DataSourceStrategy.normalizeExprs(exprs, attributes) - } - - override def extractPredicatesWithinOutputSet(condition: Expression, outputSet: AttributeSet): Option[Expression] = { - super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) - } - - override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = { - expr match { - case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) - case _ => None - } - } - - override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { - expr match { - case OrderPreservingTransformation(attrRef) => Some(attrRef) - case _ => None - } - } - - def canUpCast(fromType: DataType, toType: DataType): Boolean = - Cast.canUpCast(fromType, toType) - - override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = - expr match { - case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => - Some((castedExpr, dataType, timeZoneId, if (ansiEnabled == EvalMode.ANSI) true else false)) - case _ => None - } - - private object OrderPreservingTransformation { - def unapply(expr: Expression): Option[AttributeReference] = { - expr match { - // Date/Time Expressions - case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ParseToDate(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _, _, _) => Some(attrRef) - case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // String Expressions - case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Left API change: Improve RuntimeReplaceable - // https://issues.apache.org/jira/browse/SPARK-38240 - case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Math Expressions - // Binary - case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Unary - case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Other - case cast @ Cast(OrderPreservingTransformation(attrRef), _, _, _) - if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) - - // Identity transformation - case attrRef: AttributeReference => Some(attrRef) - // No match - case _ => None - } - } - } -} +object HoodieSpark40CatalystExpressionUtils extends HoodieSpark4CatalystExpressionUtils diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala index a6641def7e524..f4838b9bc8af1 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala @@ -18,128 +18,11 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} -import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.RepairTableCommand -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} -import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.{Assignment, UpdateAction} import org.apache.spark.sql.execution.streaming.SerializedOffset -import org.apache.spark.sql.types.StructType -object HoodieSpark40CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { - - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = - plan match { - case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) - case _ => None - } - - override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { - plan match { - case MergeIntoTable(targetTable, sourceTable, mergeCondition, _, _, _, _) => - Some((targetTable, sourceTable, mergeCondition)) - case _ => None - } - } - - override def maybeApplyForNewFileFormat(plan: LogicalPlan): LogicalPlan = { - plan match { - case s@ScanOperation(_, _, _, - l@LogicalRelation(fs: HadoopFsRelation, _, _, _, _)) - if fs.fileFormat.isInstanceOf[ParquetFileFormat with HoodieFormatTrait] - && !fs.fileFormat.asInstanceOf[ParquetFileFormat with HoodieFormatTrait].isProjected => - FileFormatUtilsForFileGroupReader.applyNewFileFormatChanges(s, l, fs) - case _ => plan - } - } - - override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = - ProjectionOverSchema(schema, output) - - override def isRepairTable(plan: LogicalPlan): Boolean = { - plan.isInstanceOf[RepairTableCommand] - } - - override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { - plan match { - case rtc: RepairTableCommand => - Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) - case _ => - None - } - } - - override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { - a.failAnalysis( - errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", - messageParameters = Map( - "objectName" -> a.sql, - "proposal" -> cols)) - } - - override def failTableNotFound(tableName: String): Unit = { - throw new AnalysisException( - errorClass = "TABLE_OR_VIEW_NOT_FOUND", - messageParameters = Map("relationName" -> s"`$tableName`")) - } - - override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { - plan match { - case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => - Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) - case _ => - None - } - } - - override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { - plan match { - case ci@DropIndex(table, indexName, ignoreIfNotExists) => - Some((table, indexName, ignoreIfNotExists)) - case _ => - None - } - } - - override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { - plan match { - case ci@HoodieShowIndexes(table, output) => - Some((table, output)) - case _ => - None - } - } - - override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { - plan match { - case ci@RefreshIndex(table, indexName) => - Some((table, indexName)) - case _ => - None - } - } - - override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { - plan match { - case insert: InsertIntoStatement => - Some((insert.table, insert.userSpecifiedCols, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists)) - case _ => - None - } - } - - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { - plan match { - case insert: InsertIntoStatement => - Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) - case _ => - None - } - } +object HoodieSpark40CatalystPlanUtils extends HoodieSpark4CatalystPlanUtils { override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { mergeAction match { diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40SchemaUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40SchemaUtils.scala index bda84f2c1bf3e..44547445db83e 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40SchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40SchemaUtils.scala @@ -19,34 +19,7 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils -import org.apache.spark.sql.jdbc.JdbcDialect -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.SchemaUtils - -import java.sql.{Connection, ResultSet} - /** - * Utils on schema for Spark 3.4+. + * Utils on schema for Spark 4.0. */ -object HoodieSpark40SchemaUtils extends HoodieSchemaUtils { - override def checkColumnNameDuplication(columnNames: Seq[String], - colType: String, - caseSensitiveAnalysis: Boolean): Unit = { - SchemaUtils.checkColumnNameDuplication(columnNames, caseSensitiveAnalysis) - } - - override def toAttributes(struct: StructType): Seq[Attribute] = { - DataTypeUtils.toAttributes(struct) - } - - override def getSchema(conn: Connection, - resultSet: ResultSet, - dialect: JdbcDialect, - alwaysNullable: Boolean = false, - isTimestampNTZ: Boolean = false): StructType = { - JdbcUtils.getSchema(conn, resultSet, dialect, alwaysNullable) - } -} +object HoodieSpark40SchemaUtils extends HoodieSpark4SchemaUtils diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark40Analysis.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark40Analysis.scala index ef92ebc6504a9..200fe39af4713 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark40Analysis.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark40Analysis.scala @@ -20,21 +20,16 @@ package org.apache.spark.sql.hudi.analysis import org.apache.hudi.{DefaultSource, EmptyRelation, HoodieBaseRelation} import org.apache.hudi.SparkAdapterSupport.sparkAdapter -import org.apache.spark.sql.{AnalysisException, SparkSession, SQLContext} -import org.apache.spark.sql.catalyst.analysis.{ResolveInsertionBase, TableOutputResolver} -import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.errors.DataTypeErrors.toSQLId -import org.apache.spark.sql.errors.QueryCompilationErrors -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PreprocessTableInsertion} -import org.apache.spark.sql.execution.datasources.LogicalRelation +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.ProvidesHoodieConfig import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.PartitioningUtils.normalizePartitionSpec /** * NOTE: PLEASE READ CAREFULLY @@ -75,28 +70,11 @@ case class HoodieSpark40DataSourceV2ToV1Fallback(sparkSession: SparkSession) ext } /** - * In Spark 3.5, the following Resolution rules are removed, - * [[ResolveUserSpecifiedColumns]] and [[ResolveDefaultColumns]] - * (see code changes in [[org.apache.spark.sql.catalyst.analysis.Analyzer]] - * from https://github.com/apache/spark/pull/41262). - * The same logic of resolving the user specified columns and default values, - * which are required for a subset of columns as user specified compared to the table - * schema to work properly, are deferred to [[PreprocessTableInsertion]] for v1 INSERT. - * - * Note that [[HoodieAnalysis]] intercepts the [[InsertIntoStatement]] after Spark's built-in - * Resolution rules are applies, the logic of resolving the user specified columns and default - * values may no longer be applied. To make INSERT with a subset of columns specified by user - * to work, this custom resolution rule [[HoodieSpark40ResolveColumnsForInsertInto]] is added - * to achieve the same, before converting [[InsertIntoStatement]] into - * [[InsertIntoHoodieTableCommand]]. - * - * The implementation is copied and adapted from [[PreprocessTableInsertion]] - * https://github.com/apache/spark/blob/d061aadf25fd258d2d3e7332a489c9c24a2b5530/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L373 - * - * Also note that, the project logic in [[ResolveImplementationsEarly]] for INSERT is still - * needed in the case of INSERT with all columns in a different ordering. + * Resolution rule resolving the user specified columns and default values of + * [[InsertIntoStatement]] for Spark 4.0; see [[HoodieSpark4ResolveColumnsForInsertInto]] + * for the shared preprocessing logic and the rationale. */ -case class HoodieSpark40ResolveColumnsForInsertInto() extends ResolveInsertionBase { +case class HoodieSpark40ResolveColumnsForInsertInto() extends HoodieSpark4ResolveColumnsForInsertInto { // NOTE: This is copied from [[PreprocessTableInsertion]] with additional handling of Hudi relations override def apply(plan: LogicalPlan): LogicalPlan = { plan match { @@ -123,90 +101,4 @@ case class HoodieSpark40ResolveColumnsForInsertInto() extends ResolveInsertionBa case _ => plan } } - - private def preprocess(insert: InsertIntoStatement, - catalogTable: Option[CatalogTable]): InsertIntoStatement = { - preprocess(insert, catalogTable, catalogTable.map(_.partitionSchema).getOrElse(new StructType())) - } - - private def preprocess(insert: InsertIntoStatement, - catalogTable: Option[CatalogTable], - partitionSchema: StructType): InsertIntoStatement = { - val tblName = catalogTable.map(_.identifier.quotedString).getOrElse("unknown") - preprocess(insert, tblName, partitionSchema, catalogTable) - } - - // NOTE: this is copied from [[PreprocessTableInsertion]] with additional logic - // to unset user-specified columns at the end - private def preprocess(insert: InsertIntoStatement, - tblName: String, - partColNames: StructType, - catalogTable: Option[CatalogTable]): InsertIntoStatement = { - - val normalizedPartSpec = normalizePartitionSpec( - insert.partitionSpec, partColNames, tblName, conf.resolver) - - val staticPartCols = normalizedPartSpec.filter(_._2.isDefined).keySet - val expectedColumns = insert.table.output.filterNot(a => staticPartCols.contains(a.name)) - - val partitionsTrackedByCatalog = catalogTable.isDefined && - catalogTable.get.partitionColumnNames.nonEmpty && - catalogTable.get.tracksPartitionsInCatalog - if (partitionsTrackedByCatalog && normalizedPartSpec.nonEmpty) { - // empty partition column value - if (normalizedPartSpec.values.flatten.exists(v => v != null && v.isEmpty)) { - val spec = normalizedPartSpec.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") - throw QueryCompilationErrors.invalidPartitionSpecError( - s"The spec ($spec) contains an empty partition column value") - } - } - - // Create a project if this INSERT has a user-specified column list. - val hasColumnList = insert.userSpecifiedCols.nonEmpty - val query = if (hasColumnList) { - createProjectForByNameQuery(tblName, insert) - } else { - insert.query - } - val newQuery = try { - TableOutputResolver.resolveOutputColumns( - tblName, - expectedColumns, - query, - byName = hasColumnList || insert.byName, - conf, - supportColDefaultValue = true) - } catch { - case e: AnalysisException if staticPartCols.nonEmpty && - (e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS" || - e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") => - val newException = e.copy( - errorClass = Some("INSERT_PARTITION_COLUMN_ARITY_MISMATCH"), - messageParameters = e.messageParameters ++ Map( - "tableColumns" -> insert.table.output.map(c => toSQLId(c.name)).mkString(", "), - "staticPartCols" -> staticPartCols.toSeq.sorted.map(c => toSQLId(c)).mkString(", ") - )) - newException.setStackTrace(e.getStackTrace) - throw newException - } - if (normalizedPartSpec.nonEmpty) { - if (normalizedPartSpec.size != partColNames.length) { - throw QueryCompilationErrors.requestedPartitionsMismatchTablePartitionsError( - tblName, normalizedPartSpec, partColNames) - } - - // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] - // and the user specified is no longer need after resolution - // (`userSpecifiedCols = Seq()`) - insert.copy(query = newQuery, partitionSpec = normalizedPartSpec, userSpecifiedCols = Seq()) - } else { - // All partition columns are dynamic because the InsertIntoTable command does - // not explicitly specify partitioning columns. - // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] - // and the user specified is no longer need after resolution - // (`userSpecifiedCols = Seq()`) - insert.copy(query = newQuery, partitionSpec = partColNames.map(_.name).map(_ -> None).toMap, - userSpecifiedCols = Seq()) - } - } } diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala index f835d7daace05..c41389f315345 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala @@ -17,101 +17,4 @@ package org.apache.spark.sql -import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, EvalMode, Exp, Expm1, Expression, FromUnixTime, FromUTCTimestamp, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUnixTimestamp, ToUTCTimestamp, Upper} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy -import org.apache.spark.sql.types.{DataType, StructType} - -object HoodieSpark41CatalystExpressionUtils extends HoodieSpark4CatalystExpressionUtils with PredicateHelper { - - override def getEncoder(schema: StructType): ExpressionEncoder[Row] = { - ExpressionEncoder.apply(schema).resolveAndBind() - } - - override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = { - DataSourceStrategy.normalizeExprs(exprs, attributes) - } - - override def extractPredicatesWithinOutputSet(condition: Expression, outputSet: AttributeSet): Option[Expression] = { - super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet) - } - - override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] = { - expr match { - case Cast(child, dataType, timeZoneId, _) => Some((child, dataType, timeZoneId)) - case _ => None - } - } - - override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = { - expr match { - case OrderPreservingTransformation(attrRef) => Some(attrRef) - case _ => None - } - } - - def canUpCast(fromType: DataType, toType: DataType): Boolean = - Cast.canUpCast(fromType, toType) - - override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] = - expr match { - case Cast(castedExpr, dataType, timeZoneId, ansiEnabled) => - Some((castedExpr, dataType, timeZoneId, if (ansiEnabled == EvalMode.ANSI) true else false)) - case _ => None - } - - private object OrderPreservingTransformation { - def unapply(expr: Expression): Option[AttributeReference] = { - expr match { - // Date/Time Expressions - case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ParseToDate(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _, _, _) => Some(attrRef) - case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef) - case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // String Expressions - case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Left API change: Improve RuntimeReplaceable - // https://issues.apache.org/jira/browse/SPARK-38240 - case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Math Expressions - // Binary - case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef) - case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef) - // Unary - case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef) - case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef) - - // Other - case cast@Cast(OrderPreservingTransformation(attrRef), _, _, _) - if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef) - - // Identity transformation - case attrRef: AttributeReference => Some(attrRef) - // No match - case _ => None - } - } - } -} +object HoodieSpark41CatalystExpressionUtils extends HoodieSpark4CatalystExpressionUtils diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala index 6a385810aef9e..24489bb4f16a4 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala @@ -18,128 +18,11 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} -import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.RepairTableCommand -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} -import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.{Assignment, UpdateAction} import org.apache.spark.sql.execution.streaming.runtime.SerializedOffset -import org.apache.spark.sql.types.StructType -object HoodieSpark41CatalystPlanUtils extends BaseHoodieCatalystPlanUtils { - - def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] = - plan match { - case ResolvedTable(catalog, identifier, table, _) => Some((catalog, identifier, table)) - case _ => None - } - - override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = { - plan match { - case MergeIntoTable(targetTable, sourceTable, mergeCondition, _, _, _, _) => - Some((targetTable, sourceTable, mergeCondition)) - case _ => None - } - } - - override def maybeApplyForNewFileFormat(plan: LogicalPlan): LogicalPlan = { - plan match { - case s@ScanOperation(_, _, _, - l@LogicalRelation(fs: HadoopFsRelation, _, _, _, _)) - if fs.fileFormat.isInstanceOf[ParquetFileFormat with HoodieFormatTrait] - && !fs.fileFormat.asInstanceOf[ParquetFileFormat with HoodieFormatTrait].isProjected => - FileFormatUtilsForFileGroupReader.applyNewFileFormatChanges(s, l, fs) - case _ => plan - } - } - - override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = - ProjectionOverSchema(schema, output) - - override def isRepairTable(plan: LogicalPlan): Boolean = { - plan.isInstanceOf[RepairTableCommand] - } - - override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = { - plan match { - case rtc: RepairTableCommand => - Some((rtc.tableName, rtc.enableAddPartitions, rtc.enableDropPartitions, rtc.cmd)) - case _ => - None - } - } - - override def failAnalysisForMIT(a: Attribute, cols: String): Unit = { - a.failAnalysis( - errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", - messageParameters = Map( - "objectName" -> a.sql, - "proposal" -> cols)) - } - - override def failTableNotFound(tableName: String): Unit = { - throw new AnalysisException( - errorClass = "TABLE_OR_VIEW_NOT_FOUND", - messageParameters = Map("relationName" -> s"`$tableName`")) - } - - override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = { - plan match { - case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) => - Some((table, indexName, indexType, ignoreIfExists, columns.map(col => (col._1.name, col._2)), properties)) - case _ => - None - } - } - - override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = { - plan match { - case ci@DropIndex(table, indexName, ignoreIfNotExists) => - Some((table, indexName, ignoreIfNotExists)) - case _ => - None - } - } - - override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = { - plan match { - case ci@HoodieShowIndexes(table, output) => - Some((table, output)) - case _ => - None - } - } - - override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = { - plan match { - case ci@RefreshIndex(table, indexName) => - Some((table, indexName)) - case _ => - None - } - } - - override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = { - plan match { - case insert: InsertIntoStatement => - Some((insert.table, insert.userSpecifiedCols, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists)) - case _ => - None - } - } - - override def createProjectForByNameQuery(lr: LogicalRelation, plan: LogicalPlan): Option[LogicalPlan] = { - plan match { - case insert: InsertIntoStatement => - Some(ResolveInsertionBase.createProjectForByNameQuery(lr.catalogTable.get.qualifiedName, insert)) - case _ => - None - } - } +object HoodieSpark41CatalystPlanUtils extends HoodieSpark4CatalystPlanUtils { override def unapplyUpdateAction(mergeAction: Any): Option[(Option[Expression], Seq[Assignment])] = { mergeAction match { diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala index c68190466ca8f..149989471f95b 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala @@ -19,34 +19,7 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils -import org.apache.spark.sql.jdbc.JdbcDialect -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.SchemaUtils - -import java.sql.{Connection, ResultSet} - /** - * Utils on schema for Spark 3.4+. + * Utils on schema for Spark 4.1. */ -object HoodieSpark41SchemaUtils extends HoodieSchemaUtils { - override def checkColumnNameDuplication(columnNames: Seq[String], - colType: String, - caseSensitiveAnalysis: Boolean): Unit = { - SchemaUtils.checkColumnNameDuplication(columnNames, caseSensitiveAnalysis) - } - - override def toAttributes(struct: StructType): Seq[Attribute] = { - DataTypeUtils.toAttributes(struct) - } - - override def getSchema(conn: Connection, - resultSet: ResultSet, - dialect: JdbcDialect, - alwaysNullable: Boolean = false, - isTimestampNTZ: Boolean = false): StructType = { - JdbcUtils.getSchema(conn, resultSet, dialect, alwaysNullable) - } -} +object HoodieSpark41SchemaUtils extends HoodieSpark4SchemaUtils diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala index 33dc723543e94..1a57c29767193 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala @@ -20,20 +20,16 @@ package org.apache.spark.sql.hudi.analysis import org.apache.hudi.{DefaultSource, EmptyRelation, HoodieBaseRelation} import org.apache.hudi.SparkAdapterSupport.sparkAdapter -import org.apache.spark.sql.{AnalysisException, SparkSession} -import org.apache.spark.sql.catalyst.analysis.{ResolveInsertionBase, TableOutputResolver} -import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.errors.DataTypeErrors.toSQLId -import org.apache.spark.sql.errors.QueryCompilationErrors -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PreprocessTableInsertion} +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.ProvidesHoodieConfig import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.PartitioningUtils.normalizePartitionSpec /** * NOTE: PLEASE READ CAREFULLY @@ -74,28 +70,11 @@ case class HoodieSpark41DataSourceV2ToV1Fallback(sparkSession: SparkSession) ext } /** - * In Spark 3.5, the following Resolution rules are removed, - * [[ResolveUserSpecifiedColumns]] and [[ResolveDefaultColumns]] - * (see code changes in [[org.apache.spark.sql.catalyst.analysis.Analyzer]] - * from https://github.com/apache/spark/pull/41262). - * The same logic of resolving the user specified columns and default values, - * which are required for a subset of columns as user specified compared to the table - * schema to work properly, are deferred to [[PreprocessTableInsertion]] for v1 INSERT. - * - * Note that [[HoodieAnalysis]] intercepts the [[InsertIntoStatement]] after Spark's built-in - * Resolution rules are applies, the logic of resolving the user specified columns and default - * values may no longer be applied. To make INSERT with a subset of columns specified by user - * to work, this custom resolution rule [[HoodieSpark41ResolveColumnsForInsertInto]] is added - * to achieve the same, before converting [[InsertIntoStatement]] into - * [[InsertIntoHoodieTableCommand]]. - * - * The implementation is copied and adapted from [[PreprocessTableInsertion]] - * https://github.com/apache/spark/blob/d061aadf25fd258d2d3e7332a489c9c24a2b5530/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L373 - * - * Also note that, the project logic in [[ResolveImplementationsEarly]] for INSERT is still - * needed in the case of INSERT with all columns in a different ordering. + * Resolution rule resolving the user specified columns and default values of + * [[InsertIntoStatement]] for Spark 4.1; see [[HoodieSpark4ResolveColumnsForInsertInto]] + * for the shared preprocessing logic and the rationale. */ -case class HoodieSpark41ResolveColumnsForInsertInto() extends ResolveInsertionBase { +case class HoodieSpark41ResolveColumnsForInsertInto() extends HoodieSpark4ResolveColumnsForInsertInto { // NOTE: This is copied from [[PreprocessTableInsertion]] with additional handling of Hudi relations override def apply(plan: LogicalPlan): LogicalPlan = { plan match { @@ -122,90 +101,4 @@ case class HoodieSpark41ResolveColumnsForInsertInto() extends ResolveInsertionBa case _ => plan } } - - private def preprocess(insert: InsertIntoStatement, - catalogTable: Option[CatalogTable]): InsertIntoStatement = { - preprocess(insert, catalogTable, catalogTable.map(_.partitionSchema).getOrElse(new StructType())) - } - - private def preprocess(insert: InsertIntoStatement, - catalogTable: Option[CatalogTable], - partitionSchema: StructType): InsertIntoStatement = { - val tblName = catalogTable.map(_.identifier.quotedString).getOrElse("unknown") - preprocess(insert, tblName, partitionSchema, catalogTable) - } - - // NOTE: this is copied from [[PreprocessTableInsertion]] with additional logic - // to unset user-specified columns at the end - private def preprocess(insert: InsertIntoStatement, - tblName: String, - partColNames: StructType, - catalogTable: Option[CatalogTable]): InsertIntoStatement = { - - val normalizedPartSpec = normalizePartitionSpec( - insert.partitionSpec, partColNames, tblName, conf.resolver) - - val staticPartCols = normalizedPartSpec.filter(_._2.isDefined).keySet - val expectedColumns = insert.table.output.filterNot(a => staticPartCols.contains(a.name)) - - val partitionsTrackedByCatalog = catalogTable.isDefined && - catalogTable.get.partitionColumnNames.nonEmpty && - catalogTable.get.tracksPartitionsInCatalog - if (partitionsTrackedByCatalog && normalizedPartSpec.nonEmpty) { - // empty partition column value - if (normalizedPartSpec.values.flatten.exists(v => v != null && v.isEmpty)) { - val spec = normalizedPartSpec.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") - throw QueryCompilationErrors.invalidPartitionSpecError( - s"The spec ($spec) contains an empty partition column value") - } - } - - // Create a project if this INSERT has a user-specified column list. - val hasColumnList = insert.userSpecifiedCols.nonEmpty - val query = if (hasColumnList) { - createProjectForByNameQuery(tblName, insert) - } else { - insert.query - } - val newQuery = try { - TableOutputResolver.resolveOutputColumns( - tblName, - expectedColumns, - query, - byName = hasColumnList || insert.byName, - conf, - supportColDefaultValue = true) - } catch { - case e: AnalysisException if staticPartCols.nonEmpty && - (e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS" || - e.getErrorClass == "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") => - val newException = e.copy( - errorClass = Some("INSERT_PARTITION_COLUMN_ARITY_MISMATCH"), - messageParameters = e.messageParameters ++ Map( - "tableColumns" -> insert.table.output.map(c => toSQLId(c.name)).mkString(", "), - "staticPartCols" -> staticPartCols.toSeq.sorted.map(c => toSQLId(c)).mkString(", ") - )) - newException.setStackTrace(e.getStackTrace) - throw newException - } - if (normalizedPartSpec.nonEmpty) { - if (normalizedPartSpec.size != partColNames.length) { - throw QueryCompilationErrors.requestedPartitionsMismatchTablePartitionsError( - tblName, normalizedPartSpec, partColNames) - } - - // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] - // and the user specified is no longer need after resolution - // (`userSpecifiedCols = Seq()`) - insert.copy(query = newQuery, partitionSpec = normalizedPartSpec, userSpecifiedCols = Seq()) - } else { - // All partition columns are dynamic because the InsertIntoTable command does - // not explicitly specify partitioning columns. - // NOTE: Hudi converts [[InsertIntoStatement]] to [[InsertIntoHoodieTableCommand]] - // and the user specified is no longer need after resolution - // (`userSpecifiedCols = Seq()`) - insert.copy(query = newQuery, partitionSpec = partColNames.map(_.name).map(_ -> None).toMap, - userSpecifiedCols = Seq()) - } - } } From ec8e678d8766413ca1194a65fd8ede49e6786392 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 2 Jul 2026 22:50:35 -0700 Subject: [PATCH 121/255] refactor(spark): consolidate duplicated small utils across Spark version modules (#19150) * refactor(spark): delete unused SparkXXDataSourceUtils from all Spark version modules The six SparkXXDataSourceUtils objects (Spark33-Spark42) were vendored copies of Spark 3.2-era rebase-mode helpers (int96RebaseMode / datetimeRebaseMode returning LegacyBehaviorPolicy). Nothing in the repository references them anymore: all rebase handling in SparkXXParquetReader, SparkXXLegacyHoodieParquetFileFormat, and the Avro (de)serializers goes through Spark's own org.apache.spark.sql.execution.datasources.DataSourceUtils (datetimeRebaseSpec / int96RebaseSpec) instead. Deleting dead code beats deduplicating it. (cherry picked from commit ed9e368510315617f4b8e59fb32ea51c5c9ca867) --- .../org/apache/hudi/HoodieFileScanRDD.scala} | 10 +- ....scala => HoodieNestedSchemaPruning.scala} | 49 ++++++++-- .../datasources/orc/SparkOrcReaderBase.scala | 76 ++++++++++++--- .../BaseResolveHudiAlterTableCommand.scala} | 43 +++++---- .../sql/hudi/analysis/HoodieAnalysis.scala | 34 ++----- .../spark/sql/adapter/BaseSpark3Adapter.scala | 12 ++- .../Spark3ResolveHudiAlterTableCommand.scala | 43 +++++++++ .../hudi/Spark33HoodieFileScanRDD.scala | 36 -------- .../spark/sql/adapter/Spark3_3Adapter.scala | 16 +--- .../Spark33NestedSchemaPruning.scala | 62 ------------- .../datasources/orc/Spark33OrcReader.scala | 85 ----------------- .../parquet/Spark33DataSourceUtils.scala | 77 ---------------- .../Spark33ResolveHudiAlterTableCommand.scala | 67 -------------- .../spark/sql/adapter/Spark3_4Adapter.scala | 16 +--- .../Spark34NestedSchemaPruning.scala | 62 ------------- .../datasources/orc/Spark34OrcReader.scala | 91 ------------------ .../parquet/Spark34DataSourceUtils.scala | 77 ---------------- .../hudi/Spark35HoodieFileScanRDD.scala | 36 -------- .../spark/sql/adapter/Spark3_5Adapter.scala | 16 +--- .../Spark35NestedSchemaPruning.scala | 61 ------------ .../datasources/orc/Spark35OrcReader.scala | 92 ------------------- .../parquet/Spark35DataSourceUtils.scala | 76 --------------- .../Spark35ResolveHudiAlterTableCommand.scala | 67 -------------- .../spark/sql/adapter/BaseSpark4Adapter.scala | 25 ++++- .../HoodieSpark4PartitionedFileUtils.scala} | 4 +- .../Spark4ResolveHudiAlterTableCommand.scala | 37 ++++++++ .../hudi/Spark40HoodieFileScanRDD.scala | 36 -------- .../spark/sql/adapter/Spark4_0Adapter.scala | 32 +------ .../HoodieSpark40PartitionedFileUtils.scala | 68 -------------- .../Spark40NestedSchemaPruning.scala | 61 ------------ .../datasources/orc/Spark40OrcReader.scala | 92 ------------------- .../parquet/Spark40DataSourceUtils.scala | 76 --------------- .../Spark40ResolveHudiAlterTableCommand.scala | 68 -------------- .../hudi/Spark41HoodieFileScanRDD.scala | 36 -------- .../spark/sql/adapter/Spark4_1Adapter.scala | 33 +------ .../Spark41NestedSchemaPruning.scala | 61 ------------ .../datasources/orc/Spark41OrcReader.scala | 92 ------------------- .../parquet/Spark41DataSourceUtils.scala | 76 --------------- .../Spark41ResolveHudiAlterTableCommand.scala | 68 -------------- 39 files changed, 272 insertions(+), 1797 deletions(-) rename hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala => hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala} (77%) rename hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/{BaseHoodieNestedSchemaPruning.scala => HoodieNestedSchemaPruning.scala} (77%) rename hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala => hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/BaseResolveHudiAlterTableCommand.scala} (52%) create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/Spark3ResolveHudiAlterTableCommand.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark34OrcReader.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark35OrcReader.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala rename hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala => hudi-spark4-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark4PartitionedFileUtils.scala} (96%) create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/Spark4ResolveHudiAlterTableCommand.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodieFileScanRDD.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark40PartitionedFileUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark40NestedSchemaPruning.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark40OrcReader.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40DataSourceUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark40ResolveHudiAlterTableCommand.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodieFileScanRDD.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark41OrcReader.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala delete mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala similarity index 77% rename from hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala rename to hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala index df86e5b169c07..92e9caf0d6350 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala @@ -24,11 +24,11 @@ import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} import org.apache.spark.sql.types.StructType -class Spark34HoodieFileScanRDD(@transient private val sparkSession: SparkSession, - read: PartitionedFile => Iterator[InternalRow], - @transient filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty) +class HoodieFileScanRDD(@transient private val sparkSession: SparkSession, + read: PartitionedFile => Iterator[InternalRow], + @transient filePartitions: Seq[FilePartition], + readDataSchema: StructType, + metadataColumns: Seq[AttributeReference] = Seq.empty) extends FileScanRDD(sparkSession, read, filePartitions, readDataSchema, metadataColumns) with HoodieUnsafeRDD { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/BaseHoodieNestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieNestedSchemaPruning.scala similarity index 77% rename from hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/BaseHoodieNestedSchemaPruning.scala rename to hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieNestedSchemaPruning.scala index 9b6e26984401f..42e59eea5c51d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/BaseHoodieNestedSchemaPruning.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieNestedSchemaPruning.scala @@ -37,12 +37,9 @@ import org.apache.spark.sql.util.SchemaUtils.restoreOriginalOutputNames * NOTE: This class is borrowed from Spark 3.2.1, with modifications adapting it to handle [[HoodieBaseRelation]], * instead of [[HadoopFsRelation]] */ -abstract class BaseHoodieNestedSchemaPruning extends Rule[LogicalPlan] { +class HoodieNestedSchemaPruning extends Rule[LogicalPlan] { import org.apache.spark.sql.catalyst.expressions.SchemaPruning._ - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], requiredSchema: StructType): Seq[AttributeReference] - override def apply(plan: LogicalPlan): LogicalPlan = if (conf.nestedSchemaPruningEnabled) { apply0(plan) @@ -50,13 +47,51 @@ abstract class BaseHoodieNestedSchemaPruning extends Rule[LogicalPlan] { plan } - protected def apply0(plan: LogicalPlan): LogicalPlan + private def apply0(plan: LogicalPlan): LogicalPlan = + plan transformDown { + // NOTE: The relation is matched by type rather than by destructuring [[LogicalRelation]], + // since the arity of its unapply differs across the Spark versions this module + // compiles against. This is modified to accommodate for Hudi's custom relations, + // given that original [[NestedSchemaPruning]] rule is tightly coupled w/ + // [[HadoopFsRelation]] + // TODO generalize to any file-based relation + case op @ PhysicalOperation(projects, filters, l: LogicalRelation) => + l.relation match { + case relation: HoodieBaseRelation if relation.canPruneRelationSchema => + prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, + prunedDataSchema => { + val prunedRelation = + relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) + buildPrunedRelation(l, prunedRelation) + }).getOrElse(op) + case _ => op + } + } + + // Prune the given output to make it consistent with `requiredSchema`. + private def getPrunedOutput(output: Seq[AttributeReference], + requiredSchema: StructType): Seq[AttributeReference] = { + // We need to replace the expression ids of the pruned relation output attributes + // with the expression ids of the original relation output attributes so that + // references to the original relation's output are not broken + val outputIdMap = output.map(att => (att.name, att.exprId)).toMap + // NOTE: The attributes are constructed inline (equivalent to StructType#toAttributes before + // Spark 3.5 and DataTypeUtils#toAttributes since, see SPARK-44353) so that this code + // compiles against every supported Spark version + requiredSchema + .map(f => AttributeReference(f.name, f.dataType, f.nullable, f.metadata)()) + .map { + case att if outputIdMap.contains(att.name) => + att.withExprId(outputIdMap(att.name)) + case att => att + } + } /** * This method returns optional logical plan. `None` is returned if no nested field is required or * all nested fields are required. */ - protected def prunePhysicalColumns(output: Seq[AttributeReference], + private def prunePhysicalColumns(output: Seq[AttributeReference], projects: Seq[NamedExpression], filters: Seq[Expression], dataSchema: StructType, @@ -148,7 +183,7 @@ abstract class BaseHoodieNestedSchemaPruning extends Rule[LogicalPlan] { * Builds a pruned logical relation from the output of the output relation and the schema of the * pruned base relation. */ - protected def buildPrunedRelation(outputRelation: LogicalRelation, + private def buildPrunedRelation(outputRelation: LogicalRelation, prunedBaseRelation: BaseRelation): LogicalRelation = { val prunedOutput = getPrunedOutput(outputRelation.output, prunedBaseRelation.schema) outputRelation.copy(relation = prunedBaseRelation, output = prunedOutput) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/orc/SparkOrcReaderBase.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/orc/SparkOrcReaderBase.scala index ca16e323246db..fa056921aeac6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/orc/SparkOrcReaderBase.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/orc/SparkOrcReaderBase.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources.orc +import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.common.util import org.apache.hudi.internal.schema.InternalSchema import org.apache.hudi.storage.StorageConfiguration @@ -32,19 +33,25 @@ import org.apache.orc.{OrcConf, OrcFile, TypeDescription} import org.apache.orc.mapred.OrcStruct import org.apache.orc.mapreduce.OrcInputFormat import org.apache.spark.TaskContext +import org.apache.spark.memory.MemoryMode import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Attribute, JoinedRow} +import org.apache.spark.sql.catalyst.expressions.JoinedRow import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.execution.datasources.{PartitionedFile, RecordReaderIterator, SparkColumnarFileReader} +import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile, RecordReaderIterator, SparkColumnarFileReader} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils -abstract class SparkOrcReaderBase(enableVectorizedReader: Boolean, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean) extends SparkColumnarFileReader { +class SparkOrcReaderBase(enableVectorizedReader: Boolean, + dataSchema: StructType, + orcFilterPushDown: Boolean, + isCaseSensitive: Boolean, + capacity: Int, + memoryMode: MemoryMode, + batchReaderFactory: (Int, MemoryMode) => OrcColumnarBatchReader) + extends SparkColumnarFileReader with SparkAdapterSupport { /** * Read an individual ORC file * @@ -62,7 +69,7 @@ abstract class SparkOrcReaderBase(enableVectorizedReader: Boolean, val resultSchema = StructType(requiredSchema.fields ++ partitionSchema.fields) val conf = storageConf.unwrap() - val filePath = partitionedFileToPath(file) + val filePath = new Path(sparkAdapter.getSparkPartitionedFileUtils.getPathFromPartitionedFile(file).toUri) val fs = filePath.getFileSystem(conf) val readerOptions = OrcFile.readerOptions(conf).filesystem(fs) @@ -96,7 +103,7 @@ abstract class SparkOrcReaderBase(enableVectorizedReader: Boolean, val taskAttemptContext = new TaskAttemptContextImpl(taskConf, attemptId) if (enableVectorizedReader) { - val batchReader = buildReader() + val batchReader = batchReaderFactory(capacity, memoryMode) // SPARK-23399 Register a task completion listener first to call `close()` in all cases. // There is a possibility that `initialize` and `initBatch` hit some errors (like OOM) // after opening a file. @@ -120,7 +127,8 @@ abstract class SparkOrcReaderBase(enableVectorizedReader: Boolean, val iter = new RecordReaderIterator[OrcStruct](orcRecordReader) Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => iter.close())) - val fullSchema = structTypeToAttributes(requiredSchema) ++ structTypeToAttributes(partitionSchema) + val schemaUtils = sparkAdapter.getSchemaUtils + val fullSchema = schemaUtils.toAttributes(requiredSchema) ++ schemaUtils.toAttributes(partitionSchema) val unsafeProjection = GenerateUnsafeProjection.generate(fullSchema, fullSchema) val deserializer = new OrcDeserializer(requiredSchema, requestedColIds) @@ -134,10 +142,52 @@ abstract class SparkOrcReaderBase(enableVectorizedReader: Boolean, } } } +} - def partitionedFileToPath(file: PartitionedFile): Path - - def buildReader(): OrcColumnarBatchReader +object SparkOrcReaderBase { + /** + * Get ORC file reader + * + * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc + * @param sqlConf the [[SQLConf]] used for the read + * @param options passed as a param to the file format + * @param hadoopConf some configs will be set for the hadoopConf + * @param dataSchema schema of the data + * @param batchReaderFactory creates the [[OrcColumnarBatchReader]] from the batch size and memory + * mode; the reader constructor differs across Spark versions + * @return ORC file reader + */ + def build(vectorized: Boolean, + sqlConf: SQLConf, + options: Map[String, String], + hadoopConf: Configuration, + dataSchema: StructType, + batchReaderFactory: (Int, MemoryMode) => OrcColumnarBatchReader): SparkOrcReaderBase = { + //set hadoopconf + hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) + hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) + hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) + + val memoryMode = if (sqlConf.offHeapColumnVectorEnabled) { + MemoryMode.OFF_HEAP + } else { + MemoryMode.ON_HEAP + } - def structTypeToAttributes(schema: StructType): Seq[Attribute] + val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && + options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, + throw new IllegalArgumentException( + "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + + "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) + .equals("true") + + new SparkOrcReaderBase( + enableVectorizedReader = enableVectorizedReader && vectorized, + dataSchema = dataSchema, + orcFilterPushDown = sqlConf.orcFilterPushDown, + isCaseSensitive = sqlConf.caseSensitiveAnalysis, + capacity = sqlConf.orcVectorizedReaderBatchSize, + memoryMode = memoryMode, + batchReaderFactory = batchReaderFactory) + } } diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/BaseResolveHudiAlterTableCommand.scala similarity index 52% rename from hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala rename to hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/BaseResolveHudiAlterTableCommand.scala index 31d2f93efbdec..029ba63b4b41c 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/BaseResolveHudiAlterTableCommand.scala @@ -30,32 +30,38 @@ import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCom * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. * for alter table column commands. */ -class Spark34ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { +abstract class BaseResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = { if (ProvidesHoodieConfig.isSchemaEvolutionEnabled(sparkSession)) { - plan.resolveOperatorsUp { - case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => - HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) - case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => - HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) - case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => - HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) - case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => - HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) - case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => - HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) - case alter@AlterColumn(ResolvedHoodieV2TablePlan(t), _, _, _, _, _, _) if alter.resolved => - HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) - case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => - HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) - } + plan.resolveOperatorsUp(resolveCommonCommand.orElse(resolveAlterColumnCommand)) } else { plan } } - object ResolvedHoodieV2TablePlan { + private def resolveCommonCommand: PartialFunction[LogicalPlan, LogicalPlan] = { + case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => + HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) + case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => + HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) + case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => + HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) + case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => + HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) + case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => + HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) + case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => + HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) + } + + /** + * Resolves the ALTER TABLE ... ALTER COLUMN command, whose logical plan differs between + * the Spark 3.x (AlterColumn) and Spark 4.x (AlterColumns) branches. + */ + protected def resolveAlterColumnCommand: PartialFunction[LogicalPlan, LogicalPlan] + + protected object ResolvedHoodieV2TablePlan { def unapply(plan: LogicalPlan): Option[HoodieInternalV2Table] = { plan match { case ResolvedTable(_, _, v2Table: HoodieInternalV2Table, _) => Some(v2Table) @@ -64,4 +70,3 @@ class Spark34ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Ru } } } - diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala index 41d6c2b9ae81c..8872c5269f50e 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala @@ -18,8 +18,8 @@ package org.apache.spark.sql.hudi.analysis import org.apache.hudi.{HoodieSchemaUtils, HoodieSparkUtils, SparkAdapterSupport} -import org.apache.hudi.common.util.{ReflectionUtils, ValidationUtils} import org.apache.hudi.common.util.ReflectionUtils.loadClass +import org.apache.hudi.common.util.ValidationUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.command._ -import org.apache.spark.sql.execution.datasources.{CreateTable, LogicalRelation} +import org.apache.spark.sql.execution.datasources.{CreateTable, HoodieNestedSchemaPruning, LogicalRelation} import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.{isMetaField, removeMetaFields} import org.apache.spark.sql.hudi.analysis.HoodieAnalysis.{sparkAdapter, MatchCreateIndex, MatchCreateTableLike, MatchDropIndex, MatchInsertIntoStatement, MatchMergeIntoTable, MatchRefreshIndex, MatchShowIndexes, ResolvesToHudiTable} import org.apache.spark.sql.hudi.blob.ReadBlobRule @@ -95,16 +95,10 @@ object HoodieAnalysis extends SparkAdapterSupport { } val resolveAlterTableCommandsClass = - if (HoodieSparkUtils.isSpark4_1) { - "org.apache.spark.sql.hudi.Spark41ResolveHudiAlterTableCommand" - } else if (HoodieSparkUtils.isSpark4_0) { - "org.apache.spark.sql.hudi.Spark40ResolveHudiAlterTableCommand" - } else if (HoodieSparkUtils.gteqSpark3_5) { - "org.apache.spark.sql.hudi.Spark35ResolveHudiAlterTableCommand" - } else if (HoodieSparkUtils.isSpark3_4) { - "org.apache.spark.sql.hudi.Spark34ResolveHudiAlterTableCommand" - } else if (HoodieSparkUtils.isSpark3_3) { - "org.apache.spark.sql.hudi.Spark33ResolveHudiAlterTableCommand" + if (HoodieSparkUtils.isSpark4) { + "org.apache.spark.sql.hudi.Spark4ResolveHudiAlterTableCommand" + } else if (HoodieSparkUtils.isSpark3) { + "org.apache.spark.sql.hudi.Spark3ResolveHudiAlterTableCommand" } else { throw new IllegalStateException("Unsupported Spark version") } @@ -143,21 +137,7 @@ object HoodieAnalysis extends SparkAdapterSupport { // Default rules ) - val nestedSchemaPruningClass = - if (HoodieSparkUtils.isSpark4_1) { - "org.apache.spark.sql.execution.datasources.Spark41NestedSchemaPruning" - } else if (HoodieSparkUtils.isSpark4_0) { - "org.apache.spark.sql.execution.datasources.Spark40NestedSchemaPruning" - } else if (HoodieSparkUtils.gteqSpark3_5) { - "org.apache.spark.sql.execution.datasources.Spark35NestedSchemaPruning" - } else if (HoodieSparkUtils.gteqSpark3_4) { - "org.apache.spark.sql.execution.datasources.Spark34NestedSchemaPruning" - } else { - // spark 3.3 - "org.apache.spark.sql.execution.datasources.Spark33NestedSchemaPruning" - } - - val nestedSchemaPruningRule = ReflectionUtils.loadClass(nestedSchemaPruningClass).asInstanceOf[Rule[LogicalPlan]] + val nestedSchemaPruningRule = new HoodieNestedSchemaPruning rules += (_ => nestedSchemaPruningRule) // NOTE: [[HoodiePruneFileSourcePartitions]] is a replica in kind to Spark's diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala index c7039c951c4ef..90be1df5068d8 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.{DefaultSource, HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, HoodieSchemaConversionUtils, Spark3HoodiePartitionCDCFileGroupMapping, Spark3HoodiePartitionFileSliceMapping} +import org.apache.hudi.{DefaultSource, HoodieFileScanRDD, HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, HoodieSchemaConversionUtils, Spark3HoodiePartitionCDCFileGroupMapping, Spark3HoodiePartitionFileSliceMapping} import org.apache.hudi.client.model.{HoodieInternalRow, Spark3HoodieInternalRow} import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.schema.HoodieSchema @@ -36,7 +36,7 @@ import org.apache.spark.sql.FileFormatUtilsForFileGroupReader.applyFiltersToPlan import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{Expression, InterpretedPredicate, Predicate, SpecializedGetters} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, InterpretedPredicate, Predicate, SpecializedGetters} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -101,6 +101,14 @@ abstract class BaseSpark3Adapter extends SparkAdapter with Logging { Predicate.createInterpreted(e) } + override def createHoodieFileScanRDD(sparkSession: SparkSession, + readFunction: PartitionedFile => Iterator[InternalRow], + filePartitions: Seq[FilePartition], + readDataSchema: StructType, + metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { + new HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) + } + override def createRelation(sqlContext: SQLContext, metaClient: HoodieTableMetaClient, schema: HoodieSchema, diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/Spark3ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/Spark3ResolveHudiAlterTableCommand.scala new file mode 100644 index 0000000000000..6246c8480ab7e --- /dev/null +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/Spark3ResolveHudiAlterTableCommand.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi + +import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{AlterColumn, LogicalPlan} +import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} + +/** + * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. + * for alter table column commands. + */ +class Spark3ResolveHudiAlterTableCommand(sparkSession: SparkSession) + extends BaseResolveHudiAlterTableCommand(sparkSession) { + + // NOTE: The command is matched by type rather than by destructuring [[AlterColumn]], since + // the arity of its unapply differs between Spark 3.3 and Spark 3.4+ + override protected def resolveAlterColumnCommand: PartialFunction[LogicalPlan, LogicalPlan] = { + case alter: AlterColumn if alter.resolved => + alter.table match { + case ResolvedHoodieV2TablePlan(t) => + HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) + case _ => alter + } + } +} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala deleted file mode 100644 index b2ed3bce23321..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.types.StructType - -class Spark33HoodieFileScanRDD(@transient private val sparkSession: SparkSession, - read: PartitionedFile => Iterator[InternalRow], - @transient filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty) - extends FileScanRDD(sparkSession, read, filePartitions, readDataSchema, metadataColumns) - with HoodieUnsafeRDD { - - override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect() -} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala index a4cb5b70e72e0..25553f8ebdd74 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.Spark33HoodieFileScanRDD import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.storage.StorageConfiguration @@ -31,14 +30,14 @@ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} +import org.apache.spark.sql.catalyst.expressions.{Expression} import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.{METADATA_COL_ATTR_KEY, RebaseDateTime} import org.apache.spark.sql.connector.catalog.{V1Table, V2TableWithV1Fallback} import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.orc.Spark33OrcReader +import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, SparkOrcReaderBase} import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters, Spark33LegacyHoodieParquetFileFormat, Spark33ParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.analysis.TableValuedFunctions @@ -100,14 +99,6 @@ class Spark3_3Adapter extends BaseSpark3Adapter { Some(new Spark33LegacyHoodieParquetFileFormat(appendPartitionValues)) } - override def createHoodieFileScanRDD(sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { - new Spark33HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) - } - override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition } @@ -163,7 +154,8 @@ class Spark3_3Adapter extends BaseSpark3Adapter { } override def createOrcFileReader(vectorized: Boolean, sqlConf: SQLConf, options: Map[String, String], hadoopConf: Configuration, dataSchema: StructType): SparkColumnarFileReader = { - Spark33OrcReader.build(vectorized, sqlConf, options, hadoopConf, dataSchema) + SparkOrcReaderBase.build(vectorized, sqlConf, options, hadoopConf, dataSchema, + (capacity, _) => new OrcColumnarBatchReader(capacity)) } override def createLanceFileReader(vectorized: Boolean, diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala deleted file mode 100644 index c2235506eee6e..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.HoodieBaseRelation - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.sources.BaseRelation -import org.apache.spark.sql.types.StructType - -class Spark33NestedSchemaPruning extends BaseHoodieNestedSchemaPruning { - - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], - requiredSchema: StructType): Seq[AttributeReference] = { - // We need to replace the expression ids of the pruned relation output attributes - // with the expression ids of the original relation output attributes so that - // references to the original relation's output are not broken - val outputIdMap = output.map(att => (att.name, att.exprId)).toMap - requiredSchema - .toAttributes - .map { - case att if outputIdMap.contains(att.name) => - att.withExprId(outputIdMap(att.name)) - case att => att - } - } - - override protected def apply0(plan: LogicalPlan): LogicalPlan = - plan transformDown { - case op @ PhysicalOperation(projects, filters, - // NOTE: This is modified to accommodate for Hudi's custom relations, given that original - // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]] - // TODO generalize to any file-based relation - l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _)) - if relation.canPruneRelationSchema => - - prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, - prunedDataSchema => { - val prunedRelation = - relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) - buildPrunedRelation(l, prunedRelation) - }).getOrElse(op) - } -} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala deleted file mode 100644 index 25ac5938b8d1c..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType - -import java.net.URI - -class Spark33OrcReader(enableVectorizedReader: Boolean, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean, - capacity: Int) extends SparkOrcReaderBase(enableVectorizedReader, dataSchema, orcFilterPushDown, isCaseSensitive) { - - override def partitionedFileToPath(file: PartitionedFile): Path = { - new Path(new URI(file.filePath)) - } - - override def buildReader(): OrcColumnarBatchReader = { - new OrcColumnarBatchReader(capacity) - } - - override def structTypeToAttributes(schema: StructType): Seq[Attribute] = { - schema.toAttributes - } -} - -object Spark33OrcReader { - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @return ORC file reader - */ - def build(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): Spark33OrcReader = { - //set hadoopconf - hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) - hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) - - val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && - options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, - throw new IllegalArgumentException( - "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + - "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) - .equals("true") - - new Spark33OrcReader( - enableVectorizedReader = enableVectorizedReader && vectorized, - isCaseSensitive = sqlConf.caseSensitiveAnalysis, - capacity = sqlConf.orcVectorizedReaderBatchSize, - orcFilterPushDown = sqlConf.orcFilterPushDown, - dataSchema = dataSchema) - } -} - diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala deleted file mode 100644 index 2aa85660eb511..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.util.Utils - -object Spark33DataSourceUtils { - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def int96RebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to - // rebase the INT96 timestamp values. - // Files written by Spark 3.1 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.1.0" || lookupFileMeta("org.apache.spark.legacyINT96") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def datetimeRebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to - // rebase the datetime values. - // Files written by Spark 3.0 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.0.0" || lookupFileMeta("org.apache.spark.legacyDateTime") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - -} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala deleted file mode 100644 index 55159bcc71b49..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hudi - -import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.ResolvedTable -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table -import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} - -/** - * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. - * for alter table column commands. - */ -class Spark33ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { - - def apply(plan: LogicalPlan): LogicalPlan = { - if (ProvidesHoodieConfig.isSchemaEvolutionEnabled(sparkSession)) { - plan.resolveOperatorsUp { - case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => - HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) - case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => - HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) - case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => - HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) - case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => - HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) - case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => - HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) - case alter@AlterColumn(ResolvedHoodieV2TablePlan(t), _, _, _, _, _) if alter.resolved => - HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) - case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => - HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) - } - } else { - plan - } - } - - object ResolvedHoodieV2TablePlan { - def unapply(plan: LogicalPlan): Option[HoodieInternalV2Table] = { - plan match { - case ResolvedTable(_, _, v2Table: HoodieInternalV2Table, _) => Some(v2Table) - case _ => None - } - } - } -} - diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala index 923c8ac91959c..3d62ba1edc0c7 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.Spark34HoodieFileScanRDD import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.storage.StorageConfiguration @@ -31,7 +30,7 @@ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} +import org.apache.spark.sql.catalyst.expressions.{Expression} import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ @@ -39,7 +38,7 @@ import org.apache.spark.sql.catalyst.util.{METADATA_COL_ATTR_KEY, RebaseDateTime import org.apache.spark.sql.connector.catalog.{V1Table, V2TableWithV1Fallback} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.lance.SparkLanceReaderBase -import org.apache.spark.sql.execution.datasources.orc.Spark34OrcReader +import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, SparkOrcReaderBase} import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters, Spark34LegacyHoodieParquetFileFormat, Spark34ParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.analysis.TableValuedFunctions @@ -101,14 +100,6 @@ class Spark3_4Adapter extends BaseSpark3Adapter { Some(new Spark34LegacyHoodieParquetFileFormat(appendPartitionValues)) } - override def createHoodieFileScanRDD(sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { - new Spark34HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) - } - override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition } @@ -164,7 +155,8 @@ class Spark3_4Adapter extends BaseSpark3Adapter { } override def createOrcFileReader(vectorized: Boolean, sqlConf: SQLConf, options: Map[String, String], hadoopConf: Configuration, dataSchema: StructType): SparkColumnarFileReader = { - Spark34OrcReader.build(vectorized, sqlConf, options, hadoopConf, dataSchema) + SparkOrcReaderBase.build(vectorized, sqlConf, options, hadoopConf, dataSchema, + (capacity, memoryMode) => new OrcColumnarBatchReader(capacity, memoryMode)) } override def createLanceFileReader(vectorized: Boolean, diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala deleted file mode 100644 index 6f3e6d12e23e8..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.HoodieBaseRelation - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.sources.BaseRelation -import org.apache.spark.sql.types.StructType - -class Spark34NestedSchemaPruning extends BaseHoodieNestedSchemaPruning { - - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], - requiredSchema: StructType): Seq[AttributeReference] = { - // We need to replace the expression ids of the pruned relation output attributes - // with the expression ids of the original relation output attributes so that - // references to the original relation's output are not broken - val outputIdMap = output.map(att => (att.name, att.exprId)).toMap - requiredSchema - .toAttributes - .map { - case att if outputIdMap.contains(att.name) => - att.withExprId(outputIdMap(att.name)) - case att => att - } - } - - override protected def apply0(plan: LogicalPlan): LogicalPlan = - plan transformDown { - case op @ PhysicalOperation(projects, filters, - // NOTE: This is modified to accommodate for Hudi's custom relations, given that original - // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]] - // TODO generalize to any file-based relation - l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _)) - if relation.canPruneRelationSchema => - - prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, - prunedDataSchema => { - val prunedRelation = - relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) - buildPrunedRelation(l, prunedRelation) - }).getOrElse(op) - } -} diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark34OrcReader.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark34OrcReader.scala deleted file mode 100644 index a1463aa01aaf3..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark34OrcReader.scala +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.memory.MemoryMode -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType - -class Spark34OrcReader(enableVectorizedReader: Boolean, - memoryMode: MemoryMode, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean, - capacity: Int) extends SparkOrcReaderBase(enableVectorizedReader, dataSchema, orcFilterPushDown, isCaseSensitive) { - - override def partitionedFileToPath(file: PartitionedFile): Path = { - file.toPath - } - - override def buildReader(): OrcColumnarBatchReader = { - new OrcColumnarBatchReader(capacity, memoryMode) - } - - override def structTypeToAttributes(schema: StructType): Seq[Attribute] = { - schema.toAttributes - } -} - -object Spark34OrcReader { - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @return ORC file reader - */ - def build(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): Spark34OrcReader = { - //set hadoopconf - hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) - hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) - - val memoryMode = if (sqlConf.offHeapColumnVectorEnabled) { - MemoryMode.OFF_HEAP - } else { - MemoryMode.ON_HEAP - } - - val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && - options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, - throw new IllegalArgumentException( - "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + - "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) - .equals("true") - - new Spark34OrcReader( - enableVectorizedReader = enableVectorizedReader && vectorized, - memoryMode = memoryMode, - isCaseSensitive = sqlConf.caseSensitiveAnalysis, - capacity = sqlConf.orcVectorizedReaderBatchSize, - orcFilterPushDown = sqlConf.orcFilterPushDown, - dataSchema = dataSchema) - } -} diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala deleted file mode 100644 index d404bc8c24b53..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.util.Utils - -object Spark34DataSourceUtils { - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def int96RebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to - // rebase the INT96 timestamp values. - // Files written by Spark 3.1 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.1.0" || lookupFileMeta("org.apache.spark.legacyINT96") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def datetimeRebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to - // rebase the datetime values. - // Files written by Spark 3.0 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.0.0" || lookupFileMeta("org.apache.spark.legacyDateTime") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala deleted file mode 100644 index 9ab3c04605d5f..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.types.StructType - -class Spark35HoodieFileScanRDD(@transient private val sparkSession: SparkSession, - read: PartitionedFile => Iterator[InternalRow], - @transient filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty) - extends FileScanRDD(sparkSession, read, filePartitions, readDataSchema, metadataColumns) - with HoodieUnsafeRDD { - - override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect() -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala index a2dfe3e7be60c..3fd01ae7085b1 100644 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.Spark35HoodieFileScanRDD import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.storage.StorageConfiguration @@ -30,7 +29,7 @@ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} +import org.apache.spark.sql.catalyst.expressions.{Expression} import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ @@ -38,7 +37,7 @@ import org.apache.spark.sql.catalyst.util.{METADATA_COL_ATTR_KEY, RebaseDateTime import org.apache.spark.sql.connector.catalog.{V1Table, V2TableWithV1Fallback} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.lance.SparkLanceReaderBase -import org.apache.spark.sql.execution.datasources.orc.Spark35OrcReader +import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, SparkOrcReaderBase} import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters, Spark35LegacyHoodieParquetFileFormat, Spark35ParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.analysis.TableValuedFunctions @@ -103,14 +102,6 @@ class Spark3_5Adapter extends BaseSpark3Adapter { Some(new Spark35LegacyHoodieParquetFileFormat(appendPartitionValues)) } - override def createHoodieFileScanRDD(sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { - new Spark35HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) - } - override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition } @@ -179,7 +170,8 @@ class Spark3_5Adapter extends BaseSpark3Adapter { options: Map[String, String], hadoopConf: Configuration, dataSchema: StructType): SparkColumnarFileReader = { - Spark35OrcReader.build(vectorized, sqlConf, options, hadoopConf, dataSchema) + SparkOrcReaderBase.build(vectorized, sqlConf, options, hadoopConf, dataSchema, + (capacity, memoryMode) => new OrcColumnarBatchReader(capacity, memoryMode)) } override def createLanceFileReader(vectorized: Boolean, diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala deleted file mode 100644 index 24d07b085af92..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.HoodieBaseRelation - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.types.StructType - -class Spark35NestedSchemaPruning extends BaseHoodieNestedSchemaPruning { - - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], - requiredSchema: StructType): Seq[AttributeReference] = { - // We need to replace the expression ids of the pruned relation output attributes - // with the expression ids of the original relation output attributes so that - // references to the original relation's output are not broken - val outputIdMap = output.map(att => (att.name, att.exprId)).toMap - DataTypeUtils.toAttributes(requiredSchema) - .map { - case att if outputIdMap.contains(att.name) => - att.withExprId(outputIdMap(att.name)) - case att => att - } - } - - override protected def apply0(plan: LogicalPlan): LogicalPlan = - plan transformDown { - case op @ PhysicalOperation(projects, filters, - // NOTE: This is modified to accommodate for Hudi's custom relations, given that original - // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]] - // TODO generalize to any file-based relation - l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _)) - if relation.canPruneRelationSchema => - - prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, - prunedDataSchema => { - val prunedRelation = - relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) - buildPrunedRelation(l, prunedRelation) - }).getOrElse(op) - } -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark35OrcReader.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark35OrcReader.scala deleted file mode 100644 index badd76c6a8851..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark35OrcReader.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.memory.MemoryMode -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes -import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType - -class Spark35OrcReader(enableVectorizedReader: Boolean, - memoryMode: MemoryMode, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean, - capacity: Int) extends SparkOrcReaderBase(enableVectorizedReader, dataSchema, orcFilterPushDown, isCaseSensitive) { - - override def partitionedFileToPath(file: PartitionedFile): Path = { - file.toPath - } - - override def buildReader(): OrcColumnarBatchReader = { - new OrcColumnarBatchReader(capacity, memoryMode) - } - - override def structTypeToAttributes(schema: StructType): Seq[Attribute] = { - toAttributes(schema) - } -} - -object Spark35OrcReader { - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @return ORC file reader - */ - def build(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): Spark35OrcReader = { - //set hadoopconf - hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) - hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) - - val memoryMode = if (sqlConf.offHeapColumnVectorEnabled) { - MemoryMode.OFF_HEAP - } else { - MemoryMode.ON_HEAP - } - - val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && - options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, - throw new IllegalArgumentException( - "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + - "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) - .equals("true") - - new Spark35OrcReader( - enableVectorizedReader = enableVectorizedReader && vectorized, - memoryMode = memoryMode, - isCaseSensitive = sqlConf.caseSensitiveAnalysis, - capacity = sqlConf.orcVectorizedReaderBatchSize, - orcFilterPushDown = sqlConf.orcFilterPushDown, - dataSchema = dataSchema) - } -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala deleted file mode 100644 index 9e3c63529b481..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY -import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} -import org.apache.spark.util.Utils - -object Spark35DataSourceUtils { - - /** - * NOTE: This method was copied from [[Spark32PlusDataSourceUtils]], and is required to maintain runtime - * compatibility against Spark 3.5.0 - */ - // scalastyle:off - def int96RebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to - // rebase the INT96 timestamp values. - // Files written by Spark 3.1 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.1.0" || lookupFileMeta("org.apache.spark.legacyINT96") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def datetimeRebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to - // rebase the datetime values. - // Files written by Spark 3.0 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.0.0" || lookupFileMeta("org.apache.spark.legacyDateTime") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala deleted file mode 100644 index 8e0f41c2b9964..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hudi - -import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.ResolvedTable -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table -import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} - -/** - * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. - * for alter table column commands. - */ -class Spark35ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { - - def apply(plan: LogicalPlan): LogicalPlan = { - if (ProvidesHoodieConfig.isSchemaEvolutionEnabled(sparkSession)) { - plan.resolveOperatorsUp { - case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => - HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) - case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => - HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) - case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => - HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) - case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => - HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) - case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => - HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) - case alter@AlterColumn(ResolvedHoodieV2TablePlan(t), _, _, _, _, _, _) if alter.resolved => - HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) - case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => - HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) - } - } else { - plan - } - } - - object ResolvedHoodieV2TablePlan { - def unapply(plan: LogicalPlan): Option[HoodieInternalV2Table] = { - plan match { - case ResolvedTable(_, _, v2Table: HoodieInternalV2Table, _) => Some(v2Table) - case _ => None - } - } - } -} - diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark4Adapter.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark4Adapter.scala index ea6e96943a692..27a835b821f26 100644 --- a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark4Adapter.scala +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark4Adapter.scala @@ -17,13 +17,14 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.{AvroConversionUtils, DefaultSource, HoodieSchemaConversionUtils} +import org.apache.hudi.{AvroConversionUtils, DefaultSource, HoodieFileScanRDD, HoodieSchemaConversionUtils} import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.util.JsonUtils import org.apache.hudi.spark.internal.ReflectUtil import org.apache.hudi.storage.StorageConfiguration +import org.apache.hadoop.conf.Configuration import org.apache.parquet.schema.{GroupType, MessageType, PrimitiveType, Type, Types} import org.apache.parquet.schema.Type.Repetition import org.apache.spark.api.java.JavaSparkContext @@ -34,7 +35,7 @@ import org.apache.spark.sql.FileFormatUtilsForFileGroupReader.applyFiltersToPlan import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{Expression, InterpretedPredicate, Predicate, SpecializedGetters} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, InterpretedPredicate, Predicate, SpecializedGetters} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -42,6 +43,7 @@ import org.apache.spark.sql.catalyst.util.DateFormatter import org.apache.spark.sql.classic.ColumnConversions import org.apache.spark.sql.execution.{PartitionedFileUtil, QueryExecution, SQLExecution} import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, SparkOrcReaderBase} import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFilters, SparkShreddingUtils} import org.apache.spark.sql.hudi.SparkAdapter import org.apache.spark.sql.internal.SQLConf @@ -101,6 +103,23 @@ abstract class BaseSpark4Adapter extends SparkAdapter with Logging { Predicate.createInterpreted(e) } + override def createHoodieFileScanRDD(sparkSession: SparkSession, + readFunction: PartitionedFile => Iterator[InternalRow], + filePartitions: Seq[FilePartition], + readDataSchema: StructType, + metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { + new HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) + } + + override def createOrcFileReader(vectorized: Boolean, + sqlConf: SQLConf, + options: Map[String, String], + hadoopConf: Configuration, + dataSchema: StructType): SparkColumnarFileReader = { + SparkOrcReaderBase.build(vectorized, sqlConf, options, hadoopConf, dataSchema, + (capacity, memoryMode) => new OrcColumnarBatchReader(capacity, memoryMode)) + } + override def createRelation(sqlContext: SQLContext, metaClient: HoodieTableMetaClient, schema: HoodieSchema, @@ -130,6 +149,8 @@ abstract class BaseSpark4Adapter extends SparkAdapter with Logging { override def getUTF8StringFactory: HoodieUTF8StringFactory = Spark4HoodieUTF8StringFactory + override def getSparkPartitionedFileUtils: HoodieSparkPartitionedFileUtils = HoodieSpark4PartitionedFileUtils + override def splitFiles(sparkSession: SparkSession, partitionDirectory: PartitionDirectory, isSplitable: Boolean, diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark4PartitionedFileUtils.scala similarity index 96% rename from hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala rename to hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark4PartitionedFileUtils.scala index d11ec6baee4f3..824ff9f93080f 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark4PartitionedFileUtils.scala @@ -27,9 +27,9 @@ import org.apache.spark.paths.SparkPath import org.apache.spark.sql.catalyst.InternalRow /** - * Utils on Spark [[PartitionedFile]] and [[PartitionDirectory]] for Spark 4.0. + * Utils on Spark [[PartitionedFile]] and [[PartitionDirectory]] for Spark 4.x. */ -object HoodieSpark41PartitionedFileUtils extends HoodieSparkPartitionedFileUtils { +object HoodieSpark4PartitionedFileUtils extends HoodieSparkPartitionedFileUtils { override def getPathFromPartitionedFile(partitionedFile: PartitionedFile): StoragePath = { new StoragePath(partitionedFile.filePath.toUri) } diff --git a/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/Spark4ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/Spark4ResolveHudiAlterTableCommand.scala new file mode 100644 index 0000000000000..73ada28f7ccd4 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/Spark4ResolveHudiAlterTableCommand.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi + +import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{AlterColumns, LogicalPlan} +import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} + +/** + * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. + * for alter table column commands. + */ +class Spark4ResolveHudiAlterTableCommand(sparkSession: SparkSession) + extends BaseResolveHudiAlterTableCommand(sparkSession) { + + override protected def resolveAlterColumnCommand: PartialFunction[LogicalPlan, LogicalPlan] = { + case alter@AlterColumns(ResolvedHoodieV2TablePlan(t), _) if alter.resolved => + HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) + } +} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodieFileScanRDD.scala deleted file mode 100644 index 5e9792a0677d1..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodieFileScanRDD.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.types.StructType - -class Spark40HoodieFileScanRDD(@transient private val sparkSession: SparkSession, - read: PartitionedFile => Iterator[InternalRow], - @transient filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty) - extends FileScanRDD(sparkSession, read, filePartitions, readDataSchema, metadataColumns) - with HoodieUnsafeRDD { - - override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect() -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_0Adapter.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_0Adapter.scala index 7a3d8bec2403c..519ec97cd7719 100644 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_0Adapter.scala +++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_0Adapter.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.{HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, Spark40HoodieFileScanRDD, Spark40HoodiePartitionCDCFileGroupMapping, Spark40HoodiePartitionFileSliceMapping} +import org.apache.hudi.{HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, Spark40HoodiePartitionCDCFileGroupMapping, Spark40HoodiePartitionFileSliceMapping} import org.apache.hudi.client.model.{HoodieInternalRow, Spark40HoodieInternalRow} import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.schema.HoodieSchema @@ -33,7 +33,7 @@ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} +import org.apache.spark.sql.catalyst.expressions.{Expression} import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ @@ -43,7 +43,6 @@ import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.connector.catalog.{V1Table, V2TableWithV1Fallback} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.lance.SparkLanceReaderBase -import org.apache.spark.sql.execution.datasources.orc.Spark40OrcReader import org.apache.spark.sql.execution.datasources.parquet.{HoodieParquetReadSupport, ParquetFileFormat, Spark40HoodieParquetReadSupport, Spark40LegacyHoodieParquetFileFormat, Spark40ParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.execution.streaming.MemoryStream @@ -98,8 +97,6 @@ class Spark4_0Adapter extends BaseSpark4Adapter { override def getSchemaUtils: HoodieSchemaUtils = HoodieSpark40SchemaUtils - override def getSparkPartitionedFileUtils: HoodieSparkPartitionedFileUtils = HoodieSpark40PartitionedFileUtils - override def newParseException(command: Option[String], exception: AnalysisException, start: Origin, @@ -136,14 +133,6 @@ class Spark4_0Adapter extends BaseSpark4Adapter { new Spark40HoodiePartitionFileSliceMapping(values, slices) } - override def createHoodieFileScanRDD(sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { - new Spark40HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) - } - override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition } @@ -209,23 +198,6 @@ class Spark4_0Adapter extends BaseSpark4Adapter { datetimeRebaseSpec, getRebaseSpec("LEGACY"), tableSchemaOpt) } - /** - * TODO - * - * @param vectorized - * @param sqlConf - * @param options - * @param hadoopConf - * @return - */ - override def createOrcFileReader(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): SparkColumnarFileReader = { - Spark40OrcReader.build(vectorized, sqlConf, options, hadoopConf, dataSchema) - } - override def createLanceFileReader(vectorized: Boolean, sqlConf: SQLConf, options: Map[String, String], diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark40PartitionedFileUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark40PartitionedFileUtils.scala deleted file mode 100644 index bc83633383a91..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark40PartitionedFileUtils.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.common.util.ReflectionUtils -import org.apache.hudi.storage.StoragePath - -import org.apache.hadoop.fs.FileStatus -import org.apache.spark.paths.SparkPath -import org.apache.spark.sql.catalyst.InternalRow - -/** - * Utils on Spark [[PartitionedFile]] and [[PartitionDirectory]] for Spark 4.0. - */ -object HoodieSpark40PartitionedFileUtils extends HoodieSparkPartitionedFileUtils { - override def getPathFromPartitionedFile(partitionedFile: PartitionedFile): StoragePath = { - new StoragePath(partitionedFile.filePath.toUri) - } - - override def getStringPathFromPartitionedFile(partitionedFile: PartitionedFile): String = { - partitionedFile.filePath.toPath.toString - } - - override def createPartitionedFile(partitionValues: InternalRow, - filePath: StoragePath, - start: Long, - length: Long): PartitionedFile = { - PartitionedFile(partitionValues, SparkPath.fromUri(filePath.toUri), start, length, Array.empty) - } - - override def toFileStatuses(partitionDirs: Seq[PartitionDirectory]): Seq[FileStatus] = { - val files: Seq[FileStatusWithMetadata] = partitionDirs.flatMap(_.files) - try { - files.map(_.fileStatus) - } catch { - case _: NoSuchMethodException | _: NoSuchMethodError | _: IllegalArgumentException => - val methodOpt = ReflectionUtils.getMethod(classOf[FileStatusWithMetadata], "toFileStatus") - if (methodOpt.isPresent) { - val method = methodOpt.get() - files.map(f => method.invoke(f).asInstanceOf[FileStatus]) - } else { - throw new RuntimeException( - "Cannot find toFileStatus method on FileStatusWithMetadata in custom Spark Runtime") - } - } - } - - override def newPartitionDirectory(internalRow: InternalRow, statuses: Seq[FileStatus]): PartitionDirectory = { - PartitionDirectory(internalRow, statuses.toArray) - } -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark40NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark40NestedSchemaPruning.scala deleted file mode 100644 index c10989d89d9fb..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark40NestedSchemaPruning.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.HoodieBaseRelation - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.types.StructType - -class Spark40NestedSchemaPruning extends BaseHoodieNestedSchemaPruning { - - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], - requiredSchema: StructType): Seq[AttributeReference] = { - // We need to replace the expression ids of the pruned relation output attributes - // with the expression ids of the original relation output attributes so that - // references to the original relation's output are not broken - val outputIdMap = output.map(att => (att.name, att.exprId)).toMap - DataTypeUtils.toAttributes(requiredSchema) - .map { - case att if outputIdMap.contains(att.name) => - att.withExprId(outputIdMap(att.name)) - case att => att - } - } - - override protected def apply0(plan: LogicalPlan): LogicalPlan = - plan transformDown { - case op @ PhysicalOperation(projects, filters, - // NOTE: This is modified to accommodate for Hudi's custom relations, given that original - // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]] - // TODO generalize to any file-based relation - l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _, _)) - if relation.canPruneRelationSchema => - - prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, - prunedDataSchema => { - val prunedRelation = - relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) - buildPrunedRelation(l, prunedRelation) - }).getOrElse(op) - } -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark40OrcReader.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark40OrcReader.scala deleted file mode 100644 index d1da60f5bf085..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark40OrcReader.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.memory.MemoryMode -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes -import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType - -class Spark40OrcReader(enableVectorizedReader: Boolean, - memoryMode: MemoryMode, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean, - capacity: Int) extends SparkOrcReaderBase(enableVectorizedReader, dataSchema, orcFilterPushDown, isCaseSensitive) { - - override def partitionedFileToPath(file: PartitionedFile): Path = { - file.toPath - } - - override def buildReader(): OrcColumnarBatchReader = { - new OrcColumnarBatchReader(capacity, memoryMode) - } - - override def structTypeToAttributes(schema: StructType): Seq[Attribute] = { - toAttributes(schema) - } -} - -object Spark40OrcReader { - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @return ORC file reader - */ - def build(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): Spark40OrcReader = { - //set hadoopconf - hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) - hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) - - val memoryMode = if (sqlConf.offHeapColumnVectorEnabled) { - MemoryMode.OFF_HEAP - } else { - MemoryMode.ON_HEAP - } - - val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && - options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, - throw new IllegalArgumentException( - "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + - "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) - .equals("true") - - new Spark40OrcReader( - enableVectorizedReader = enableVectorizedReader && vectorized, - memoryMode = memoryMode, - isCaseSensitive = sqlConf.caseSensitiveAnalysis, - capacity = sqlConf.orcVectorizedReaderBatchSize, - orcFilterPushDown = sqlConf.orcFilterPushDown, - dataSchema = dataSchema) - } -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40DataSourceUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40DataSourceUtils.scala deleted file mode 100644 index 3b84a3e164be1..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40DataSourceUtils.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY -import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} -import org.apache.spark.util.Utils - -object Spark40DataSourceUtils { - - /** - * NOTE: This method was copied from [[Spark32PlusDataSourceUtils]], and is required to maintain runtime - * compatibility against Spark 3.5.0 - */ - // scalastyle:off - def int96RebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to - // rebase the INT96 timestamp values. - // Files written by Spark 3.1 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.1.0" || lookupFileMeta("org.apache.spark.legacyINT96") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def datetimeRebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to - // rebase the datetime values. - // Files written by Spark 3.0 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.0.0" || lookupFileMeta("org.apache.spark.legacyDateTime") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - -} diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark40ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark40ResolveHudiAlterTableCommand.scala deleted file mode 100644 index e64dae370d4eb..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark40ResolveHudiAlterTableCommand.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hudi - -import org.apache.hudi.common.config.HoodieCommonConfig -import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.ResolvedTable -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table -import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} - -/** - * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. - * for alter table column commands. - */ -class Spark40ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { - - def apply(plan: LogicalPlan): LogicalPlan = { - if (ProvidesHoodieConfig.isSchemaEvolutionEnabled(sparkSession)) { - plan.resolveOperatorsUp { - case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => - HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) - case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => - HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) - case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => - HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) - case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => - HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) - case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => - HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) - case alter@AlterColumns(ResolvedHoodieV2TablePlan(t), _) if alter.resolved => - HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) - case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => - HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) - } - } else { - plan - } - } - - object ResolvedHoodieV2TablePlan { - def unapply(plan: LogicalPlan): Option[HoodieInternalV2Table] = { - plan match { - case ResolvedTable(_, _, v2Table: HoodieInternalV2Table, _) => Some(v2Table) - case _ => None - } - } - } -} - diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodieFileScanRDD.scala deleted file mode 100644 index 4b27188b51c13..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodieFileScanRDD.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.types.StructType - -class Spark41HoodieFileScanRDD(@transient private val sparkSession: SparkSession, - read: PartitionedFile => Iterator[InternalRow], - @transient filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty) - extends FileScanRDD(sparkSession, read, filePartitions, readDataSchema, metadataColumns) - with HoodieUnsafeRDD { - - override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect() -} diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala index 195979548bc4f..f6a8e19b0e6c8 100644 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala +++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.adapter -import org.apache.hudi.{HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, Spark41HoodieFileScanRDD, Spark41HoodiePartitionCDCFileGroupMapping, Spark41HoodiePartitionFileSliceMapping} +import org.apache.hudi.{HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, Spark41HoodiePartitionCDCFileGroupMapping, Spark41HoodiePartitionFileSliceMapping} import org.apache.hudi.client.model.{HoodieInternalRow, Spark41HoodieInternalRow} import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.schema.HoodieSchema @@ -32,7 +32,7 @@ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, CreateNamedStruct, Expression, Literal, UnsafeProjection} +import org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStruct, Expression, Literal, UnsafeProjection} import org.apache.spark.sql.catalyst.expressions.variant.VariantGet import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} import org.apache.spark.sql.catalyst.planning.PhysicalOperation @@ -43,7 +43,6 @@ import org.apache.spark.sql.catalyst.util.{METADATA_COL_ATTR_KEY, RebaseDateTime import org.apache.spark.sql.connector.catalog.{V1Table, V2TableWithV1Fallback} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.lance.SparkLanceReaderBase -import org.apache.spark.sql.execution.datasources.orc.Spark41OrcReader import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, Spark41LegacyHoodieParquetFileFormat, Spark41ParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.execution.streaming.runtime.MemoryStream @@ -98,8 +97,6 @@ class Spark4_1Adapter extends BaseSpark4Adapter { override def getSchemaUtils: HoodieSchemaUtils = HoodieSpark41SchemaUtils - override def getSparkPartitionedFileUtils: HoodieSparkPartitionedFileUtils = HoodieSpark41PartitionedFileUtils - override def newParseException(command: Option[String], exception: AnalysisException, start: Origin, @@ -138,14 +135,6 @@ class Spark4_1Adapter extends BaseSpark4Adapter { new Spark41HoodiePartitionFileSliceMapping(values, slices) } - override def createHoodieFileScanRDD(sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - readDataSchema: StructType, - metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = { - new Spark41HoodieFileScanRDD(sparkSession, readFunction, filePartitions, readDataSchema, metadataColumns) - } - override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition } @@ -200,24 +189,6 @@ class Spark4_1Adapter extends BaseSpark4Adapter { Spark41ParquetReader.build(vectorized, sqlConf, options, hadoopConf) } - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @param dataSchema the data schema of the ORC file - * @return ORC file reader - */ - override def createOrcFileReader(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): SparkColumnarFileReader = { - Spark41OrcReader.build(vectorized, sqlConf, options, hadoopConf, dataSchema) - } - override def createLanceFileReader(vectorized: Boolean, sqlConf: SQLConf, options: Map[String, String], diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala deleted file mode 100644 index e9cac2a66ac23..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources - -import org.apache.hudi.HoodieBaseRelation - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.types.StructType - -class Spark41NestedSchemaPruning extends BaseHoodieNestedSchemaPruning { - - // Prune the given output to make it consistent with `requiredSchema`. - protected def getPrunedOutput(output: Seq[AttributeReference], - requiredSchema: StructType): Seq[AttributeReference] = { - // We need to replace the expression ids of the pruned relation output attributes - // with the expression ids of the original relation output attributes so that - // references to the original relation's output are not broken - val outputIdMap = output.map(att => (att.name, att.exprId)).toMap - DataTypeUtils.toAttributes(requiredSchema) - .map { - case att if outputIdMap.contains(att.name) => - att.withExprId(outputIdMap(att.name)) - case att => att - } - } - - override protected def apply0(plan: LogicalPlan): LogicalPlan = - plan transformDown { - case op@PhysicalOperation(projects, filters, - // NOTE: This is modified to accommodate for Hudi's custom relations, given that original - // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]] - // TODO generalize to any file-based relation - l@LogicalRelation(relation: HoodieBaseRelation, _, _, _, _)) - if relation.canPruneRelationSchema => - - prunePhysicalColumns(l.output, projects, filters, relation.dataSchema, - prunedDataSchema => { - val prunedRelation = - relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema) - buildPrunedRelation(l, prunedRelation) - }).getOrElse(op) - } -} diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark41OrcReader.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark41OrcReader.scala deleted file mode 100644 index 413196a236a49..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark41OrcReader.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.memory.MemoryMode -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes -import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType - -class Spark41OrcReader(enableVectorizedReader: Boolean, - memoryMode: MemoryMode, - dataSchema: StructType, - orcFilterPushDown: Boolean, - isCaseSensitive: Boolean, - capacity: Int) extends SparkOrcReaderBase(enableVectorizedReader, dataSchema, orcFilterPushDown, isCaseSensitive) { - - override def partitionedFileToPath(file: PartitionedFile): Path = { - file.toPath - } - - override def buildReader(): OrcColumnarBatchReader = { - new OrcColumnarBatchReader(capacity, memoryMode) - } - - override def structTypeToAttributes(schema: StructType): Seq[Attribute] = { - toAttributes(schema) - } -} - -object Spark41OrcReader { - /** - * Get ORC file reader - * - * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc - * @param sqlConf the [[SQLConf]] used for the read - * @param options passed as a param to the file format - * @param hadoopConf some configs will be set for the hadoopConf - * @return ORC file reader - */ - def build(vectorized: Boolean, - sqlConf: SQLConf, - options: Map[String, String], - hadoopConf: Configuration, - dataSchema: StructType): Spark41OrcReader = { - //set hadoopconf - hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone) - hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis) - - val memoryMode = if (sqlConf.offHeapColumnVectorEnabled) { - MemoryMode.OFF_HEAP - } else { - MemoryMode.ON_HEAP - } - - val enableVectorizedReader = sqlConf.orcVectorizedReaderEnabled && - options.getOrElse(FileFormat.OPTION_RETURNING_BATCH, - throw new IllegalArgumentException( - "OPTION_RETURNING_BATCH should always be set for OrcFileFormat. " + - "To workaround this issue, set spark.sql.orc.enableVectorizedReader=false.")) - .equals("true") - - new Spark41OrcReader( - enableVectorizedReader = enableVectorizedReader && vectorized, - memoryMode = memoryMode, - isCaseSensitive = sqlConf.caseSensitiveAnalysis, - capacity = sqlConf.orcVectorizedReaderBatchSize, - orcFilterPushDown = sqlConf.orcFilterPushDown, - dataSchema = dataSchema) - } -} diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala deleted file mode 100644 index 9c4b28d2423a5..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY -import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} -import org.apache.spark.util.Utils - -object Spark41DataSourceUtils { - - /** - * NOTE: This method was copied from [[Spark32PlusDataSourceUtils]], and is required to maintain runtime - * compatibility against Spark 3.5.0 - */ - // scalastyle:off - def int96RebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to - // rebase the INT96 timestamp values. - // Files written by Spark 3.1 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.1.0" || lookupFileMeta("org.apache.spark.legacyINT96") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - - /** - * NOTE: This method was copied from Spark 3.2.0, and is required to maintain runtime - * compatibility against Spark 3.2.0 - */ - // scalastyle:off - def datetimeRebaseMode(lookupFileMeta: String => String, - modeByConfig: String): LegacyBehaviorPolicy.Value = { - if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") { - return LegacyBehaviorPolicy.CORRECTED - } - // If there is no version, we return the mode specified by the config. - Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version => - // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to - // rebase the datetime values. - // Files written by Spark 3.0 and latter may also need the rebase if they were written with - // the "LEGACY" rebase mode. - if (version < "3.0.0" || lookupFileMeta("org.apache.spark.legacyDateTime") != null) { - LegacyBehaviorPolicy.LEGACY - } else { - LegacyBehaviorPolicy.CORRECTED - } - }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig)) - } - // scalastyle:on - -} diff --git a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala deleted file mode 100644 index 92cbc2803bf6b..0000000000000 --- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hudi - -import org.apache.hudi.common.config.HoodieCommonConfig -import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.ResolvedTable -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table -import org.apache.spark.sql.hudi.command.{AlterTableCommand => HudiAlterTableCommand} - -/** - * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity. - * for alter table column commands. - */ -class Spark41ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] { - - def apply(plan: LogicalPlan): LogicalPlan = { - if (ProvidesHoodieConfig.isSchemaEvolutionEnabled(sparkSession)) { - plan.resolveOperatorsUp { - case set@SetTableProperties(ResolvedHoodieV2TablePlan(t), _) if set.resolved => - HudiAlterTableCommand(t.v1Table, set.changes, ColumnChangeID.PROPERTY_CHANGE) - case unSet@UnsetTableProperties(ResolvedHoodieV2TablePlan(t), _, _) if unSet.resolved => - HudiAlterTableCommand(t.v1Table, unSet.changes, ColumnChangeID.PROPERTY_CHANGE) - case drop@DropColumns(ResolvedHoodieV2TablePlan(t), _, _) if drop.resolved => - HudiAlterTableCommand(t.v1Table, drop.changes, ColumnChangeID.DELETE) - case add@AddColumns(ResolvedHoodieV2TablePlan(t), _) if add.resolved => - HudiAlterTableCommand(t.v1Table, add.changes, ColumnChangeID.ADD) - case renameColumn@RenameColumn(ResolvedHoodieV2TablePlan(t), _, _) if renameColumn.resolved => - HudiAlterTableCommand(t.v1Table, renameColumn.changes, ColumnChangeID.UPDATE) - case alter@AlterColumns(ResolvedHoodieV2TablePlan(t), _) if alter.resolved => - HudiAlterTableCommand(t.v1Table, alter.changes, ColumnChangeID.UPDATE) - case replace@ReplaceColumns(ResolvedHoodieV2TablePlan(t), _) if replace.resolved => - HudiAlterTableCommand(t.v1Table, replace.changes, ColumnChangeID.REPLACE) - } - } else { - plan - } - } - - object ResolvedHoodieV2TablePlan { - def unapply(plan: LogicalPlan): Option[HoodieInternalV2Table] = { - plan match { - case ResolvedTable(_, _, v2Table: HoodieInternalV2Table, _) => Some(v2Table) - case _ => None - } - } - } -} - From 49736bf3bc480fb5e526fb242fc06c7a5700e93b Mon Sep 17 00:00:00 2001 From: voonhous Date: Fri, 3 Jul 2026 18:51:55 +0800 Subject: [PATCH 122/255] refactor: Retire leftover Avro Schema usages in AvroSchemaUtils and LSMTimelineWriter (#19153) * refactor(common): Remove dead AvroSchemaUtils.createSchemaErrorString overload The Avro-typed createSchemaErrorString(String, Schema, Schema) has no callers in main or test; all sites use the HoodieSchema twin HoodieSchemaUtils.createSchemaErrorString(String, HoodieSchema, HoodieSchema). Retire the legacy Avro overload. Part of the Avro Schema -> HoodieSchema migration (#14263). * refactor(client): Drop redundant Avro Schema local in LSMTimelineWriter Inline HoodieSchema.fromAvroSchema(HoodieLSMTimelineInstant.getClassSchema()) as the sibling write paths already do (previously an intermediate Avro Schema local was wrapped a line later), removing the local and the now-unused org.apache.avro.Schema import. Log output is unchanged: HoodieSchema.toString() delegates to the wrapped Avro schema. Incidental (editor auto-format): two concatenated log statements in this file are normalized to parameterized slf4j form. Part of the Avro Schema -> HoodieSchema migration (#14263). (cherry picked from commit e38e043de9e59f5b455defb6739802e2eb94f15b) --- .../timeline/versioning/v2/LSMTimelineWriter.java | 10 ++++------ .../java/org/apache/hudi/avro/AvroSchemaUtils.java | 4 ---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java index b555a0646183c..97b2dc01d3acc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java @@ -49,7 +49,6 @@ import org.apache.hudi.table.HoodieTable; import lombok.extern.slf4j.Slf4j; -import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import java.io.IOException; @@ -137,9 +136,8 @@ public void write( throw new HoodieIOException("Failed to check archiving file before write: " + filePath, ioe); } try (HoodieFileWriter writer = openWriter(filePath)) { - Schema wrapperSchema = HoodieLSMTimelineInstant.getClassSchema(); - log.info("Writing schema " + wrapperSchema.toString()); - HoodieSchema schema = HoodieSchema.fromAvroSchema(wrapperSchema); + HoodieSchema schema = HoodieSchema.fromAvroSchema(HoodieLSMTimelineInstant.getClassSchema()); + log.info("Writing schema {}", schema); for (ActiveAction activeAction : activeActions) { try { preWriteCallback.ifPresent(callback -> callback.accept(activeAction)); @@ -147,7 +145,7 @@ public void write( final HoodieLSMTimelineInstant metaEntry = MetadataConversionUtils.createLSMTimelineInstant(activeAction, metaClient); writer.write(metaEntry.getInstantTime(), new HoodieAvroIndexedRecord(metaEntry), schema); } catch (Exception e) { - log.error("Failed to write instant: " + activeAction.getInstantTime(), e); + log.error("Failed to write instant: {}", activeAction.getInstantTime(), e); exceptionHandler.ifPresent(handler -> handler.accept(e)); } } @@ -290,7 +288,7 @@ private Option doCompact(HoodieLSMTimelineManifest manifest, int layer) compactFiles(candidateFiles, compactedFileName); // 4. update the manifest file updateManifest(candidateFiles, compactedFileName); - log.info("Finishes compaction of source files: " + candidateFiles); + log.info("Finishes compaction of source files: {}", candidateFiles); return Option.of(compactedFileName); } return Option.empty(); diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java index 90a8e9274b8aa..78ffcf5723129 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java @@ -114,10 +114,6 @@ public static Schema createNullableSchema(Schema schema) { return Schema.createUnion(Schema.create(Schema.Type.NULL), schema); } - public static String createSchemaErrorString(String errorMessage, Schema writerSchema, Schema tableSchema) { - return String.format("%s\nwriterSchema: %s\ntableSchema: %s", errorMessage, writerSchema, tableSchema); - } - /** * Create a new schema by force changing all the fields as nullable. * From a91e4443bcfd9464fbfecf036ee01420e2a9a950 Mon Sep 17 00:00:00 2001 From: voonhous Date: Fri, 3 Jul 2026 18:52:22 +0800 Subject: [PATCH 123/255] refactor: Migrate reconcileSchema/reconcileSchemaRequirements to HoodieSchema (#19154) The reconcile adapters in AvroSchemaEvolutionUtils took Avro Schema in/out and immediately wrapped inputs via HoodieSchema.fromAvroSchema, while every caller already held a HoodieSchema and passed .toAvroSchema(). Migrate the three signatures to HoodieSchema: - reconcileSchema(Schema, InternalSchema) -> (HoodieSchema, InternalSchema) - reconcileSchema(Schema, Schema): Schema -> (HoodieSchema, HoodieSchema): HoodieSchema - reconcileSchemaRequirements(Schema, Schema): Schema -> HoodieSchema in/out Callers (BaseHoodieWriteClient, FileGroupReaderBasedMergeHandle, HoodieMergeHelper, HoodieSchemaUtils.scala x3, and the unit test) drop the .toAvroSchema()/.getAvroSchema() round-trips; the InternalSchema convert(...) boundary is unchanged. Drops the org.apache.avro.Schema import from AvroSchemaEvolutionUtils. Part of the Avro Schema -> HoodieSchema migration (#14263). (cherry picked from commit 369ca5cb3596c1efe8fd8c2ea4ab922fda21bdf1) --- .../hudi/client/BaseHoodieWriteClient.java | 2 +- .../io/FileGroupReaderBasedMergeHandle.java | 2 +- .../action/commit/HoodieMergeHelper.java | 2 +- .../utils/AvroSchemaEvolutionUtils.java | 26 +++++++++---------- .../utils/TestAvroSchemaEvolutionUtils.java | 6 ++--- .../org/apache/hudi/HoodieSchemaUtils.scala | 8 +++--- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 8284f61baa15f..f5d050d880986 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -369,7 +369,7 @@ private void saveInternalSchema(HoodieTable table, String instantTime, HoodieCom internalSchema = InternalSchemaUtils.searchSchema(Long.parseLong(instantTime), SerDeHelper.parseSchemas(historySchemaStr)); } - InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(schema.toAvroSchema(), internalSchema, config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS)); + InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(schema, internalSchema, config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS)); if (evolvedSchema.equals(internalSchema)) { metadata.addMetadata(SerDeHelper.LATEST_SCHEMA, SerDeHelper.toJson(evolvedSchema)); //TODO save history schema by metaTable diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java index 2893913a0d13a..ef0d7e08dc31f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java @@ -257,7 +257,7 @@ public void doMerge() { } boolean usePosition = config.getBooleanOrDefault(MERGE_USE_RECORD_POSITIONS); Option internalSchemaOption = SerDeHelper.fromJson(config.getInternalSchema()) - .map(internalSchema -> AvroSchemaEvolutionUtils.reconcileSchema(writeSchemaWithMetaFields.toAvroSchema(), internalSchema, + .map(internalSchema -> AvroSchemaEvolutionUtils.reconcileSchema(writeSchemaWithMetaFields, internalSchema, config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS))); long maxMemoryPerCompaction = getMaxMemoryForMerge(); props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java index f887e4d5ac746..2f571e0d3e2ba 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java @@ -170,7 +170,7 @@ private Option> composeSchemaEvolutionTrans // TODO support bootstrap if (querySchemaOpt.isPresent() && !baseFile.getBootstrapBaseFile().isPresent()) { // check implicitly add columns, and position reorder(spark sql may change cols order) - InternalSchema querySchema = AvroSchemaEvolutionUtils.reconcileSchema(writerSchema.toAvroSchema(), + InternalSchema querySchema = AvroSchemaEvolutionUtils.reconcileSchema(writerSchema, querySchemaOpt.get(), writeConfig.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS)); long commitInstantTime = Long.parseLong(baseFile.getCommitTime()); InternalSchema fileSchema = InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, metaClient); diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java index 5ae6f203f0f3b..c31b5c057181a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java @@ -24,8 +24,6 @@ import org.apache.hudi.internal.schema.action.TableChanges; import org.apache.hudi.internal.schema.action.TableChangesHelper; -import org.apache.avro.Schema; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -64,12 +62,12 @@ public class AvroSchemaEvolutionUtils { * nullable in the result. Otherwise, no updates will be made to those fields. * @return reconcile Schema */ - public static InternalSchema reconcileSchema(Schema incomingSchema, InternalSchema oldTableSchema, boolean makeMissingFieldsNullable) { + public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, InternalSchema oldTableSchema, boolean makeMissingFieldsNullable) { /* If incoming schema is null, we fall back on table schema. */ - if (incomingSchema.getType() == Schema.Type.NULL) { + if (incomingSchema.isSchemaNull()) { return oldTableSchema; } - InternalSchema inComingInternalSchema = convert(HoodieSchema.fromAvroSchema(incomingSchema), oldTableSchema.getNameToPosition()); + InternalSchema inComingInternalSchema = convert(incomingSchema, oldTableSchema.getNameToPosition()); // check column add/missing List colNamesFromIncoming = inComingInternalSchema.getAllColsFullName(); List colNamesFromOldSchema = oldTableSchema.getAllColsFullName(); @@ -149,8 +147,8 @@ public static InternalSchema reconcileSchema(Schema incomingSchema, InternalSche return evolvedSchema; } - public static Schema reconcileSchema(Schema incomingSchema, Schema oldTableSchema, boolean makeMissingFieldsNullable) { - return convert(reconcileSchema(incomingSchema, convert(HoodieSchema.fromAvroSchema(oldTableSchema)), makeMissingFieldsNullable), oldTableSchema.getFullName()).toAvroSchema(); + public static HoodieSchema reconcileSchema(HoodieSchema incomingSchema, HoodieSchema oldTableSchema, boolean makeMissingFieldsNullable) { + return convert(reconcileSchema(incomingSchema, convert(oldTableSchema), makeMissingFieldsNullable), oldTableSchema.getFullName()); } /** @@ -169,18 +167,18 @@ public static Schema reconcileSchema(Schema incomingSchema, Schema oldTableSchem * @param targetSchema target schema that source schema will be reconciled against * @return schema (based off {@code source} one) that has nullability constraints and datatypes reconciled */ - public static Schema reconcileSchemaRequirements(Schema sourceSchema, Schema targetSchema, boolean shouldReorderColumns) { - if (targetSchema.getType() == Schema.Type.NULL || targetSchema.getFields().isEmpty()) { + public static HoodieSchema reconcileSchemaRequirements(HoodieSchema sourceSchema, HoodieSchema targetSchema, boolean shouldReorderColumns) { + if (targetSchema.isSchemaNull() || targetSchema.getFields().isEmpty()) { return sourceSchema; } - if (sourceSchema == null || sourceSchema.getType() == Schema.Type.NULL || sourceSchema.getFields().isEmpty()) { + if (sourceSchema == null || sourceSchema.isSchemaNull() || sourceSchema.getFields().isEmpty()) { return targetSchema; } - InternalSchema targetInternalSchema = convert(HoodieSchema.fromAvroSchema(targetSchema)); + InternalSchema targetInternalSchema = convert(targetSchema); // Use existing fieldIds for consistent field ordering between commits when shouldReorderColumns is true - InternalSchema sourceInternalSchema = convert(HoodieSchema.fromAvroSchema(sourceSchema), shouldReorderColumns ? targetInternalSchema.getNameToPosition() : Collections.emptyMap()); + InternalSchema sourceInternalSchema = convert(sourceSchema, shouldReorderColumns ? targetInternalSchema.getNameToPosition() : Collections.emptyMap()); List colNamesSourceSchema = sourceInternalSchema.getAllColsFullName(); List colNamesTargetSchema = targetInternalSchema.getAllColsFullName(); @@ -200,7 +198,7 @@ public static Schema reconcileSchemaRequirements(Schema sourceSchema, Schema tar if (nullableUpdateColsInSource.isEmpty() && typeUpdateColsInSource.isEmpty()) { //standardize order of unions - return convert(sourceInternalSchema, sourceSchema.getFullName()).toAvroSchema(); + return convert(sourceInternalSchema, sourceSchema.getFullName()); } TableChanges.ColumnUpdateChange schemaChange = TableChanges.ColumnUpdateChange.get(sourceInternalSchema); @@ -218,7 +216,7 @@ public static Schema reconcileSchemaRequirements(Schema sourceSchema, Schema tar } - return convert(SchemaChangeUtils.applyTableChanges2Schema(sourceInternalSchema, schemaChange), sourceSchema.getFullName()).toAvroSchema(); + return convert(SchemaChangeUtils.applyTableChanges2Schema(sourceInternalSchema, schemaChange), sourceSchema.getFullName()); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java index 662436de9d757..19d4205d35561 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java @@ -486,7 +486,7 @@ public void testEvolutionSchemaFromNewAvroSchema() { ); evolvedRecord = (Types.RecordType)InternalSchemaBuilder.getBuilder().refreshNewId(evolvedRecord, new AtomicInteger(0)); HoodieSchema evolvedSchema = InternalSchemaConverter.convert(evolvedRecord, "test1"); - InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(evolvedSchema.getAvroSchema(), oldSchema, false); + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(evolvedSchema, oldSchema, false); Types.RecordType checkedRecord = Types.RecordType.get( Types.Field.get(0, false, "id", Types.IntType.get()), Types.Field.get(1, true, "data", Types.StringType.get()), @@ -541,7 +541,7 @@ public void testReconcileSchema() { + "{\"name\":\"d2\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null}]}"); HoodieSchema simpleReconcileSchema = InternalSchemaConverter.convert(AvroSchemaEvolutionUtils - .reconcileSchema(incomingSchema.getAvroSchema(), InternalSchemaConverter.convert(schema), false), "schemaNameFallback"); + .reconcileSchema(incomingSchema, InternalSchemaConverter.convert(schema), false), "schemaNameFallback"); Assertions.assertEquals(simpleCheckSchema, simpleReconcileSchema); } @@ -563,7 +563,7 @@ public void testNotEvolveSchemaIfReconciledSchemaUnchanged() { InternalSchema oldInternalSchema = InternalSchemaConverter.convert(oldSchema); // set a non-default schema id for old table schema, e.g., 2. oldInternalSchema.setSchemaId(2); - InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema.getAvroSchema(), oldInternalSchema, false); + InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldInternalSchema, false); // the evolved schema should be the old table schema, since there is no type change at all. Assertions.assertEquals(oldInternalSchema, evolvedSchema); } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala index 5a40f45c12412..6c353481595ea 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala @@ -173,7 +173,7 @@ object HoodieSchemaUtils { if (!mergeIntoWrites && !shouldValidateSchemasCompatibility && !allowAutoEvolutionColumnDrop) { // Default behaviour val reconciledSchema = if (setNullForMissingColumns) { - HoodieSchema.fromAvroSchema(AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema.toAvroSchema(), latestTableSchema.toAvroSchema(), setNullForMissingColumns)) + AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, latestTableSchema, setNullForMissingColumns) } else { canonicalizedSourceSchema } @@ -205,7 +205,7 @@ object HoodieSchemaUtils { // Apply schema evolution, by auto-merging write schema and read schema val setNullForMissingColumns = opts.getOrElse(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key(), HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.defaultValue()).toBoolean - val mergedInternalSchema = AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema.toAvroSchema(), internalSchema, setNullForMissingColumns) + val mergedInternalSchema = AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, internalSchema, setNullForMissingColumns) val evolvedSchema = InternalSchemaConverter.convert(mergedInternalSchema, latestTableSchema.getFullName) val shouldRemoveMetaDataFromInternalSchema = sourceSchema.getFields.asScala.filter(f => f.name().equalsIgnoreCase(HoodieRecord.RECORD_KEY_METADATA_FIELD)).isEmpty if (shouldRemoveMetaDataFromInternalSchema) HoodieCommonSchemaUtils.removeMetadataFields(evolvedSchema) else evolvedSchema @@ -257,9 +257,7 @@ object HoodieSchemaUtils { */ private def canonicalizeSchema(sourceSchema: HoodieSchema, latestTableSchema: HoodieSchema, opts : Map[String, String], shouldReorderColumns: Boolean): HoodieSchema = { - HoodieSchema.fromAvroSchema( - reconcileSchemaRequirements(sourceSchema.toAvroSchema(), latestTableSchema.toAvroSchema(), shouldReorderColumns) - ) + reconcileSchemaRequirements(sourceSchema, latestTableSchema, shouldReorderColumns) } From 5906248205cf4ff7c627e85ba370cab334068700 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Fri, 3 Jul 2026 06:42:52 -0700 Subject: [PATCH 124/255] perf(common): replace BitSet with a fixed word array in the ported bloom filter (#19140) * perf(common): replace BitSet with a fixed word array in the ported bloom filter * Add manual microbenchmark for bloom filter hot paths (cherry picked from commit 3697001e8485b5aea1e759bca994016b63ae88af) --- .../common/bloom/InternalBloomFilter.java | 89 ++++--- .../bloom/InternalBloomFilterBenchmark.java | 127 ++++++++++ .../common/bloom/TestInternalBloomFilter.java | 234 ++++++++++++++++++ 3 files changed, 409 insertions(+), 41 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bloom/InternalBloomFilterBenchmark.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bloom/TestInternalBloomFilter.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalBloomFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalBloomFilter.java index 7ef766a2a3c5a..af45cb0245c62 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalBloomFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalBloomFilter.java @@ -82,21 +82,15 @@ * @see Space/Time Trade-Offs in Hash Coding with Allowable Errors */ public class InternalBloomFilter extends InternalFilter { - private static final byte[] BIT_VALUES = new byte[] { - (byte) 0x01, - (byte) 0x02, - (byte) 0x04, - (byte) 0x08, - (byte) 0x10, - (byte) 0x20, - (byte) 0x40, - (byte) 0x80 - }; - /** - * The bit vector. + * The bit vector, as little-endian 64-bit words: bit {@code i} lives at + * {@code words[i >> 6]} under mask {@code 1L << (i & 63)}. The serialized layout + * (bit {@code i} at byte {@code i >> 3} under mask {@code 1 << (i & 7)}) is the + * little-endian byte view of this array, so {@link #write} and {@link #readFields} + * translate between the two by byte position alone. Bits at positions greater than + * or equal to {@code vectorSize} are always zero. */ - BitSet bits; + long[] words; /** * Default constructor - use with readFields @@ -116,7 +110,7 @@ public InternalBloomFilter() { public InternalBloomFilter(int vectorSize, int nbHash, int hashType) { super(vectorSize, nbHash, hashType); - bits = new BitSet(this.vectorSize); + words = new long[wordCount(this.vectorSize)]; } /** @@ -134,7 +128,7 @@ public void add(Key key) { hash.clear(); for (int i = 0; i < nbHash; i++) { - bits.set(h[i]); + words[h[i] >>> 6] |= 1L << (h[i] & 63); } } @@ -147,7 +141,10 @@ public void and(InternalFilter filter) { throw new IllegalArgumentException("filters cannot be and-ed"); } - this.bits.and(((InternalBloomFilter) filter).bits); + long[] other = ((InternalBloomFilter) filter).words; + for (int i = 0; i < words.length; i++) { + words[i] &= other[i]; + } } @Override @@ -159,7 +156,7 @@ public boolean membershipTest(Key key) { int[] h = hash.hash(key); hash.clear(); for (int i = 0; i < nbHash; i++) { - if (!bits.get(h[i])) { + if ((words[h[i] >>> 6] & (1L << (h[i] & 63))) == 0) { return false; } } @@ -168,7 +165,10 @@ public boolean membershipTest(Key key) { @Override public void not() { - bits.flip(0, vectorSize); + for (int i = 0; i < words.length; i++) { + words[i] = ~words[i]; + } + clearUnusedBits(); } @Override @@ -179,7 +179,10 @@ public void or(InternalFilter filter) { || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } - bits.or(((InternalBloomFilter) filter).bits); + long[] other = ((InternalBloomFilter) filter).words; + for (int i = 0; i < words.length; i++) { + words[i] |= other[i]; + } } @Override @@ -190,12 +193,15 @@ public void xor(InternalFilter filter) { || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be xor-ed"); } - bits.xor(((InternalBloomFilter) filter).bits); + long[] other = ((InternalBloomFilter) filter).words; + for (int i = 0; i < words.length; i++) { + words[i] ^= other[i]; + } } @Override public String toString() { - return bits.toString(); + return BitSet.valueOf(words).toString(); } /** @@ -209,17 +215,8 @@ public int getVectorSize() { public void write(DataOutput out) throws IOException { super.write(out); byte[] bytes = new byte[getNBytes()]; - for (int i = 0, byteIndex = 0, bitIndex = 0; i < vectorSize; i++, bitIndex++) { - if (bitIndex == 8) { - bitIndex = 0; - byteIndex++; - } - if (bitIndex == 0) { - bytes[byteIndex] = 0; - } - if (bits.get(i)) { - bytes[byteIndex] |= BIT_VALUES[bitIndex]; - } + for (int byteIndex = 0; byteIndex < bytes.length; byteIndex++) { + bytes[byteIndex] = (byte) (words[byteIndex >>> 3] >>> ((byteIndex & 7) << 3)); } out.write(bytes); } @@ -227,22 +224,32 @@ public void write(DataOutput out) throws IOException { @Override public void readFields(DataInput in) throws IOException { super.readFields(in); - bits = new BitSet(this.vectorSize); + words = new long[wordCount(vectorSize)]; byte[] bytes = new byte[getNBytes()]; in.readFully(bytes); - for (int i = 0, byteIndex = 0, bitIndex = 0; i < vectorSize; i++, bitIndex++) { - if (bitIndex == 8) { - bitIndex = 0; - byteIndex++; - } - if ((bytes[byteIndex] & BIT_VALUES[bitIndex]) != 0) { - bits.set(i); - } + for (int byteIndex = 0; byteIndex < bytes.length; byteIndex++) { + words[byteIndex >>> 3] |= (bytes[byteIndex] & 0xFFL) << ((byteIndex & 7) << 3); } + clearUnusedBits(); } /* @return number of bytes needed to hold bit vector */ private int getNBytes() { return (int) (((long) vectorSize + 7) / 8); } + + private static int wordCount(int vectorSize) { + return (vectorSize + 63) >>> 6; + } + + /** + * Clears bits at positions greater than or equal to {@code vectorSize}, such as the unused + * trailing bits of the last serialized byte, so bitwise ops and serialization stay exact. + */ + private void clearUnusedBits() { + int usedBitsInLastWord = vectorSize & 63; + if (usedBitsInLastWord != 0) { + words[words.length - 1] &= (1L << usedBitsInLastWord) - 1; + } + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/bloom/InternalBloomFilterBenchmark.java b/hudi-common/src/test/java/org/apache/hudi/common/bloom/InternalBloomFilterBenchmark.java new file mode 100644 index 0000000000000..3c3751ee3f21e --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/bloom/InternalBloomFilterBenchmark.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.bloom; + +import org.junit.jupiter.api.Test; + +import java.util.Locale; +import java.util.Random; +import java.util.function.Supplier; + +/** + * Manual microbenchmark for bloom filter hot paths: key adds (the write-path cost paid per + * record by the HFile writer), membership tests, and serialization round trips. + *

    + * The class name intentionally does not match the surefire test patterns, so it never runs + * in CI. Run it explicitly with: + * 

    + * mvn test -pl hudi-common -Dtest=InternalBloomFilterBenchmark -Dsurefire.failIfNoSpecifiedTests=false
+ *
    + */ +public class InternalBloomFilterBenchmark { + + private static final int WARMUP_ROUNDS = 1; + private static final int MEASURED_ROUNDS = 3; + private static final int KEY_POOL_SIZE = 1_000_000; + private static final int MEMBERSHIP_PROBES = 100_000; + + @Test + public void benchmarkBloomFilterHotPaths() { + runScenario("SIMPLE, 1M entries, fpp 1e-3, 1M adds", + () -> BloomFilterFactory.createBloomFilter( + 1_000_000, 0.001, -1, BloomFilterTypeCode.SIMPLE.name()), + 1_000_000); + runScenario("SIMPLE, 10M entries, fpp 1e-9, 10M adds", + () -> BloomFilterFactory.createBloomFilter( + 10_000_000, 0.000000001, -1, BloomFilterTypeCode.SIMPLE.name()), + 10_000_000); + runScenario("DYNAMIC_V0, 60K entries, fpp 1e-9, max 100K, 10M adds", + () -> BloomFilterFactory.createBloomFilter( + 60_000, 0.000000001, 100_000, BloomFilterTypeCode.DYNAMIC_V0.name()), + 10_000_000); + } + + private void runScenario(String name, Supplier filterSupplier, int numAdds) { + String[] keys = generateKeys(KEY_POOL_SIZE, 42); + String[] absentKeys = generateKeys(MEMBERSHIP_PROBES, 4242); + System.out.println("== " + name + " =="); + BloomFilter filter = null; + for (int round = 0; round < WARMUP_ROUNDS + MEASURED_ROUNDS; round++) { + filter = filterSupplier.get(); + long start = System.nanoTime(); + for (int i = 0; i < numAdds; i++) { + filter.add(keys[i % KEY_POOL_SIZE]); + } + long addMs = (System.nanoTime() - start) / 1_000_000; + + start = System.nanoTime(); + int hits = 0; + for (int i = 0; i < MEMBERSHIP_PROBES; i++) { + if (filter.mightContain(keys[i])) { + hits++; + } + } + long hitMs = (System.nanoTime() - start) / 1_000_000; + + start = System.nanoTime(); + int falsePositives = 0; + for (String absentKey : absentKeys) { + if (filter.mightContain(absentKey)) { + falsePositives++; + } + } + long missMs = (System.nanoTime() - start) / 1_000_000; + + String label = round < WARMUP_ROUNDS ? "warmup" : "round" + (round - WARMUP_ROUNDS + 1); + System.out.println(String.format(Locale.ROOT, + "%s: adds(%d)=%d ms, membership hits(%d)=%d ms, misses(%d)=%d ms (hits=%d, falsePositives=%d)", + label, numAdds, addMs, MEMBERSHIP_PROBES, hitMs, absentKeys.length, missMs, hits, falsePositives)); + } + + for (int round = 0; round < WARMUP_ROUNDS + MEASURED_ROUNDS; round++) { + long start = System.nanoTime(); + String serialized = filter.serializeToString(); + long serMs = (System.nanoTime() - start) / 1_000_000; + start = System.nanoTime(); + BloomFilterFactory.fromString(serialized, filter.getBloomFilterTypeCode().name()); + long deserMs = (System.nanoTime() - start) / 1_000_000; + String label = round < WARMUP_ROUNDS ? "warmup" : "round" + (round - WARMUP_ROUNDS + 1); + System.out.println(String.format(Locale.ROOT, + "%s: serialize=%d ms, deserialize=%d ms (serialized length=%d)", + label, serMs, deserMs, serialized.length())); + } + } + + /** Generates fixed-seed 32-character hex keys, mirroring hash-based record keys. */ + private static String[] generateKeys(int count, long seed) { + Random random = new Random(seed); + String[] keys = new String[count]; + byte[] buffer = new byte[16]; + StringBuilder sb = new StringBuilder(32); + for (int i = 0; i < count; i++) { + random.nextBytes(buffer); + sb.setLength(0); + for (byte b : buffer) { + sb.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + keys[i] = sb.toString(); + } + return keys; + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/bloom/TestInternalBloomFilter.java b/hudi-common/src/test/java/org/apache/hudi/common/bloom/TestInternalBloomFilter.java new file mode 100644 index 0000000000000..175fe725ba6c0 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/bloom/TestInternalBloomFilter.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.bloom; + +import org.apache.hudi.common.util.hash.Hash; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests {@link InternalBloomFilter}, pinning the serialized byte layout (bit {@code i} at + * byte {@code i >> 3} under mask {@code 1 << (i & 7)}, the format shared with Hadoop's + * {@code BloomFilter}) against a {@link BitSet}-based oracle and pre-captured golden strings. + */ +public class TestInternalBloomFilter { + + private static final String SIMPLE_GOLDEN = "/////wAAAAoBAAACz+cFPAf5vcr6feFygnZHoFAOwLY7/WznVO6WN9QF7fMmM1m+zXrzC9ICIvydFz8bNEUfQN/L" + + "vtxjs9bkOxNklqSWK6H6aacyEc1SNA0+iZW9Ae0xTLgQp2k6dg=="; + private static final String DYNAMIC_GOLDEN = "/////wAAAAoBAAABIAAAABQAAAA8AAAAA/////8AAAAKAQAAASBSsn9fX63/7M15X+Rmc+75/96eez/Tdme7//nv" + + "1JuP6WZX/tv/////AAAACgEAAAEg6fC+36p6fdz/Lr98Ppl+/QvlV2rfp5+9/Pm358cj1x/f/Hes/////wAAAAoB" + + "AAABINvt/Ha3Ped32/7//3Wtp+3rLH2o/08eqTXZa/u/j77vpvzdvg=="; + + @Test + public void testSerializedBitsMatchBitSetOracle() throws IOException { + int[][] configs = {{63, 3}, {64, 3}, {65, 3}, {127, 5}, {128, 5}, {1000, 7}, {43133, 30}}; + for (int[] config : configs) { + int vectorSize = config[0]; + int nbHash = config[1]; + InternalBloomFilter filter = new InternalBloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH); + HashFunction hashFunction = new HashFunction(vectorSize, nbHash, Hash.MURMUR_HASH); + BitSet oracle = new BitSet(vectorSize); + Random random = new Random(vectorSize); + for (int i = 0; i < 200; i++) { + Key key = randomKey(random); + filter.add(key); + for (int pos : hashFunction.hash(key)) { + oracle.set(pos); + } + assertTrue(filter.membershipTest(key)); + } + assertArrayEquals(oracleBits(vectorSize, oracle), serializedBits(filter), + "Serialized bits mismatch for vectorSize=" + vectorSize); + for (int i = 0; i < 200; i++) { + Key key = randomKey(random); + boolean oracleContains = Arrays.stream(hashFunction.hash(key)).allMatch(oracle::get); + assertEquals(oracleContains, filter.membershipTest(key), + "Membership mismatch for vectorSize=" + vectorSize); + } + } + } + + @Test + public void testWriteReadFieldsRoundTrip() throws IOException { + for (int vectorSize : new int[] {63, 64, 65, 127, 128, 1000}) { + InternalBloomFilter filter = new InternalBloomFilter(vectorSize, 3, Hash.MURMUR_HASH); + Random random = new Random(vectorSize); + Key[] keys = new Key[100]; + for (int i = 0; i < keys.length; i++) { + keys[i] = randomKey(random); + filter.add(keys[i]); + } + byte[] serialized = serialize(filter); + InternalBloomFilter deserialized = new InternalBloomFilter(); + deserialized.readFields(new DataInputStream(new ByteArrayInputStream(serialized))); + for (Key key : keys) { + assertTrue(deserialized.membershipTest(key)); + } + assertArrayEquals(serialized, serialize(deserialized), + "Round-trip bytes mismatch for vectorSize=" + vectorSize); + } + } + + @Test + public void testReadFieldsIgnoresUnusedTrailingBits() throws IOException { + int vectorSize = 61; + InternalBloomFilter filter = new InternalBloomFilter(vectorSize, 3, Hash.MURMUR_HASH); + Random random = new Random(vectorSize); + Key[] keys = new Key[50]; + for (int i = 0; i < keys.length; i++) { + keys[i] = randomKey(random); + filter.add(keys[i]); + } + byte[] serialized = serialize(filter); + byte[] mutated = Arrays.copyOf(serialized, serialized.length); + // The last byte carries bits 56..60; bits 61..63 are beyond vectorSize and must be ignored. + mutated[mutated.length - 1] |= (byte) 0xE0; + InternalBloomFilter deserialized = new InternalBloomFilter(); + deserialized.readFields(new DataInputStream(new ByteArrayInputStream(mutated))); + for (Key key : keys) { + assertTrue(deserialized.membershipTest(key)); + } + assertArrayEquals(serialized, serialize(deserialized), "Unused trailing bits must not survive a round trip"); + } + + @Test + public void testBitwiseOpsMatchBitSetOracle() throws IOException { + int vectorSize = 127; + int nbHash = 5; + HashFunction hashFunction = new HashFunction(vectorSize, nbHash, Hash.MURMUR_HASH); + Random random = new Random(42); + InternalBloomFilter first = new InternalBloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH); + InternalBloomFilter second = new InternalBloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH); + BitSet firstOracle = new BitSet(vectorSize); + BitSet secondOracle = new BitSet(vectorSize); + for (int i = 0; i < 100; i++) { + Key key = randomKey(random); + first.add(key); + for (int pos : hashFunction.hash(key)) { + firstOracle.set(pos); + } + key = randomKey(random); + second.add(key); + for (int pos : hashFunction.hash(key)) { + secondOracle.set(pos); + } + } + + InternalBloomFilter orFilter = copy(first); + orFilter.or(second); + BitSet orOracle = (BitSet) firstOracle.clone(); + orOracle.or(secondOracle); + assertArrayEquals(oracleBits(vectorSize, orOracle), serializedBits(orFilter)); + + InternalBloomFilter andFilter = copy(first); + andFilter.and(second); + BitSet andOracle = (BitSet) firstOracle.clone(); + andOracle.and(secondOracle); + assertArrayEquals(oracleBits(vectorSize, andOracle), serializedBits(andFilter)); + + InternalBloomFilter xorFilter = copy(first); + xorFilter.xor(second); + BitSet xorOracle = (BitSet) firstOracle.clone(); + xorOracle.xor(secondOracle); + assertArrayEquals(oracleBits(vectorSize, xorOracle), serializedBits(xorFilter)); + + InternalBloomFilter notFilter = copy(first); + notFilter.not(); + BitSet notOracle = (BitSet) firstOracle.clone(); + notOracle.flip(0, vectorSize); + assertArrayEquals(oracleBits(vectorSize, notOracle), serializedBits(notFilter)); + } + + @Test + public void testSerializedStringGoldens() { + BloomFilter simple = BloomFilterFactory.createBloomFilter( + 50, 0.001, -1, BloomFilterTypeCode.SIMPLE.name()); + for (int i = 0; i < 50; i++) { + simple.add(goldenKey(i)); + } + assertEquals(SIMPLE_GOLDEN, simple.serializeToString()); + BloomFilter simpleFromGolden = BloomFilterFactory.fromString(SIMPLE_GOLDEN, BloomFilterTypeCode.SIMPLE.name()); + for (int i = 0; i < 50; i++) { + assertTrue(simpleFromGolden.mightContain(goldenKey(i))); + } + + BloomFilter dynamic = BloomFilterFactory.createBloomFilter( + 20, 0.001, 60, BloomFilterTypeCode.DYNAMIC_V0.name()); + for (int i = 0; i < 100; i++) { + dynamic.add(goldenKey(i)); + } + assertEquals(DYNAMIC_GOLDEN, dynamic.serializeToString()); + BloomFilter dynamicFromGolden = BloomFilterFactory.fromString(DYNAMIC_GOLDEN, BloomFilterTypeCode.DYNAMIC_V0.name()); + for (int i = 0; i < 100; i++) { + assertTrue(dynamicFromGolden.mightContain(goldenKey(i))); + } + } + + private static String goldenKey(int i) { + return String.format("key-%03d", i); + } + + private static Key randomKey(Random random) { + byte[] keyBytes = new byte[1 + random.nextInt(40)]; + random.nextBytes(keyBytes); + return new Key(keyBytes); + } + + /** Serializes the filter and strips the 13-byte header (version, nbHash, hashType, vectorSize). */ + private static byte[] serializedBits(InternalBloomFilter filter) throws IOException { + byte[] serialized = serialize(filter); + return Arrays.copyOfRange(serialized, 13, serialized.length); + } + + private static byte[] serialize(InternalBloomFilter filter) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + filter.write(new DataOutputStream(baos)); + return baos.toByteArray(); + } + + /** Packs the oracle bits with the Hadoop BloomFilter byte layout: bit i at byte i >> 3, mask 1 << (i & 7). */ + private static byte[] oracleBits(int vectorSize, BitSet bits) { + byte[] bytes = new byte[(vectorSize + 7) / 8]; + for (int i = 0; i < vectorSize; i++) { + if (bits.get(i)) { + bytes[i >> 3] |= (byte) (1 << (i & 7)); + } + } + return bytes; + } + + private static InternalBloomFilter copy(InternalBloomFilter filter) throws IOException { + InternalBloomFilter copied = new InternalBloomFilter(); + copied.readFields(new DataInputStream(new ByteArrayInputStream(serialize(filter)))); + return copied; + } +} From 2d84d0ca5540a90009ef8f9607b9c0fd02a0ce74 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Fri, 3 Jul 2026 07:57:48 -0700 Subject: [PATCH 125/255] test: handle expected OCC conflict in concurrent Java writer test (#19124) testOccWithMultipleWriters runs concurrent Java write clients under optimistic concurrency control and intentionally contends on the same file groups. OCC aborts one of two conflicting commits by throwing HoodieWriteConflictException, which the test did not handle, so it failed intermittently. Catch the expected conflict and always return the writer to the pool so the test validates deadlock freedom without flaking. (cherry picked from commit 562c3d94ba235144b0777554184b490352c9bc5f) --- .../TestMultipleHoodieJavaWriteClient.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java index f9d95afa8d9fc..50e8d56832830 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java @@ -35,6 +35,7 @@ import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieWriteConflictException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.hudi.storage.StorageConfiguration; @@ -207,13 +208,24 @@ record -> { String startCommitTime = writer.startCommit(); List s = writer.upsert(Collections.singletonList(record), startCommitTime); - writer.commit(startCommitTime, s); - LOGGER.info("Completed commit"); - synchronized (writerQueue) { - try { - writerQueue.put(writer); - } catch (InterruptedException e) { - throw new RuntimeException(e); + try { + writer.commit(startCommitTime, s); + LOGGER.info("Completed commit"); + } catch (HoodieWriteConflictException e) { + // Under OPTIMISTIC_CONCURRENCY_CONTROL, two writers updating overlapping file + // groups conflict and one of them is aborted by design. This test validates that + // concurrent writers do not deadlock, so an aborted commit is an expected outcome + // and must not fail the test. + LOGGER.info("Commit {} was aborted by an expected OCC conflict", startCommitTime); + } finally { + // Always return the writer to the pool, even when its commit was aborted, so the + // remaining records are not starved of writers. + synchronized (writerQueue) { + try { + writerQueue.put(writer); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } } From 98598d6e885f54afa65e81285022ca0f49aba0a8 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 4 Jul 2026 11:44:30 +0800 Subject: [PATCH 126/255] refactor(flink): use SLF4J parameterized logging instead of string concatenation (#19156) * refactor(flink): use SLF4J parameterized logging instead of string concatenation Convert `+`-concatenated log messages to SLF4J `{}` placeholders across hudi-flink-datasource (hudi-flink and the flink1.18.x-2.1.x version modules). The rewrite is behaviour-preserving: each concatenated expression becomes a placeholder argument in order, and a trailing throwable is kept as the last argument so its stack trace is still recorded. This also avoids building the message string eagerly when the log level is disabled. * refactor(flink): reword "Error ?" shutdown log to "Error:" With `{}` templating, "Error ?{}" reads oddly (the `?` looks like a stray ternary fragment). Reword to "Error: {}" in both the clustering job and compactor graceful-shutdown logs. Addresses review nit on #19156. (cherry picked from commit fc91325b63812851d42b5fc66b3738564c0225f2) --- .../org/apache/hudi/configuration/OptionsInference.java | 3 +-- .../hudi/sink/bucket/BucketBulkInsertWriterHelper.java | 2 +- .../org/apache/hudi/sink/bulk/BulkInsertWriterHelper.java | 6 +++--- .../apache/hudi/sink/clustering/ClusteringCommitSink.java | 2 +- .../hudi/sink/clustering/ClusteringPlanOperator.java | 6 +++--- .../hudi/sink/clustering/HoodieFlinkClusteringJob.java | 6 +++--- .../apache/hudi/sink/compact/HoodieFlinkCompactor.java | 8 ++++---- .../org/apache/hudi/sink/partitioner/BucketAssigner.java | 2 +- .../hudi/sink/v2/clustering/ClusteringCommitSinkV2.java | 2 +- .../hudi/sink/v2/compact/CompactionCommitSinkV2.java | 2 +- .../org/apache/hudi/source/IncrementalInputSplits.java | 7 +++---- .../java/org/apache/hudi/table/HoodieTableSource.java | 2 +- .../apache/hudi/table/lookup/HoodieLookupFunction.java | 2 +- .../src/main/java/org/apache/hudi/util/ClientIds.java | 2 +- .../main/java/org/apache/hudi/util/ClusteringUtil.java | 2 +- .../main/java/org/apache/hudi/util/CompactionUtil.java | 4 ++-- .../java/org/apache/hudi/util/ViewStorageProperties.java | 2 +- .../cow/vector/reader/BaseVectorizedColumnReader.java | 7 +------ .../cow/vector/reader/BaseVectorizedColumnReader.java | 7 +------ .../cow/vector/reader/BaseVectorizedColumnReader.java | 7 +------ .../cow/vector/reader/BaseVectorizedColumnReader.java | 7 +------ .../cow/vector/reader/BaseVectorizedColumnReader.java | 7 +------ 22 files changed, 34 insertions(+), 61 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsInference.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsInference.java index 6b5fddfacfab9..70a44b1df7c29 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsInference.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsInference.java @@ -149,8 +149,7 @@ public static void setupIndexConfigs(Configuration conf) { conf.set(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS, hashingConfig.getExpressions()); conf.set(FlinkOptions.BUCKET_INDEX_PARTITION_RULE, hashingConfig.getRule()); conf.set(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, hashingConfig.getDefaultBucketNumber()); - log.info("Loaded Latest Hashing Config " + hashingConfig - + ". Reset hoodie.bucket.index.num.buckets to " + hashingConfig.getDefaultBucketNumber()); + log.info("Loaded Latest Hashing Config {}. Reset hoodie.bucket.index.num.buckets to {}", hashingConfig, hashingConfig.getDefaultBucketNumber()); } } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java index fad1f7e9272ba..37d8b8294c09d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java @@ -64,7 +64,7 @@ public void write(RowData tuple) throws IOException { String partitionPath = keyGen.getPartitionPath(record); String fileId = tuple.getString(0).toString(); if ((lastFileId == null) || !lastFileId.equals(fileId)) { - log.info("Creating new file for partition path " + partitionPath); + log.info("Creating new file for partition path {}", partitionPath); handle = getRowCreateHandle(partitionPath, fileId); lastFileId = fileId; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/BulkInsertWriterHelper.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/BulkInsertWriterHelper.java index 8cb5ef9fdeb45..27f9b375de314 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/BulkInsertWriterHelper.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/BulkInsertWriterHelper.java @@ -144,7 +144,7 @@ private HoodieRowDataCreateHandle getRowCreateHandle(String partitionPath) throw close(); } - log.info("Creating new file for partition path " + partitionPath); + log.info("Creating new file for partition path {}", partitionPath); writeMetrics.ifPresent(FlinkStreamWriteMetrics::startHandleCreation); HoodieRowDataCreateHandle rowCreateHandle = new HoodieRowDataCreateHandle(hoodieTable, writeConfig, partitionPath, getNextFileId(), instantTime, taskPartitionId, totalSubtaskNum, taskEpochId, rowType, preserveHoodieMetadata, isAppendMode && !populateMetaFields); @@ -154,7 +154,7 @@ private HoodieRowDataCreateHandle getRowCreateHandle(String partitionPath) throw } else if (!handles.get(partitionPath).canWrite()) { // even if there is a handle to the partition path, it could have reached its max size threshold. So, we close the handle here and // create a new one. - log.info("Rolling max-size file for partition path " + partitionPath); + log.info("Rolling max-size file for partition path {}", partitionPath); writeStatusList.add(closeWriteHandle(handles.remove(partitionPath))); HoodieRowDataCreateHandle rowCreateHandle = createWriteHandle(partitionPath); handles.put(partitionPath, rowCreateHandle); @@ -172,7 +172,7 @@ public void close() throws IOException { allOf(handles.values().stream() .map(rowCreateHandle -> CompletableFuture.supplyAsync(() -> { try { - log.info("Closing bulk insert file " + rowCreateHandle.getFileName()); + log.info("Closing bulk insert file {}", rowCreateHandle.getFileName()); return rowCreateHandle.close(); } catch (IOException e) { throw new HoodieIOException("IOE during rowCreateHandle.close()", e); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringCommitSink.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringCommitSink.java index c4c1efc21c0af..2bc668d2afa7d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringCommitSink.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringCommitSink.java @@ -175,7 +175,7 @@ private void commitIfNecessary(String instant, Collection doCommit(instant, clusteringPlan, events); } catch (Throwable throwable) { // make it fail-safe - log.error("Error while committing clustering instant: " + instant, throwable); + log.error("Error while committing clustering instant: {}", instant, throwable); } finally { // reset the status reset(instant); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringPlanOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringPlanOperator.java index 609abbdaf862c..49340fc5f465c 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringPlanOperator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringPlanOperator.java @@ -117,7 +117,7 @@ public void notifyCheckpointComplete(long checkpointId) { scheduleClustering(table, checkpointId); } catch (Throwable throwable) { // make it fail-safe - log.error("Error while scheduling clustering plan for checkpoint: " + checkpointId, throwable); + log.error("Error while scheduling clustering plan for checkpoint: {}", checkpointId, throwable); } } @@ -135,7 +135,7 @@ private void scheduleClustering(HoodieFlinkTable table, long checkpointId) { if (!firstRequested.isPresent()) { // do nothing. - log.info("No clustering plan for checkpoint " + checkpointId); + log.info("No clustering plan for checkpoint {}", checkpointId); return; } @@ -158,7 +158,7 @@ private void scheduleClustering(HoodieFlinkTable table, long checkpointId) { if (clusteringPlan == null || (clusteringPlan.getInputGroups() == null) || (clusteringPlan.getInputGroups().isEmpty())) { // do nothing. - log.info("Empty clustering plan for instant " + clusteringInstantTime); + log.info("Empty clustering plan for instant {}", clusteringInstantTime); } else { // Mark instant as clustering inflight ClusteringUtils.transitionClusteringOrReplaceRequestedToInflight(clusteringInstant, Option.empty(), table.getActiveTimeline()); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java index d2f6419515ac2..af9f011715b05 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/HoodieFlinkClusteringJob.java @@ -339,7 +339,7 @@ private void cluster() throws Exception { Option inflightInstantOpt = ClusteringUtils.getInflightClusteringInstant(clusteringInstant.requestedTime(), table.getActiveTimeline(), table.getInstantGenerator()); if (inflightInstantOpt.isPresent()) { - LOG.info("Rollback inflight clustering instant: [" + clusteringInstant + "]"); + LOG.info("Rollback inflight clustering instant: [{}]", clusteringInstant); table.rollbackInflightClustering(inflightInstantOpt.get(), commitToRollback -> writeClient.getTableServiceClient().getPendingRollbackInfo(table.getMetaClient(), commitToRollback, false), writeClient.getTransactionManager()); @@ -362,7 +362,7 @@ private void cluster() throws Exception { if (clusteringPlan == null || (clusteringPlan.getInputGroups() == null) || (clusteringPlan.getInputGroups().isEmpty())) { // no clustering plan, do nothing and return. - LOG.info("No clustering plan for instant " + clusteringInstant.requestedTime()); + LOG.info("No clustering plan for instant {}", clusteringInstant.requestedTime()); return; } @@ -418,7 +418,7 @@ private void cluster() throws Exception { * Shutdown async services like compaction/clustering as DeltaSync is shutdown. */ public void shutdownAsyncService(boolean error) { - LOG.info("Gracefully shutting down clustering job. Error ?" + error); + LOG.info("Gracefully shutting down clustering job. Error: {}", error); executor.shutdown(); writeClient.close(); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/HoodieFlinkCompactor.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/HoodieFlinkCompactor.java index 34ac7e801570f..2ef5d261cd1ef 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/HoodieFlinkCompactor.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/HoodieFlinkCompactor.java @@ -300,7 +300,7 @@ private void compact() throws Exception { compactionInstantTimes.forEach(timestamp -> { HoodieInstant inflightInstant = table.getInstantGenerator().getCompactionInflightInstant(timestamp); if (pendingCompactionTimeline.containsInstant(inflightInstant)) { - LOG.info("Rollback inflight compaction instant: [" + timestamp + "]"); + LOG.info("Rollback inflight compaction instant: [{}]", timestamp); table.rollbackInflightCompaction(inflightInstant, writeClient.getTransactionManager()); table.getMetaClient().reloadActiveTimeline(); } @@ -322,7 +322,7 @@ private void compact() throws Exception { if (compactionPlans.isEmpty()) { // No compaction plan, do nothing and return. - LOG.info("No compaction plan for instant " + String.join(",", compactionInstantTimes)); + LOG.info("No compaction plan for instant {}", String.join(",", compactionInstantTimes)); return; } @@ -336,7 +336,7 @@ private void compact() throws Exception { ? totalOperations : Math.min(conf.get(FlinkOptions.COMPACTION_TASKS), totalOperations); - LOG.info("Start to compaction for instant " + compactionInstantTimes); + LOG.info("Start to compaction for instant {}", compactionInstantTimes); // Mark instant as compaction inflight for (HoodieInstant instant : instants) { @@ -367,7 +367,7 @@ private void compact() throws Exception { * Shutdown async services like compaction/clustering as DeltaSync is shutdown. */ public void shutdownAsyncService(boolean error) { - LOG.info("Gracefully shutting down compactor. Error ?" + error); + LOG.info("Gracefully shutting down compactor. Error: {}", error); executor.shutdown(); writeClient.close(); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigner.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigner.java index fe848d7bcc203..0544f700d15da 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigner.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigner.java @@ -177,7 +177,7 @@ private synchronized SmallFileAssign getSmallFileAssign(String partitionPath) { } List smallFiles = smallFilesOfThisTask(writeProfile.getSmallFiles(partitionPath)); if (smallFiles.size() > 0) { - log.info("For partitionPath : " + partitionPath + " Small Files => " + smallFiles); + log.info("For partitionPath : {} Small Files => {}", partitionPath, smallFiles); SmallFileAssignState[] states = smallFiles.stream() .map(smallFile -> new SmallFileAssignState(config.getParquetMaxFileSize(), smallFile, writeProfile.getAvgSize())) .toArray(SmallFileAssignState[]::new); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/clustering/ClusteringCommitSinkV2.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/clustering/ClusteringCommitSinkV2.java index b6b087c62a1f1..4f31ee877f7ac 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/clustering/ClusteringCommitSinkV2.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/clustering/ClusteringCommitSinkV2.java @@ -185,7 +185,7 @@ private void commitIfNecessary(String instant, Collection doCommit(instant, clusteringPlan, events); } catch (Throwable throwable) { // make it fail-safe - log.error("Error while committing clustering instant: " + instant, throwable); + log.error("Error while committing clustering instant: {}", instant, throwable); } finally { // reset the status reset(instant); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/compact/CompactionCommitSinkV2.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/compact/CompactionCommitSinkV2.java index 848cb10d0c774..7e81112bb0b63 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/compact/CompactionCommitSinkV2.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/compact/CompactionCommitSinkV2.java @@ -175,7 +175,7 @@ private void commitIfNecessary(String instant, Collection doCommit(instant, events); } catch (Throwable throwable) { // make it fail-safe - log.error("Error while committing compaction instant: " + instant, throwable); + log.error("Error while committing compaction instant: {}", instant, throwable); this.compactionMetrics.markCompactionRolledBack(); } finally { // reset the status diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java index 344b24bc10c4b..68e81271d6c86 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/IncrementalInputSplits.java @@ -147,7 +147,7 @@ public Result inputSplits( IncrementalQueryAnalyzer.QueryContext analyzingResult = analyzer.analyze(); if (analyzingResult.isEmpty()) { - log.info("No new instant found for the table under path " + path + ", skip reading"); + log.info("No new instant found for the table under path {}, skip reading", path); return Result.EMPTY; } final HoodieTimeline commitTimeline = analyzingResult.getActiveTimeline(); @@ -276,7 +276,7 @@ public Result inputSplits( IncrementalQueryAnalyzer.QueryContext queryContext = analyzer.analyze(); if (queryContext.isEmpty()) { - log.info("No new instant found for the table under path " + path + ", skip reading"); + log.info("No new instant found for the table under path {}, skip reading", path); return Result.EMPTY; } @@ -501,8 +501,7 @@ private Set getReadPartitions(List metadataList) { double total = partitions.size(); double selectedNum = selectedPartitions.size(); double percentPruned = total == 0 ? 0 : (1 - selectedNum / total) * 100; - log.info("Selected " + selectedNum + " partitions out of " + total - + ", pruned " + percentPruned + "% partitions."); + log.info("Selected {} partitions out of {}, pruned {}% partitions.", selectedNum, total, percentPruned); return selectedPartitions; } return partitions; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java index b5fe77b8e0826..5167c1f623421 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java @@ -467,7 +467,7 @@ private PartitionPruners.PartitionPruner createPartitionPruner(List joiner.add(f.asSummaryString())); - log.info("Partition pruner for hoodie source, condition is:\n" + joiner); + log.info("Partition pruner for hoodie source, condition is:\n{}", joiner); List evaluators = ExpressionEvaluators.fromExpression(partitionFilters); List partitionTypes = this.partitionKeys.stream().map(name -> this.schema.getColumn(name).orElseThrow(() -> new HoodieValidationException("Field " + name + " does not exist"))) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java index 8768222463979..10c47b2fe1c97 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java @@ -145,7 +145,7 @@ private void checkCacheReload() throws IOException { // Determine whether to reload data by comparing instant if (latestCommitInstant.get().equals(currentCommit)) { scheduleNextLoad(); - log.info("Ignore loading data because the commit instant " + currentCommit + " has not changed."); + log.info("Ignore loading data because the commit instant {} has not changed.", currentCommit); return; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java index 0a18ac029e92f..c2b1c1d3a14fd 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java @@ -138,7 +138,7 @@ public static boolean isHeartbeatExpired(FileSystem fs, Path path, long timeoutT } } catch (IOException e) { // if any exception happens, just return false. - log.error("Check heartbeat file existence error: " + path); + log.error("Check heartbeat file existence error: {}", path); } return false; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClusteringUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClusteringUtil.java index 41dee1cd1b07c..e0d3a9d50018d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClusteringUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClusteringUtil.java @@ -87,7 +87,7 @@ public static void rollbackClustering(HoodieFlinkTable table, HoodieFlinkWrit .filter(instant -> instant.getState() == HoodieInstant.State.INFLIGHT) .collect(Collectors.toList()); inflightInstants.forEach(inflightInstant -> { - log.info("Rollback the inflight clustering instant: " + inflightInstant + " for failover"); + log.info("Rollback the inflight clustering instant: {} for failover", inflightInstant); table.rollbackInflightClustering(inflightInstant, commitToRollback -> writeClient.getTableServiceClient().getPendingRollbackInfo(table.getMetaClient(), commitToRollback, false), writeClient.getTransactionManager()); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/CompactionUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/CompactionUtil.java index 8e5456b8f1a69..39b54fefb2e21 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/CompactionUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/CompactionUtil.java @@ -245,7 +245,7 @@ public static void rollbackCompaction(HoodieFlinkTable table, HoodieFlinkWrit .filter(instant -> instant.getState() == HoodieInstant.State.INFLIGHT); inflightCompactionTimeline.getInstants().forEach(inflightInstant -> { - log.info("Rollback the inflight compaction instant: " + inflightInstant + " for failover"); + log.info("Rollback the inflight compaction instant: {} for failover", inflightInstant); table.rollbackInflightCompaction(inflightInstant, commitToRollback -> writeClient.getTableServiceClient().getPendingRollbackInfo(table.getMetaClient(), commitToRollback, false), writeClient.getTransactionManager()); table.getMetaClient().reloadActiveTimeline(); @@ -269,7 +269,7 @@ public static void rollbackEarliestCompaction(HoodieFlinkTable table, Configu String currentTime = HoodieInstantTimeGenerator.getCurrentInstantTimeStr(); int timeout = conf.get(FlinkOptions.COMPACTION_TIMEOUT_SECONDS); if (StreamerUtil.instantTimeDiffSeconds(currentTime, instant.requestedTime()) >= timeout) { - log.info("Rollback the inflight compaction instant: " + instant + " for timeout(" + timeout + "s)"); + log.info("Rollback the inflight compaction instant: {} for timeout({}s)", instant, timeout); try (TransactionManager transactionManager = new TransactionManager(table.getConfig(), table.getStorage())) { table.rollbackInflightCompaction(instant, transactionManager); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ViewStorageProperties.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ViewStorageProperties.java index e0f715c1c85a8..94c2104d44d14 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ViewStorageProperties.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ViewStorageProperties.java @@ -68,7 +68,7 @@ public static void createProperties( */ public static FileSystemViewStorageConfig loadFromProperties(String basePath, Configuration conf) { Path propertyPath = getPropertiesFilePath(basePath, conf.get(FlinkOptions.WRITE_CLIENT_ID)); - log.info("Loading filesystem view storage properties from " + propertyPath); + log.info("Loading filesystem view storage properties from {}", propertyPath); FileSystem fs = HadoopFSUtils.getFs(basePath, HadoopConfigurations.getHadoopConf(conf)); Properties props = new Properties(); try { diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java index 7c9fd994a0c25..700d7505fbc73 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java @@ -226,12 +226,7 @@ private void readPageV2(DataPageV2 page) { this.definitionLevelColumn = newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - log.debug( - "page data size " - + page.getData().size() - + " bytes and " - + pageValueCount - + " records"); + log.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader( page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); } catch (IOException e) { diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java index 7c9fd994a0c25..700d7505fbc73 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java @@ -226,12 +226,7 @@ private void readPageV2(DataPageV2 page) { this.definitionLevelColumn = newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - log.debug( - "page data size " - + page.getData().size() - + " bytes and " - + pageValueCount - + " records"); + log.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader( page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); } catch (IOException e) { diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java index 7c9fd994a0c25..700d7505fbc73 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java @@ -226,12 +226,7 @@ private void readPageV2(DataPageV2 page) { this.definitionLevelColumn = newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - log.debug( - "page data size " - + page.getData().size() - + " bytes and " - + pageValueCount - + " records"); + log.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader( page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); } catch (IOException e) { diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java index 7c9fd994a0c25..700d7505fbc73 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java @@ -226,12 +226,7 @@ private void readPageV2(DataPageV2 page) { this.definitionLevelColumn = newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - log.debug( - "page data size " - + page.getData().size() - + " bytes and " - + pageValueCount - + " records"); + log.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader( page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); } catch (IOException e) { diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java index fbb09823e9b96..c5a170c33e104 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java @@ -228,12 +228,7 @@ private void readPageV2(DataPageV2 page) { this.definitionLevelColumn = newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - LOG.debug( - "page data size " - + page.getData().size() - + " bytes and " - + pageValueCount - + " records"); + LOG.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader( page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); } catch (IOException e) { From 185bfae35a49afb11eacb5f128662cc358023175 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 4 Jul 2026 11:49:18 +0800 Subject: [PATCH 127/255] refactor(hadoop-mr): use SLF4J parameterized logging instead of string concatenation (#19157) Convert LOG.info/error string concatenation to {} placeholders across hudi-hadoop-mr. Mechanical, behavior-preserving. HoodieRealtimeInputFormatUtils: the debug message wraps the value in literal braces; written as {{}} so SLF4J fills the inner {} and keeps the outer braces literal. Dropped the enclosing isDebugEnabled() guard, which only existed to avoid the eager string build that parameterization removes. (cherry picked from commit 9e20f34e81cb2ddd48df1869909f8d26144914e1) --- .../BootstrapColumnStichingRecordReader.java | 2 +- .../HoodieCopyOnWriteTableInputFormat.java | 2 +- .../hudi/hadoop/HoodieParquetInputFormat.java | 4 ++-- .../apache/hudi/hadoop/InputPathHandler.java | 2 +- .../hive/HoodieCombineHiveInputFormat.java | 18 ++++++++---------- .../realtime/AbstractRealtimeRecordReader.java | 10 +++++----- .../HoodieHFileRealtimeInputFormat.java | 7 ++----- .../HoodieParquetRealtimeInputFormat.java | 7 ++----- .../realtime/HoodieRealtimeRecordReader.java | 2 +- .../RealtimeCompactedRecordReader.java | 4 ++-- .../hudi/hadoop/utils/HoodieHiveUtils.java | 6 +++--- .../utils/HoodieRealtimeInputFormatUtils.java | 4 +--- 12 files changed, 29 insertions(+), 39 deletions(-) diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java index 7f9aa77648f94..5c24766d3868f 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java @@ -60,7 +60,7 @@ public BootstrapColumnStichingRecordReader(RecordReader createBootstrappingRecordReade List> colNamesWithTypesForExternal = colNameWithTypes.stream() .filter(p -> !HoodieRecord.HOODIE_META_COLUMNS.contains(p.getKey())).collect(Collectors.toList()); - LOG.info("colNameWithTypes =" + colNameWithTypes + ", Num Entries =" + colNameWithTypes.size()); + LOG.info("colNameWithTypes ={}, Num Entries ={}", colNameWithTypes, colNameWithTypes.size()); if (hoodieColsProjected.isEmpty()) { return getRecordReaderInternal(eSplit.getBootstrapFileSplit(), job, reporter); @@ -225,7 +225,7 @@ private RecordReader createBootstrappingRecordReade jobConfCopy.unset(TableScanDesc.FILTER_EXPR_CONF_STR); jobConfCopy.unset(ConvertAstToSearchArg.SARG_PUSHDOWN); - LOG.info("Generating column stitching reader for " + eSplit.getPath() + " and " + rightSplit.getPath()); + LOG.info("Generating column stitching reader for {} and {}", eSplit.getPath(), rightSplit.getPath()); return new BootstrapColumnStichingRecordReader(getRecordReaderInternal(eSplit, jobConfCopy, reporter), HoodieRecord.HOODIE_META_COLUMNS.size(), getRecordReaderInternal(rightSplit, jobConfCopy, reporter), diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java index 88e96d29e1be2..09f6f4bbabdb4 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java @@ -114,7 +114,7 @@ private void parseInputPaths(Path[] inputPaths, List incrementalTables) tagAsIncrementalOrSnapshot(inputPath, metaClient, incrementalTables); } catch (TableNotFoundException | InvalidTableException e) { // This is a non Hoodie inputPath - LOG.info("Handling a non-hoodie path " + inputPath); + LOG.info("Handling a non-hoodie path {}", inputPath); nonHoodieInputPaths.add(inputPath); } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java index 9634b7f6b097c..c726bda2e69a3 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java @@ -174,7 +174,7 @@ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map inputFormatClass = part.getInputFileFormatClass(); String inputFormatClassName = inputFormatClass.getName(); InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); - LOG.info("Input Format => " + inputFormatClass.getName()); + LOG.info("Input Format => {}", inputFormatClass.getName()); // **MOD** Set the hoodie filter in the combine if (inputFormatClass.getName().equals(getParquetInputFormatClassName())) { combine.setHoodieFilter(true); @@ -186,7 +186,7 @@ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map partitions = new ArrayList<>(part.getPartSpec().keySet()); if (!partitions.isEmpty()) { String partitionStr = String.join("/", partitions); - LOG.info("Setting Partitions in jobConf - Partition Keys for Path : " + path + " is :" + partitionStr); + LOG.info("Setting Partitions in jobConf - Partition Keys for Path : {} is :{}", path, partitionStr); job.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, partitionStr); } else { job.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, ""); @@ -224,11 +224,11 @@ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map getNonCombinablePathIndices(JobConf job, Path[] paths, int numThreads) throws ExecutionException, InterruptedException { - LOG.info("Total number of paths: " + paths.length + ", launching " + numThreads - + " threads to check non-combinable ones."); + LOG.info("Total number of paths: {}, launching {} threads to check non-combinable ones.", paths.length, numThreads); int numPathPerThread = (int) Math.ceil((double) paths.length / numThreads); ExecutorService executor = Executors.newFixedThreadPool(numThreads); @@ -559,7 +558,7 @@ private List sampleSplits(List splits) { retLists.add(split); long splitgLength = split.getLength(); if (size + splitgLength >= targetSize) { - LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits"); + LOG.info("Sample alias {} using {}splits", entry.getValue(), (i + 1)); if (size + splitgLength > targetSize) { ((InputSplitShim) split).shrinkSplit(targetSize - size); } @@ -963,8 +962,7 @@ public CombineFileSplit[] getSplits(JobConf job, int numSplits) throws IOExcepti if (job.getLong(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.SPLIT_MAXSIZE, 0L) == 0L) { super.setMaxSplitSize(minSize); } - LOG.info("mapreduce.input.fileinputformat.split.minsize=" + minSize - + ", mapreduce.input.fileinputformat.split.maxsize=" + maxSize); + LOG.info("mapreduce.input.fileinputformat.split.minsize={}, mapreduce.input.fileinputformat.split.maxsize={}", minSize, maxSize); if (isRealTime) { job.set("hudi.hive.realtime", "true"); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java index 913e670095273..ce58db552e23d 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java @@ -94,9 +94,9 @@ public abstract class AbstractRealtimeRecordReader { public AbstractRealtimeRecordReader(RealtimeSplit split, JobConf job) { this.split = split; this.jobConf = job; - LOG.info("cfg ==> " + job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR)); - LOG.info("columnIds ==> " + job.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); - LOG.info("partitioningColumns ==> " + job.get(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "")); + LOG.info("cfg ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR)); + LOG.info("columnIds ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + LOG.info("partitioningColumns ==> {}", job.get(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "")); this.supportPayload = Boolean.parseBoolean(job.get("hoodie.support.payload", "true")); try { metaClient = HoodieTableMetaClient.builder() @@ -106,7 +106,7 @@ public AbstractRealtimeRecordReader(RealtimeSplit split, JobConf job) { this.payloadProps.setProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY, metaClient.getTableConfig().getOrderingFieldsStr().orElse(null)); } this.usesCustomPayload = usesCustomPayload(metaClient); - LOG.info("usesCustomPayload ==> " + this.usesCustomPayload); + LOG.info("usesCustomPayload ==> {}", this.usesCustomPayload); // get timestamp columns supportTimestamp = HoodieColumnProjectionUtils.supportTimestamp(jobConf); @@ -190,7 +190,7 @@ private void init() throws Exception { public HoodieSchema constructHiveOrderedSchema(HoodieSchema writerSchema, Map schemaFieldsMap, String hiveColumnString) { String[] hiveColumns = hiveColumnString.isEmpty() ? new String[0] : hiveColumnString.split(","); - LOG.info("Hive Columns : " + hiveColumnString); + LOG.info("Hive Columns : {}", hiveColumnString); List hiveSchemaFields = new ArrayList<>(); for (String columnName : hiveColumns) { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java index c7655abbbf3d0..ef12dee0ad068 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java @@ -62,9 +62,7 @@ public RecordReader getRecordReader(final InputSpli // actual heavy lifting of reading the parquet files happen. if (jobConf.get(HoodieInputFormatUtils.HOODIE_READ_COLUMNS_PROP) == null) { synchronized (jobConf) { - LOG.info( - "Before adding Hoodie columns, Projections :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) - + ", Ids :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + LOG.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); if (jobConf.get(HoodieInputFormatUtils.HOODIE_READ_COLUMNS_PROP) == null) { // Hive (across all versions) fails for queries like select count(`_hoodie_commit_time`) from table; // In this case, the projection fields gets removed. Looking at HiveInputFormat implementation, in some cases @@ -82,8 +80,7 @@ public RecordReader getRecordReader(final InputSpli } HoodieRealtimeInputFormatUtils.cleanProjectionColumnIds(jobConf); - LOG.info("Creating record reader with readCols :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) - + ", Ids :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + LOG.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); // sanity check ValidationUtils.checkArgument(split instanceof HoodieRealtimeFileSplit, "HoodieRealtimeRecordReader can only work on HoodieRealtimeFileSplit and not with " + split); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java index a911bbab788b6..48e78f1476445 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java @@ -80,8 +80,7 @@ public RecordReader getRecordReader(final InputSpli HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(getStorageConf(jobConf)).setBasePath(realtimeSplit.getBasePath()).build(); HoodieTableConfig tableConfig = metaClient.getTableConfig(); addProjectionToJobConf(realtimeSplit, jobConf, tableConfig); - LOG.info("Creating record reader with readCols :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) - + ", Ids :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + LOG.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); // for log only split, set the parquet reader as empty. if (isLogFile(realtimeSplit.getPath())) { @@ -100,9 +99,7 @@ void addProjectionToJobConf(final RealtimeSplit realtimeSplit, final JobConf job // actual heavy lifting of reading the parquet files happen. if (HoodieRealtimeInputFormatUtils.canAddProjectionToJobConf(realtimeSplit, jobConf)) { synchronized (jobConf) { - LOG.info( - "Before adding Hoodie columns, Projections :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) - + ", Ids :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + LOG.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); if (HoodieRealtimeInputFormatUtils.canAddProjectionToJobConf(realtimeSplit, jobConf)) { // Hive (across all versions) fails for queries like select count(`_hoodie_commit_time`) from table; // In this case, the projection fields gets removed. Looking at HiveInputFormat implementation, in some cases diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java index 79d8e6ad64a65..eea8200d7fe02 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java @@ -66,7 +66,7 @@ private static RecordReader constructRecordReader(R LOG.info("Enabling un-merged reading of realtime records"); return new RealtimeUnmergedRecordReader(split, jobConf, realReader); } - LOG.info("Enabling merged reading of realtime records for split " + split); + LOG.info("Enabling merged reading of realtime records for split {}", split); return new RealtimeCompactedRecordReader(split, jobConf, realReader); } catch (Exception e) { LOG.error("Got exception when constructing record reader", e); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java index c68b984e5686c..de676624c6c38 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java @@ -187,8 +187,8 @@ private void setUpWritable(Option rec, ArrayWritable ar arrayWritable.set(originalValue); } catch (RuntimeException re) { LOG.error("Got exception when doing array copy", re); - LOG.error("Base record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable)); - LOG.error("Log record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable)); + LOG.error("Base record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable)); + LOG.error("Log record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable)); String errMsg = "Base-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable) + " ,Log-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable) + " ,Error :" + re.getMessage(); throw new RuntimeException(errMsg, re); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java index 981ab9ce54b3c..a781f3c53d092 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java @@ -94,7 +94,7 @@ public static Option getMaxCommit(JobConf job, String tableName) { public static boolean stopAtCompaction(JobContext job, String tableName) { String compactionPropName = String.format(HOODIE_STOP_AT_COMPACTION_PATTERN, tableName); boolean stopAtCompaction = job.getConfiguration().getBoolean(compactionPropName, true); - LOG.info("Read stop at compaction - " + stopAtCompaction); + LOG.info("Read stop at compaction - {}", stopAtCompaction); return stopAtCompaction; } @@ -104,13 +104,13 @@ public static Integer readMaxCommits(JobContext job, String tableName) { if (maxCommits == MAX_COMMIT_ALL) { maxCommits = Integer.MAX_VALUE; } - LOG.info("Read max commits - " + maxCommits); + LOG.info("Read max commits - {}", maxCommits); return maxCommits; } public static String readStartCommitTime(JobContext job, String tableName) { String startCommitTimestampName = String.format(HOODIE_START_COMMIT_PATTERN, tableName); - LOG.info("Read start commit time - " + job.getConfiguration().get(startCommitTimestampName)); + LOG.info("Read start commit time - {}", job.getConfiguration().get(startCommitTimestampName)); return job.getConfiguration().get(startCommitTimestampName); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java index 0e1539fab9bfb..f40eca1308729 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java @@ -137,9 +137,7 @@ public static void cleanProjectionColumnIds(Configuration conf) { String columnIds = conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); if (!columnIds.isEmpty() && columnIds.charAt(0) == ',') { conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, columnIds.substring(1)); - if (LOG.isDebugEnabled()) { - LOG.debug("The projection Ids: {" + columnIds + "} start with ','. First comma is removed"); - } + LOG.debug("The projection Ids: {{}} start with ','. First comma is removed", columnIds); } } } From eb67451ea600d81a5aed2992a5d1c95f443c2425 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 4 Jul 2026 11:50:45 +0800 Subject: [PATCH 128/255] refactor(hadoop-common): use SLF4J parameterized logging instead of string concatenation (#19160) Convert log/LOG string concatenation to {} placeholders across hudi-hadoop-common (main + tests). Mechanical and behavior-preserving; the trailing throwable in HoodieParquetFileBinaryCopier stays the last arg so its stacktrace is logged as before. (cherry picked from commit 9a8edd6e6495ab0b0233d46f5f22ce933abc1993) --- .../hudi/common/config/DFSPropertiesConfiguration.java | 4 ++-- .../hudi/io/storage/hadoop/HoodieAvroHFileWriter.java | 2 +- .../hudi/parquet/io/HoodieParquetFileBinaryCopier.java | 2 +- .../table/view/TestHoodieTableFileSystemView.java | 2 +- .../common/table/view/TestIncrementalFSViewSync.java | 10 +++++----- .../common/testutils/minicluster/HdfsTestService.java | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java index daabcf41dbb00..e048d269625f6 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java @@ -194,7 +194,7 @@ void addPropsFromFile(StoragePath filePath, boolean tolerateMissing) { visitedFilePaths.add(filePath.toString()); addPropsFromStream(reader, filePath); } catch (IOException ioe) { - log.error("Error reading in properties from dfs from file " + filePath); + log.error("Error reading in properties from dfs from file {}", filePath); throw new HoodieIOException("Cannot read properties from dfs from file " + filePath, ioe); } } @@ -279,7 +279,7 @@ public TypedProperties getProps(boolean includeGlobalProps) { private static Option getConfPathFromEnv() { String confDir = System.getenv(CONF_FILE_DIR_ENV_NAME); if (confDir == null) { - log.debug("Environment variable " + CONF_FILE_DIR_ENV_NAME + ", not set. If desired, set it to the folder containing: " + DEFAULT_PROPERTIES_FILE); + log.debug("Environment variable {}, not set. If desired, set it to the folder containing: {}", CONF_FILE_DIR_ENV_NAME, DEFAULT_PROPERTIES_FILE); return Option.empty(); } if (StringUtils.isNullOrEmpty(URI.create(confDir).getScheme())) { diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java index 9031390af1bb8..864fbc9f80b04 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java @@ -133,7 +133,7 @@ public void writeAvro(String recordKey, IndexedRecord record) throws IOException if (!this.hfileConfig.isAllowDuplicatesOnHfileWrites()) { // When allowDuplicatesOnHfileWrites is true, allow duplicates to be written to hFile. if (prevRecordKey.equals(recordKey)) { - LOG.info("Duplicate recordKey " + recordKey + " found while writing to HFile. Record payload " + record); + LOG.info("Duplicate recordKey {} found while writing to HFile. Record payload {}", recordKey, record); throw new HoodieDuplicateKeyException("Duplicate recordKey " + recordKey + " found while writing to HFile."); } } diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/parquet/io/HoodieParquetFileBinaryCopier.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/parquet/io/HoodieParquetFileBinaryCopier.java index fda8780a46c29..c2a8265573cc5 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/parquet/io/HoodieParquetFileBinaryCopier.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/parquet/io/HoodieParquetFileBinaryCopier.java @@ -315,7 +315,7 @@ private void triggerPrefetch() { } return new PrefetchResult(targetBuffer, requiredSize); } catch (IOException e) { - log.error("Failed to prefetch file: " + fileToPrefetch, e); + log.error("Failed to prefetch file: {}", fileToPrefetch, e); throw new RuntimeException(e); } }, prefetchExecutor); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java index ff9e303f7772a..f89033fecae61 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java @@ -1198,7 +1198,7 @@ protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingData roView.getAllBaseFiles(partitionPath); fileSliceList = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); - log.info("FILESLICE LIST=" + fileSliceList); + log.info("FILESLICE LIST={}", fileSliceList); dataFiles = fileSliceList.stream().map(FileSlice::getBaseFile).filter(Option::isPresent).map(Option::get) .collect(Collectors.toList()); assertEquals(1, dataFiles.size(), "Expect only one data-files in latest view as there is only one file-group"); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index 6535f6be700e1..0e6285f274e07 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -527,7 +527,7 @@ private void testCleans(SyncableFileSystemView view, List newCleanerInst final int netFilesAddedPerInstant = numFilesAddedPerInstant - numFilesReplacedPerInstant; assertEquals(newCleanerInstants.size(), cleanedInstants.size()); long exp = PARTITIONS.stream().mapToLong(p1 -> view.getAllFileSlices(p1).count()).findAny().getAsLong(); - log.info("Initial File Slices :" + exp); + log.info("Initial File Slices :{}", exp); for (int idx = 0; idx < newCleanerInstants.size(); idx++) { String instant = cleanedInstants.get(idx); try { @@ -544,8 +544,8 @@ private void testCleans(SyncableFileSystemView view, List newCleanerInst assertEquals(State.COMPLETED, view.getLastInstant().get().getState()); assertEquals(HoodieTimeline.CLEAN_ACTION, view.getLastInstant().get().getAction()); PARTITIONS.forEach(p -> { - log.info("PARTITION : " + p); - log.info("\tFileSlices :" + view.getAllFileSlices(p).collect(Collectors.toList())); + log.info("PARTITION : {}", p); + log.info("\tFileSlices :{}", view.getAllFileSlices(p).collect(Collectors.toList())); }); final int instantIdx = newCleanerInstants.size() - idx; @@ -593,7 +593,7 @@ private void testRestore(SyncableFileSystemView view, List newRestoreIns isDeltaCommit ? initialFileSlices : initialFileSlices - ((idx + 1) * (FILE_IDS_PER_PARTITION.size() - totalReplacedFileSlicesPerPartition)); view.sync(); assertTrue(view.getLastInstant().isPresent()); - log.info("Last Instant is :" + view.getLastInstant().get()); + log.info("Last Instant is :{}", view.getLastInstant().get()); if (isRestore) { assertEquals(newRestoreInstants.get(idx), view.getLastInstant().get().requestedTime()); assertEquals(HoodieTimeline.RESTORE_ACTION, view.getLastInstant().get().getAction()); @@ -881,7 +881,7 @@ private Map> testMultipleWriteSteps(SyncableFileSystemView int multiple = begin; for (int idx = 0; idx < instants.size(); idx++) { String instant = instants.get(idx); - log.info("Adding instant=" + instant); + log.info("Adding instant={}", instant); HoodieInstant lastInstant = lastInstants.get(idx); // Add a non-empty ingestion to COW table List filePaths = addInstant(metaClient, instant, deltaCommit); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java index 2a763b77c6769..1a1bec072d9ae 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java @@ -63,7 +63,7 @@ public MiniDFSCluster start(boolean format) throws IOException { // If clean, then remove the work dir so we can start fresh. if (format) { - log.info("Cleaning HDFS cluster data at: " + dfsBaseDirPath + " and starting fresh."); + log.info("Cleaning HDFS cluster data at: {} and starting fresh.", dfsBaseDirPath); Files.deleteIfExists(dfsBaseDirPath); } @@ -114,7 +114,7 @@ public void stop() { private static Configuration configureDFSCluster(Configuration config, String dfsBaseDir, String bindIP, int namenodeRpcPort, int datanodePort, int datanodeIpcPort, int datanodeHttpPort) { - log.info("HDFS force binding to ip: " + bindIP); + log.info("HDFS force binding to ip: {}", bindIP); config.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + bindIP + ":" + namenodeRpcPort); config.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, bindIP + ":" + datanodePort); config.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, bindIP + ":" + datanodeIpcPort); From f0590792592034739487c7c189b6be72edbe2a97 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 4 Jul 2026 17:44:47 +0800 Subject: [PATCH 129/255] refactor(common): use SLF4J parameterized logging instead of string concatenation (#19158) Convert log string concatenation to {} placeholders across hudi-common. Mechanical and behavior-preserving. Notable manual handling: - ZookeeperTestService: pass e.toString() (not e) so the exception stays an inline message value instead of being peeled as the SLF4J stacktrace arg, preserving the existing 'ignore as expected' behavior. - ConfigUtils: drop redundant path.toString() in the parameterized arg. - JmxMetricsReporter: the message wraps the value in literal braces; written as "{port: {}}" so SLF4J fills the inner {} and keeps the outer braces. (cherry picked from commit 1030cf9d1af7192e2328fe0cf0ce4041abc024bf) --- .../common/model/HoodieCommitMetadata.java | 2 +- .../common/table/timeline/TimelineUtils.java | 2 +- .../table/view/RocksDbBasedFileSystemView.java | 10 ++++------ .../apache/hudi/common/util/ConfigUtils.java | 2 +- .../hudi/metrics/JmxMetricsReporter.java | 8 ++++---- .../hudi/metrics/MetricsReporterFactory.java | 2 +- .../hudi/TestReportJvmConfiguration.java | 18 +++++++++--------- .../minicluster/ZookeeperTestService.java | 6 +++--- 8 files changed, 24 insertions(+), 26 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java index 51171b83b0e6b..949f766e13ccf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java @@ -220,7 +220,7 @@ public Map getFileIdToInfo(String basePath) { public String toJsonString() throws IOException { if (partitionToWriteStats.containsKey(null)) { - log.info("partition path is null for " + partitionToWriteStats.get(null)); + log.info("partition path is null for {}", partitionToWriteStats.get(null)); partitionToWriteStats.remove(null); } return JsonUtils.getObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(this); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java index 6c27d1f8e2a67..647cab77123d6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java @@ -263,7 +263,7 @@ public static Map> getAllExtraMetadataForKey(HoodieTableM private static Option getMetadataValue(HoodieTableMetaClient metaClient, String extraMetadataKey, HoodieInstant instant) { try { - log.info("reading checkpoint info for:" + instant + " key: " + extraMetadataKey); + log.info("reading checkpoint info for:{} key: {}", instant, extraMetadataKey); byte[] contents = metaClient.getCommitsTimeline().getInstantDetails(instant).get(); if (instant.isCompleted()) { if (contents == null || contents.length == 0) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index fee37f33380bc..c8df4063abf0a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -207,8 +207,7 @@ public Stream> fetchFileGroupsInPendingCl @Override void resetFileGroupsInPendingClustering(Map fgIdToInstantMap) { - log.info("Resetting file groups in pending clustering to ROCKSDB based file-system view at " - + config.getRocksdbBasePath() + ", Total file-groups=" + fgIdToInstantMap.size()); + log.info("Resetting file groups in pending clustering to ROCKSDB based file-system view at {}, Total file-groups={}", config.getRocksdbBasePath(), fgIdToInstantMap.size()); // Delete all replaced file groups rocksDB.prefixDelete(schemaHelper.getColFamilyForFileGroupsInPendingClustering(), "part="); @@ -517,8 +516,7 @@ Option fetchHoodieFileGroup(String partitionPath, String fileId @Override protected void resetReplacedFileGroups(final Map replacedFileGroups) { - log.info("Resetting replacedFileGroups to ROCKSDB based file-system view at " - + config.getRocksdbBasePath() + ", Total file-groups=" + replacedFileGroups.size()); + log.info("Resetting replacedFileGroups to ROCKSDB based file-system view at {}, Total file-groups={}", config.getRocksdbBasePath(), replacedFileGroups.size()); // Delete all replaced file groups rocksDB.prefixDelete(schemaHelper.getColFamilyForReplacedFileGroups(), "part="); @@ -543,8 +541,8 @@ protected void addReplacedFileGroups(final Map }) ); - log.info("Finished adding replaced file groups to partition (" + partitionPath + ") to ROCKSDB based view at " - + config.getRocksdbBasePath() + ", Total file-groups=" + partitionToReplacedFileGroupsEntry.getValue().size()); + log.info("Finished adding replaced file groups to partition ({}) to ROCKSDB based view at {}, Total file-groups={}", + partitionPath, config.getRocksdbBasePath(), partitionToReplacedFileGroupsEntry.getValue().size()); }); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java index 5f0ce3bbb9a06..6fb81390ccda9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java @@ -682,7 +682,7 @@ public static TypedProperties fetchConfigs( return props; } catch (IOException e) { if (HoodieExceptionUtil.isPermissionDeniedException(e)) { - log.error("Permission denied for " + path.toString() + " file.", e); + log.error("Permission denied for {} file.", path, e); throw new HoodieIOException("Permission denied for " + path + " file path. User does not have read access on the dataset.", e); } else { log.warn("Could not read properties from {}: {}", path, e); diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java b/hudi-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java index 623a055100a78..da36131652850 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java @@ -60,7 +60,7 @@ public JmxMetricsReporter(HoodieMetricsConfig config, MetricRegistry registry) { "Could not start JMX server on any configured port. Ports: " + portsConfig + ". Maybe require port range for multiple hoodie tables"); } - log.info("Configured JMXReporter with {port:" + portsConfig + "}"); + log.info("Configured JMXReporter with {port: {}}", portsConfig); } catch (Exception e) { String msg = "Jmx initialize failed: "; log.error(msg, e); @@ -76,13 +76,13 @@ private void initializeJmxReporterServer(String host, int[] ports) { for (int port : ports) { try { jmxReporterServer = createJmxReport(host, port); - log.info("Started JMX server on port " + port + "."); + log.info("Started JMX server on port {}.", port); break; } catch (Exception e) { if (e.getCause() instanceof ExportException) { - log.info("Skip for initializing jmx port " + port + " because of already in use"); + log.info("Skip for initializing jmx port {} because of already in use", port); } else { - log.info("Failed to initialize jmx port " + port + ". " + e.getMessage()); + log.info("Failed to initialize jmx port {}. {}", port, e.getMessage()); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index b93968e23d73f..e2b850e8732f3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -94,7 +94,7 @@ public static Option createReporter(HoodieMetricsConfig metrics reporter = new Slf4jMetricsReporter(registry); break; default: - log.error("Reporter type[" + type + "] is not supported."); + log.error("Reporter type[{}] is not supported.", type); break; } return Option.ofNullable(reporter); diff --git a/hudi-common/src/test/java/org/apache/hudi/TestReportJvmConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/TestReportJvmConfiguration.java index ea3eeade1baaf..57edd95e1c65b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/TestReportJvmConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/TestReportJvmConfiguration.java @@ -42,14 +42,14 @@ private void reportMemoryUsageWithMXBean() { MemoryUsage nonHeapMemoryUsage = memoryBean.getNonHeapMemoryUsage(); LOG.warn("Heap Memory Usage (MemoryMXBean):"); - LOG.warn(" Used: " + heapMemoryUsage.getUsed() + " bytes"); - LOG.warn(" Committed: " + heapMemoryUsage.getCommitted() + " bytes"); - LOG.warn(" Max: " + heapMemoryUsage.getMax() + " bytes"); + LOG.warn(" Used: {} bytes", heapMemoryUsage.getUsed()); + LOG.warn(" Committed: {} bytes", heapMemoryUsage.getCommitted()); + LOG.warn(" Max: {} bytes", heapMemoryUsage.getMax()); LOG.warn("Non-Heap Memory Usage (MemoryMXBean):"); - LOG.warn(" Used: " + nonHeapMemoryUsage.getUsed() + " bytes"); - LOG.warn(" Committed: " + nonHeapMemoryUsage.getCommitted() + " bytes"); - LOG.warn(" Max: " + nonHeapMemoryUsage.getMax() + " bytes"); + LOG.warn(" Used: {} bytes", nonHeapMemoryUsage.getUsed()); + LOG.warn(" Committed: {} bytes", nonHeapMemoryUsage.getCommitted()); + LOG.warn(" Max: {} bytes", nonHeapMemoryUsage.getMax()); } private void reportMemoryUsageWithRuntime() { @@ -60,8 +60,8 @@ private void reportMemoryUsageWithRuntime() { long usedMemory = totalMemory - freeMemory; LOG.warn("Memory Usage (Runtime):"); - LOG.warn(" Total Memory: " + totalMemory + " bytes"); - LOG.warn(" Free Memory: " + freeMemory + " bytes"); - LOG.warn(" Used Memory: " + usedMemory + " bytes"); + LOG.warn(" Total Memory: {} bytes", totalMemory); + LOG.warn(" Free Memory: {} bytes", freeMemory); + LOG.warn(" Used Memory: {} bytes", usedMemory); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java index d3e8359662420..779f4cb89ef75 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java @@ -101,7 +101,7 @@ public ZooKeeperServer start() throws IOException, InterruptedException { // NOTE: Changed from the original, where InetSocketAddress was // originally created to bind to the wildcard IP, we now configure it. - log.info("Zookeeper force binding to: " + this.bindIP); + log.info("Zookeeper force binding to: {}", this.bindIP); standaloneServerFactory.configure(new InetSocketAddress(bindIP, clientPort), 1000); // Start up this ZK server @@ -118,7 +118,7 @@ public ZooKeeperServer start() throws IOException, InterruptedException { } started = true; - log.info("Zookeeper Minicluster service started on client port: " + clientPort); + log.info("Zookeeper Minicluster service started on client port: {}", clientPort); return zooKeeperServer; } @@ -217,7 +217,7 @@ private static boolean waitForServerUp(String hostname, int port, long timeout) } } catch (IOException e) { // ignore as this is expected - log.info("server " + hostname + ":" + port + " not up " + e); + log.info("server {}:{} not up {}", hostname, port, e.toString()); } if (System.currentTimeMillis() > start + timeout) { From 4007c392ef9488e00ef95f9c613d451c51bb757a Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Sat, 4 Jul 2026 02:50:10 -0700 Subject: [PATCH 130/255] fix(spark): correct self-recursive equals in ProcedureParameterImpl (#19167) equals() called this == other, which dispatches back to equals() and recurses to a StackOverflowError, and it cast other to ProcedureParameterImpl before the null/type guard, throwing ClassCastException for a wrong-type argument. Use reference equality for the identity check and move the cast after the guard. (cherry picked from commit 88d34dc77146aafabb719ed24c84b3dd4ec3afdf) --- .../sql/hudi/command/procedures/ProcedureParameterImpl.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ProcedureParameterImpl.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ProcedureParameterImpl.scala index a7f4117047457..f5ff30c1b90b2 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ProcedureParameterImpl.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ProcedureParameterImpl.scala @@ -25,15 +25,14 @@ case class ProcedureParameterImpl(index: Int, name: String, dataType: DataType, extends ProcedureParameter { override def equals(other: Any): Boolean = { - val that = other.asInstanceOf[ProcedureParameterImpl] - val rtn = if (this == other) { + if (this eq other.asInstanceOf[AnyRef]) { true } else if (other == null || (getClass ne other.getClass)) { false } else { + val that = other.asInstanceOf[ProcedureParameterImpl] index == that.index && required == that.required && default == that.default && Objects.equals(name, that.name) && Objects.equals(dataType, that.dataType) } - rtn } override def hashCode: Int = Seq(index, name, dataType, required, default).hashCode() From 71c368d3a4720d4e2ebc6bc5cee1a9cd2700e1c1 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 4 Jul 2026 19:02:49 +0800 Subject: [PATCH 131/255] refactor(cli): use SLF4J parameterized logging instead of string concatenation (#19159) * refactor(cli): use SLF4J parameterized logging instead of string concatenation Convert log string concatenation to {} placeholders across hudi-cli. Mechanical and behavior-preserving. Notable handling: - MetadataCommand: two long multi-line concatenations (FS/metadata file size mismatch) rewritten as wrapped parameterized calls; the inline equality expression becomes a boolean {} arg. - RepairsCommand: drop redundant hoodieInstant.toString() in the parameterized arg. - TableCommand: kept schema.toString(true) as-is (formatted print, not a redundant no-arg toString). * refactor(cli): serialize table schema once in fetchTableSchema Extract schema.toString(true) to a local in the outputFilePath branch so it is computed once and reused for both the log statement and writeToFile, instead of serializing the schema twice. Per review. (cherry picked from commit d585f83462d7f6f220df733dd66a7942173e7c55) --- .../hudi/cli/commands/CompactionCommand.java | 2 +- .../hudi/cli/commands/ExportCommand.java | 2 +- .../cli/commands/LockAuditingCommand.java | 6 ++-- .../hudi/cli/commands/MetadataCommand.java | 31 +++++++++---------- .../hudi/cli/commands/RepairsCommand.java | 6 ++-- .../hudi/cli/commands/TableCommand.java | 5 +-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java index 61a8ba5c6b12c..53c48c847570d 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java @@ -432,7 +432,7 @@ private T deSerializeOperationResult(StoragePath inputPath, ObjectInputStream in = new ObjectInputStream(inputStream); try { T result = (T) in.readObject(); - log.info("Result : " + result); + log.info("Result : {}", result); return result; } finally { in.close(); diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java index 8692efb8f6baa..83bdbbdb9c5c0 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java @@ -172,7 +172,7 @@ private int copyArchivedInstants(List pathInfoList, final String instantTime = archiveEntryRecord.get("commitTime").toString(); if (metadata == null) { - log.error("Could not load metadata for action " + action + " at instant time " + instantTime); + log.error("Could not load metadata for action {} at instant time {}", action, instantTime); continue; } final String outPath = localFolder + StoragePath.SEPARATOR + instantTime + "." + action; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/LockAuditingCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/LockAuditingCommand.java index 8aa709646d3fc..3550916373e4b 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/LockAuditingCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/LockAuditingCommand.java @@ -503,7 +503,7 @@ private CleanupResult performAuditCleanup(boolean dryRun, int ageDays) { deletedCount++; } catch (Exception e) { failedCount++; - log.warn("Failed to delete audit file: " + pathInfo.getPath(), e); + log.warn("Failed to delete audit file: {}", pathInfo.getPath(), e); } } @@ -543,7 +543,7 @@ private Option parseAuditFile(StoragePathInfo pathInfo) { AuditRecord entry = OBJECT_MAPPER.readValue(line, AuditRecord.class); entries.add(entry); } catch (Exception e) { - log.warn("Failed to parse JSON line in file " + filename + ": " + line, e); + log.warn("Failed to parse JSON line in file {}: {}", filename, line, e); } } } @@ -592,7 +592,7 @@ private Option parseAuditFile(StoragePathInfo pathInfo) { filename )); } catch (Exception e) { - log.warn("Failed to parse audit file: " + filename, e); + log.warn("Failed to parse audit file: {}", filename, e); return Option.empty(); } } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java index dbdc32211a96b..e2e86193b2ee9 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java @@ -307,8 +307,8 @@ public String validateFiles( if (!fsPartitions.equals(metadataPartitions)) { log.error("FS partition listing is not matching with metadata partition listing!"); - log.error("All FS partitions: " + Arrays.toString(fsPartitions.toArray())); - log.error("All Metadata partitions: " + Arrays.toString(metadataPartitions.toArray())); + log.error("All FS partitions: {}", Arrays.toString(fsPartitions.toArray())); + log.error("All Metadata partitions: {}", Arrays.toString(metadataPartitions.toArray())); } final List rows = new ArrayList<>(); @@ -351,34 +351,33 @@ public String validateFiles( } if (metadataPathInfoList.size() != pathInfoList.size()) { - log.error(" FS and metadata files count not matching for " + partition - + ". FS files count " + pathInfoList.size() - + ", metadata base files count " + metadataPathInfoList.size()); + log.error(" FS and metadata files count not matching for {}. FS files count {}, metadata base files count {}", partition, pathInfoList.size(), metadataPathInfoList.size()); } for (Map.Entry entry : pathInfoMap.entrySet()) { if (!metadataPathInfoMap.containsKey(entry.getKey())) { - log.error("FS file not found in metadata " + entry.getKey()); + log.error("FS file not found in metadata {}", entry.getKey()); } else { if (entry.getValue().getLength() != metadataPathInfoMap.get(entry.getKey()).getLength()) { - log.error(" FS file size mismatch " + entry.getKey() + ", size equality " - + (entry.getValue().getLength() - == metadataPathInfoMap.get(entry.getKey()).getLength()) - + ". FS size " + entry.getValue().getLength() - + ", metadata size " + metadataPathInfoMap.get(entry.getKey()).getLength()); + log.error(" FS file size mismatch {}, size equality {}. FS size {}, metadata size {}", + entry.getKey(), + entry.getValue().getLength() == metadataPathInfoMap.get(entry.getKey()).getLength(), + entry.getValue().getLength(), + metadataPathInfoMap.get(entry.getKey()).getLength()); } } } for (Map.Entry entry : metadataPathInfoMap.entrySet()) { if (!pathInfoMap.containsKey(entry.getKey())) { - log.error("Metadata file not found in FS " + entry.getKey()); + log.error("Metadata file not found in FS {}", entry.getKey()); } else { if (entry.getValue().getLength() != pathInfoMap.get(entry.getKey()).getLength()) { - log.error(" Metadata file size mismatch " + entry.getKey() + ", size equality " - + (entry.getValue().getLength() == pathInfoMap.get(entry.getKey()).getLength()) - + ". Metadata size " + entry.getValue().getLength() + ", FS size " - + metadataPathInfoMap.get(entry.getKey()).getLength()); + log.error(" Metadata file size mismatch {}, size equality {}. Metadata size {}, FS size {}", + entry.getKey(), + entry.getValue().getLength() == pathInfoMap.get(entry.getKey()).getLength(), + entry.getValue().getLength(), + metadataPathInfoMap.get(entry.getKey()).getLength()); } } } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java index 6003e936b819d..872ff342dce5b 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java @@ -197,12 +197,12 @@ public void removeCorruptedPendingCleanAction() { try { CleanerUtils.getCleanerPlan(client, instant); } catch (AvroRuntimeException e) { - log.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant); + log.warn("Corruption found. Trying to remove corrupted clean instant file: {}", instant); TimelineUtils.deleteInstantFile(client.getStorage(), client.getTimelinePath(), instant, client.getInstantFileNameGenerator()); } catch (IOException ioe) { if (ioe.getMessage().contains("Not an Avro data file")) { - log.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant); + log.warn("Corruption found. Trying to remove corrupted clean instant file: {}", instant); TimelineUtils.deleteInstantFile(client.getStorage(), client.getTimelinePath(), instant, client.getInstantFileNameGenerator()); } else { @@ -216,7 +216,7 @@ public void removeCorruptedPendingCleanAction() { public void showFailedCommits() { HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); - activeTimeline.filterCompletedInstants().getInstantsAsStream().filter(activeTimeline::isEmpty).forEach(hoodieInstant -> log.warn("Empty Commit: " + hoodieInstant.toString())); + activeTimeline.filterCompletedInstants().getInstantsAsStream().filter(activeTimeline::isEmpty).forEach(hoodieInstant -> log.warn("Empty Commit: {}", hoodieInstant)); } @ShellMethod(key = "repair migrate-partition-meta", value = "Migrate all partition meta file currently stored in text format " diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java index 0e3f7a4029dcd..ed4b75b3924a1 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java @@ -206,8 +206,9 @@ public String fetchTableSchema( TableSchemaResolver tableSchemaResolver = new TableSchemaResolver(client); HoodieSchema schema = tableSchemaResolver.getTableSchema(); if (outputFilePath != null) { - log.info("Latest table schema : " + schema.toString(true)); - writeToFile(outputFilePath, schema.toString(true)); + String schemaStr = schema.toString(true); + log.info("Latest table schema : {}", schemaStr); + writeToFile(outputFilePath, schemaStr); return String.format("Latest table schema written to %s", outputFilePath); } else { return String.format("Latest table schema %s", schema.toString(true)); From 75c587bd6372895858f9a81d79ed44ab5056d82a Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Sat, 4 Jul 2026 09:20:17 -0700 Subject: [PATCH 132/255] test(spark): add streaming source and writer support coverage (#19166) (cherry picked from commit f6ca15ffd011fe6a08fcee8ab9835cc4132577bf) --- .../hudi/functional/TestStreamingSource.scala | 66 ++++++++++++++++++- .../functional/TestStructuredStreaming.scala | 46 +++++++++++++ .../hudi/feature/TestDataSkippingQuery.scala | 62 +++++++++++++++++ 3 files changed, 172 insertions(+), 2 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala index a35d49993bd58..ffaae686eb4d1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala @@ -20,6 +20,7 @@ package org.apache.hudi.functional import org.apache.hudi.DataSourceReadOptions import org.apache.hudi.DataSourceReadOptions.{START_OFFSET, STREAMING_READ_TABLE_VERSION} import org.apache.hudi.DataSourceWriteOptions.{ORDERING_FIELDS, RECORDKEY_FIELD} +import org.apache.hudi.common.config.HoodieReaderConfig import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, MERGE_ON_READ} import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} @@ -294,6 +295,66 @@ class TestStreamingSource extends StreamTest { }) } + /** + * Exercises the legacy incremental streaming path in [[HoodieStreamSourceV1]], which is taken + * when the streaming read table version is below EIGHT and the file group reader is disabled. + * This drives the [[IncrementalRelationV1]] (COW) / [[MergeOnReadIncrementalRelationV1]] (MOR) + * branches of `getBatch` rather than the newer HadoopFsRelation factory path. + */ + private def testLegacyIncrementalStreamSource(tableType: HoodieTableType): Unit = { + withTempDir { inputDir => + val tablePath = s"${inputDir.getCanonicalPath}/test_${tableType.name}_legacy_stream" + HoodieTableMetaClient.newTableBuilder() + .setTableType(tableType) + .setTableName(getTableName(tablePath)) + .setTableVersion(HoodieTableVersion.SIX) + .setRecordKeyFields("id") + .setOrderingFields("ts") + .initTable(HadoopFSUtils.getStorageConf(spark.sessionState.newHadoopConf()), tablePath) + + addData(tablePath, Seq(("1", "a1", "10", "000")), tableVersion = HoodieTableVersion.SIX) + val df = spark.readStream + .format("org.apache.hudi") + .option(WRITE_TABLE_VERSION.key, HoodieTableVersion.SIX.versionCode().toString) + .option(STREAMING_READ_TABLE_VERSION.key, HoodieTableVersion.SIX.versionCode().toString) + // force the legacy (non file-group-reader) incremental relation path + .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key, "false") + .load(tablePath) + .select("id", "name", "price", "ts") + + testStream(df)( + AssertOnQuery { q => q.processAllAvailable(); true }, + CheckAnswerRows(Seq(Row("1", "a1", "10", "000")), lastOnly = true, isSorted = false), + StopStream, + + addDataToQuery(tablePath, + Seq(("2", "a2", "12", "000"), + ("3", "a3", "12", "000")), + tableVersion = HoodieTableVersion.SIX), + StartStream(), + AssertOnQuery { q => q.processAllAvailable(); true }, + CheckAnswerRows( + Seq(Row("2", "a2", "12", "000"), + Row("3", "a3", "12", "000")), + lastOnly = true, isSorted = false), + StopStream, + + addDataToQuery(tablePath, Seq(("4", "a4", "13", "000")), tableVersion = HoodieTableVersion.SIX), + StartStream(), + AssertOnQuery { q => q.processAllAvailable(); true }, + CheckAnswerRows(Seq(Row("4", "a4", "13", "000")), lastOnly = true, isSorted = false) + ) + } + } + + test("test cow stream source with legacy file group reader disabled") { + testLegacyIncrementalStreamSource(COPY_ON_WRITE) + } + + test("test mor stream source with legacy file group reader disabled") { + testLegacyIncrementalStreamSource(MERGE_ON_READ) + } + private def testCheckpointTranslation(tableName: String, tableType: HoodieTableType, writeTableVersion: HoodieTableVersion, @@ -413,9 +474,10 @@ class TestStreamingSource extends StreamTest { } private def addDataToQuery(inputPath: String, - rows: Seq[(String, String, String, String)]): AssertOnQuery = { + rows: Seq[(String, String, String, String)], + tableVersion: HoodieTableVersion = HoodieTableVersion.current): AssertOnQuery = { AssertOnQuery { _=> - addData(inputPath, rows) + addData(inputPath, rows, tableVersion = tableVersion) true } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala index 2ae305a8ccc08..b9a672bb08826 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala @@ -508,6 +508,52 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase { assertEquals(25, metaClient.getActiveTimeline.countInstants()) } + /** + * Injects a failing micro-batch into [[HoodieStreamingSink]] by pointing the record key at a + * non-existent column so that every Hudi write throws. With STREAMING_IGNORE_FAILED_BATCH + * enabled the sink must swallow the failure (unpersisting the write RDDs), let the streaming + * query complete without crashing, and produce no commit. + */ + @Test + def testStructuredStreamingIgnoreFailedBatch(): Unit = { + val (sourcePath, destPath) = initStreamingSourceAndDestPath("source", "dest") + val records1 = recordsToStrings(dataGen.generateInsertsForPartition( + "000", 100, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).asScala.toList + val inputDF1 = spark.read.json(spark.sparkContext.parallelize(records1, 2)) + val schema = inputDF1.schema + inputDF1.coalesce(1).write.mode(SaveMode.Append).json(sourcePath) + + val failingOpts = commonOpts ++ Map( + DataSourceWriteOptions.RECORDKEY_FIELD.key -> "non_existent_field", + DataSourceWriteOptions.STREAMING_IGNORE_FAILED_BATCH.key -> "true", + DataSourceWriteOptions.STREAMING_RETRY_CNT.key -> "1" + ) + + val query = spark.readStream + .schema(schema) + .json(sourcePath) + .writeStream + .format("org.apache.hudi") + .options(failingOpts) + .outputMode(OutputMode.Append) + .option("checkpointLocation", s"$basePath/checkpoint_ignore") + .start(destPath) + + // Must not throw even though the micro-batch fails; the failure is ignored. + query.processAllAvailable() + query.stop() + + // No completed commit must exist since every batch failed and was ignored. + val completedCommits = + try { + HoodieTestUtils.createMetaClient(storage, destPath) + .getActiveTimeline.getCommitsTimeline.filterCompletedInstants().countInstants() + } catch { + case _: TableNotFoundException => 0 + } + assertEquals(0, completedCommits) + } + @ParameterizedTest @CsvSource(Array( "COPY_ON_WRITE,EVENT_TIME_ORDERING", diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala index 60513bd9c0288..38cc323f58c6b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala @@ -208,4 +208,66 @@ class TestDataSkippingQuery extends HoodieSparkSqlTestBase { } } } + + test("Test column stats data skipping across multiple files with range, IN and equality predicates") { + Seq("cow", "mor").foreach { tableType => + withTempDir { tmp => + val tableName = generateTableName + withSQLConf( + "hoodie.metadata.enable" -> "true", + "hoodie.metadata.index.column.stats.enable" -> "true", + "hoodie.enable.data.skipping" -> "true", + "hoodie.metadata.index.column.stats.column.list" -> "id,price", + // keep every commit in its own base file so column-stats pruning has multiple files to skip + "hoodie.parquet.small.file.limit" -> "0" + ) { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | tblproperties (primaryKey = 'id', orderingFields = 'ts', type = '$tableType') + | location '${tmp.getCanonicalPath}' + """.stripMargin) + // Three separate commits, each landing in its own base file with disjoint id / price ranges. + spark.sql(s"insert into $tableName values (1, 'a1', 10, 1000), (2, 'a2', 20, 1000)") + spark.sql(s"insert into $tableName values (11, 'b1', 110, 2000), (12, 'b2', 120, 2000)") + spark.sql(s"insert into $tableName values (21, 'c1', 210, 3000), (22, 'c2', 220, 3000)") + + // Equality on the indexed key column -> only the first file qualifies. + checkAnswer(s"select id, name, price from $tableName where id = 1")( + Seq(1, "a1", 10.0) + ) + // Range predicate that only the last file can satisfy. + checkAnswer(s"select id, name, price from $tableName where id > 20")( + Seq(21, "c1", 210.0), + Seq(22, "c2", 220.0) + ) + // IN predicate touching the first and last files, skipping the middle one. + checkAnswer(s"select id, name, price from $tableName where id in (2, 22)")( + Seq(2, "a2", 20.0), + Seq(22, "c2", 220.0) + ) + // Range on a second indexed column keeps only the middle file. + checkAnswer(s"select id, name, price from $tableName where price >= 110 and price < 210")( + Seq(11, "b1", 110.0), + Seq(12, "b2", 120.0) + ) + // Predicate that matches nothing -> all files pruned, empty result. + checkAnswer(s"select id, name, price from $tableName where id = 999")() + + // Data skipping must not change results compared to a full scan. + withSQLConf("hoodie.enable.data.skipping" -> "false") { + checkAnswer(s"select id, name, price from $tableName where id in (2, 22)")( + Seq(2, "a2", 20.0), + Seq(22, "c2", 220.0) + ) + } + } + } + } + } } From fd3823a42c3c5ad3f3f82dea883db87ed408bd6f Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 02:55:56 +0800 Subject: [PATCH 133/255] refactor(spark): use SLF4J parameterized logging instead of string concatenation (#19184) (cherry picked from commit 79543235ebdcf66ce36caa450ca335a2f2bb5688) --- .../hudi/client/TestHoodieClientMultiWriter.java | 12 ++++++------ ...stDataValidationCheckForLogCompactionActions.java | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index a53f167c2efd4..1b00e1376e4b8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -1030,7 +1030,7 @@ public void testConcurrentCompactionExecutionOnSamePlan() throws Exception { writer1Succeeded.set(true); } catch (Exception e) { // Expected - one writer may fail due to concurrent execution - LOG.info("Writer 1 failed with exception: " + e.getMessage()); + LOG.info("Writer 1 failed with exception: {}", e.getMessage()); writer1Succeeded.set(false); } }); @@ -1045,7 +1045,7 @@ public void testConcurrentCompactionExecutionOnSamePlan() throws Exception { writer2Succeeded.set(true); } catch (Exception e) { // Expected - one writer may fail due to concurrent execution - LOG.info("Writer 2 failed with exception: " + e.getMessage()); + LOG.info("Writer 2 failed with exception: {}", e.getMessage()); writer2Succeeded.set(false); } }); @@ -1528,9 +1528,9 @@ private void runConcurrentAndAssert(JavaRDD writeRecords1, JavaRDD try { ingestBatch(writeFn, client1, newCommitTime1, writeRecords1, runCountDownLatch); } catch (IOException e) { - LOG.error("IOException thrown " + e.getMessage()); + LOG.error("IOException thrown {}", e.getMessage()); } catch (InterruptedException e) { - LOG.error("Interrupted Exception thrown " + e.getMessage()); + LOG.error("Interrupted Exception thrown {}", e.getMessage()); } catch (Exception e) { client1Succeeded.set(false); } @@ -1541,9 +1541,9 @@ private void runConcurrentAndAssert(JavaRDD writeRecords1, JavaRDD try { ingestBatch(writeFn, client2, newCommitTime2, writeRecords2, runCountDownLatch); } catch (IOException e) { - LOG.error("IOException thrown " + e.getMessage()); + LOG.error("IOException thrown {}", e.getMessage()); } catch (InterruptedException e) { - LOG.error("Interrupted Exception thrown " + e.getMessage()); + LOG.error("Interrupted Exception thrown {}", e.getMessage()); } catch (Exception e) { client2Succeeded.set(false); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java index bb6c04a781f16..a0e5d6dce7734 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java @@ -122,12 +122,12 @@ public void stressTestCompactionAndLogCompactionOperations(int seed) throws Exce // Total ingestion writes. int totalWrites = 15; - LOG.warn("Starting trial with seed " + seed); + LOG.warn("Starting trial with seed {}", seed); // Current ingestion commit. int curr = 1; while (curr < totalWrites) { - LOG.warn("Starting write No. " + curr); + LOG.warn("Starting write No. {}", curr); // Pick an action. It can be insert/update/delete and write data to main table. boolean status = writeOnMainTable(mainTable, curr); @@ -145,7 +145,7 @@ public void stressTestCompactionAndLogCompactionOperations(int seed) throws Exce // Verify the records in both the tables. verifyRecords(mainTable, experimentTable); - LOG.warn("For write No." + curr + ", verification passed. Last ingestion commit timestamp is " + mainTable.commitTimeOnMainTable); + LOG.warn("For write No.{}, verification passed. Last ingestion commit timestamp is {}", curr, mainTable.commitTimeOnMainTable); } curr++; } @@ -195,7 +195,7 @@ private boolean writeOnMainTable(TestTableContents mainTable, int curr) throws I result = deleteDataIntoMainTable(mainTable, commitTime); } } catch (IllegalArgumentException e) { - LOG.warn(e.getMessage() + " ignoring current command."); + LOG.warn("{} ignoring current command.", e.getMessage()); return false; } } From 16cdb7dc90cc6db030cbe29d4dae6b37e578cb5e Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 04:27:35 +0800 Subject: [PATCH 134/255] refactor(aws): use SLF4J parameterized logging instead of string concatenation (#19186) (cherry picked from commit 60a3e9a9695795aaa2c662feabd3a33b2975a59a) --- .../apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java | 8 ++++---- .../java/org/apache/hudi/aws/utils/DynamoTableUtils.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index 10f0b5a1592d2..eb0a50ca0638c 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -252,7 +252,7 @@ public List getAllPartitions(String tableName) { @Override public List getPartitionsFromList(String tableName, List partitionList) { if (partitionList.isEmpty()) { - log.info("No partitions to read for " + tableId(this.databaseName, tableName)); + log.info("No partitions to read for {}", tableId(this.databaseName, tableName)); return Collections.emptyList(); } HoodieTimer timer = HoodieTimer.start(); @@ -310,7 +310,7 @@ public void addPartitionsToTable(String tableName, List partitionsToAdd) HoodieTimer timer = HoodieTimer.start(); try { if (partitionsToAdd.isEmpty()) { - log.info("No partitions to add for " + tableId(this.databaseName, tableName)); + log.info("No partitions to add for {}", tableId(this.databaseName, tableName)); return; } Table table = getTable(awsGlue, databaseName, tableName); @@ -373,7 +373,7 @@ public void updatePartitionsToTable(String tableName, List changedPartit HoodieTimer timer = HoodieTimer.start(); try { if (changedPartitions.isEmpty()) { - log.info("No partitions to update for " + tableId(this.databaseName, tableName)); + log.info("No partitions to update for {}", tableId(this.databaseName, tableName)); return; } Table table = getTable(awsGlue, databaseName, tableName); @@ -413,7 +413,7 @@ public void dropPartitions(String tableName, List partitionsToDrop) { HoodieTimer timer = HoodieTimer.start(); try { if (partitionsToDrop.isEmpty()) { - log.info("No partitions to drop for " + tableId(this.databaseName, tableName)); + log.info("No partitions to drop for {}", tableId(this.databaseName, tableName)); return; } parallelizeChange(partitionsToDrop, this.changeParallelism, partitions -> this.dropPartitionsInternal(tableName, partitions), MAX_DELETE_PARTITIONS_PER_REQUEST); diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/DynamoTableUtils.java b/hudi-aws/src/main/java/org/apache/hudi/aws/utils/DynamoTableUtils.java index ace74a1fbc796..1a70419c79d03 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/DynamoTableUtils.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/utils/DynamoTableUtils.java @@ -222,7 +222,7 @@ public static boolean createTableIfNotExists(final DynamoDbClient dynamo, final return true; } catch (final ResourceInUseException e) { if (log.isTraceEnabled()) { - log.trace("Table " + createTableRequest.tableName() + " already exists", e); + log.trace("Table {} already exists", createTableRequest.tableName(), e); } } return false; @@ -240,7 +240,7 @@ public static boolean deleteTableIfExists(final DynamoDbClient dynamo, final Del return true; } catch (final ResourceNotFoundException e) { if (log.isTraceEnabled()) { - log.trace("Table " + deleteTableRequest.tableName() + " does not exist", e); + log.trace("Table {} does not exist", deleteTableRequest.tableName(), e); } } return false; From 51d1c8ade755ddf6bf01f1f1f95e7a69ef18e4b0 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 04:28:57 +0800 Subject: [PATCH 135/255] refactor(utilities): use SLF4J parameterized logging instead of string concatenation (#19185) (cherry picked from commit bf787205bbfbf094600e0856fbe2f0d95aa91c03) --- .../org/apache/hudi/utilities/HoodieDataTableValidator.java | 4 ++-- .../hudi/utilities/sources/helpers/DFSPathSelector.java | 5 ++--- .../utilities/sources/helpers/S3EventsMetaSelector.java | 2 +- .../utilities/functional/TestHoodieSnapshotExporter.java | 2 +- .../hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java | 6 +++--- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java index f34fe8650401e..79ca3b6471940 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java @@ -252,7 +252,7 @@ public static void main(String[] args) { try { validator.run(); } catch (Throwable throwable) { - log.error("Fail to do hoodie Data table validation for " + validator.cfg, throwable); + log.error("Fail to do hoodie Data table validation for {}", validator.cfg, throwable); } finally { jsc.stop(); } @@ -320,7 +320,7 @@ public void doDataTableValidation() { if (!danglingFilePaths.isEmpty() && danglingFilePaths.size() > 0) { log.error("Data table validation failed due to dangling files count {}, found before active timeline", danglingFilePaths.size()); - danglingFilePaths.forEach(entry -> log.error("Dangling file: " + entry.toString())); + danglingFilePaths.forEach(entry -> log.error("Dangling file: {}", entry)); finalResult = false; if (!cfg.ignoreFailed) { throw new HoodieValidationException( diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java index 071499d13a9b0..1c7788efe3e09 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java @@ -91,7 +91,7 @@ public static DFSPathSelector createSourceSelector(TypedProperties props, new Class[] {TypedProperties.class, Configuration.class}, props, conf); - log.info("Using path selector " + selector.getClass().getName()); + log.info("Using path selector {}", selector.getClass().getName()); return selector; } catch (Exception e) { throw new HoodieException("Could not load source selector class " + sourceSelectorClass, e); @@ -125,8 +125,7 @@ public Pair, Checkpoint> getNextFilePathsAndMaxModificationTime(O long sourceLimit) { try { // obtain all eligible files under root folder. - log.info("Root path => " + getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH) - + " source limit => " + sourceLimit); + log.info("Root path => {} source limit => {}", getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH), sourceLimit); long lastCheckpointTime = lastCheckpointStr.map(e -> Long.parseLong(e.getCheckpointKey())).orElse(Long.MIN_VALUE); List eligibleFiles = listEligibleFiles( fs, new Path(getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH)), lastCheckpointTime); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java index b5ccaa58f374a..e3ec3bdc57f86 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java @@ -73,7 +73,7 @@ public static S3EventsMetaSelector createSourceSelector(TypedProperties props) { ReflectionUtils.loadClass( sourceSelectorClass, new Class[] {TypedProperties.class}, props); - log.info("Using path selector " + selector.getClass().getName()); + log.info("Using path selector {}", selector.getClass().getName()); return selector; } catch (Exception e) { throw new HoodieException("Could not load source selector class " + sourceSelectorClass, e); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java index 542349240189e..ebe26c89e9c34 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java @@ -112,7 +112,7 @@ public void init() throws Exception { } List pathInfoList = storage.listFiles(new StoragePath(sourcePath)); for (StoragePathInfo pathInfo : pathInfoList) { - LOG.info(">>> Prepared test file: " + pathInfo.getPath()); + LOG.info(">>> Prepared test file: {}", pathInfo.getPath()); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java index 4731c269040b1..634c434d3845f 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java @@ -110,7 +110,7 @@ static class TestHelpers { static void assertNCompletedCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getWriteTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants=" + meta.getActiveTimeline().getInstants()); + LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCommits = timeline.countInstants(); assertEquals(expected, numCommits, "Got=" + numCommits + ", exp =" + expected); } @@ -118,7 +118,7 @@ static void assertNCompletedCommits(int expected, String tablePath) { static void assertNCleanCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCleanerTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants=" + meta.getActiveTimeline().getInstants()); + LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCleanCommits = timeline.countInstants(); assertEquals(expected, numCleanCommits, "Got=" + numCleanCommits + ", exp =" + expected); } @@ -126,7 +126,7 @@ static void assertNCleanCommits(int expected, String tablePath) { static void assertNClusteringCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCompletedReplaceTimeline(); - LOG.info("Timeline Instants=" + meta.getActiveTimeline().getInstants()); + LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCommits = timeline.countInstants(); assertEquals(expected, numCommits, "Got=" + numCommits + ", exp =" + expected); } From 6f8b046e137d96dcc262a07a788dd13a8e1a9d73 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 04:37:30 +0800 Subject: [PATCH 136/255] refactor(examples): use SLF4J parameterized logging instead of string concatenation (#19187) (cherry picked from commit 51b57330a8e93c30b39a721e5da8aa9dec7619b3) --- .../hudi/examples/spark/HoodieWriteClientExample.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/spark/HoodieWriteClientExample.java b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/spark/HoodieWriteClientExample.java index e4300af3c5efc..1f431563aa5f0 100644 --- a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/spark/HoodieWriteClientExample.java +++ b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/spark/HoodieWriteClientExample.java @@ -101,7 +101,7 @@ public static void main(String[] args) throws Exception { // inserts String newCommitTime = client.startCommit(); - log.info("Starting commit " + newCommitTime); + log.info("Starting commit {}", newCommitTime); List> records = dataGen.generateInserts(newCommitTime, 10); List> recordsSoFar = new ArrayList<>(records); @@ -110,7 +110,7 @@ public static void main(String[] args) throws Exception { // updates newCommitTime = client.startCommit(); - log.info("Starting commit " + newCommitTime); + log.info("Starting commit {}", newCommitTime); List> toBeUpdated = dataGen.generateUpdates(newCommitTime, 2); records.addAll(toBeUpdated); recordsSoFar.addAll(toBeUpdated); @@ -119,7 +119,7 @@ public static void main(String[] args) throws Exception { // Delete newCommitTime = client.startCommit(); - log.info("Starting commit " + newCommitTime); + log.info("Starting commit {}", newCommitTime); // just delete half of the records int numToDelete = recordsSoFar.size() / 2; List toBeDeleted = recordsSoFar.stream().map(HoodieRecord::getKey).limit(numToDelete).collect(Collectors.toList()); @@ -128,7 +128,7 @@ public static void main(String[] args) throws Exception { // Delete by partition newCommitTime = client.startCommit(HoodieTimeline.REPLACE_COMMIT_ACTION); - log.info("Starting commit " + newCommitTime); + log.info("Starting commit {}", newCommitTime); // The partition where the data needs to be deleted List partitionList = toBeDeleted.stream().map(s -> s.getPartitionPath()).distinct().collect(Collectors.toList()); List deleteList = recordsSoFar.stream().filter(f -> !partitionList.contains(f.getPartitionPath())) From 18a2eb849fea7babf2dc56656686ee35d244c393 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Sat, 4 Jul 2026 17:03:33 -0700 Subject: [PATCH 137/255] fix(spark): make export_instants desc ordering work (#19172) (cherry picked from commit 8347be48ce31ef3aee70964a9d0931e32b857136) --- .../procedures/ExportInstantsProcedure.scala | 14 +- .../TestExportInstantsProcedure.scala | 122 +++++++++++++++--- 2 files changed, 110 insertions(+), 26 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala index 5ba9960d1d3c4..3281e9de653b6 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala @@ -87,17 +87,19 @@ class ExportInstantsProcedure extends BaseProcedure with ProcedureBuilder with L if (!new File(localFolder).isDirectory) throw new HoodieException(localFolder + " is not a valid local directory") - // The non archived instants can be listed from the Timeline. - val nonArchivedInstants: util.List[HoodieInstant] = metaClient + // The non archived instants can be listed from the Timeline. Wrap in a mutable list because the + // desc branch below reverses it in place via Collections.reverse, which fails on the read-only + // list produced by asJava over an immutable Scala collection. + val nonArchivedInstants: util.List[HoodieInstant] = new util.ArrayList[HoodieInstant](metaClient .getActiveTimeline .filterCompletedInstants.getInstants.iterator().asScala .filter((i: HoodieInstant) => actionSet.contains(i.getAction)) - .toList.asJava + .toList.asJava) - // Archived instants are in the commit archive files + // Archived instants are in the commit archive files (also mutable, for the same reason). val statuses: Array[FileStatus] = HadoopFSUtils.getFs(basePath, jsc.hadoopConfiguration()).globStatus(archivePath) - val archivedStatuses = List(statuses: _*) - .sortWith((f1, f2) => (f1.getModificationTime - f2.getModificationTime).toInt > 0).asJava + val archivedStatuses = new util.ArrayList[FileStatus](List(statuses: _*) + .sortWith((f1, f2) => (f1.getModificationTime - f2.getModificationTime).toInt > 0).asJava) if (desc) { Collections.reverse(nonArchivedInstants) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestExportInstantsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestExportInstantsProcedure.scala index 372c6cc5e7c55..2b5a9357473f0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestExportInstantsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestExportInstantsProcedure.scala @@ -17,34 +17,116 @@ package org.apache.spark.sql.hudi.procedure +import org.apache.spark.sql.Row + +import java.io.File + class TestExportInstantsProcedure extends HoodieSparkProcedureTestBase { + private def createCowTable(tableName: String, path: String): Unit = { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | location '$path' + | tblproperties ( + | primaryKey = 'id', + | orderingFields = 'ts' + | ) + """.stripMargin) + } + + private def newExportDir(tmp: File, name: String): File = { + val dir = new File(tmp, name) + assert(dir.mkdirs(), s"Failed to create export dir $dir") + dir + } + + private def exportedCount(result: Array[Row]): Int = { + assertResult(1)(result.length) + val detail = result.head.getString(0) + val matched = "Exported (\\d+) Instants".r.findFirstMatchIn(detail) + assert(matched.isDefined, s"Unexpected export detail: $detail") + matched.get.group(1).toInt + } + test("Test Call export_instants Procedure") { withTempDir { tmp => val tableName = generateTableName - // create table - spark.sql( - s""" - |create table $tableName ( - | id int, - | name string, - | price double, - | ts long - |) using hudi - | location '${tmp.getCanonicalPath}/$tableName' - | tblproperties ( - | primaryKey = 'id', - | orderingFields = 'ts' - | ) - """.stripMargin) + createCowTable(tableName, s"${tmp.getCanonicalPath}/$tableName") + + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + + val exportDir = newExportDir(tmp, "export_basic") + val result = spark.sql( + s"""call export_instants(table => '$tableName', local_folder => '${exportDir.getCanonicalPath}')""").collect() + + // A single insert produces one exportable commit instant that is written to disk. + assertResult(1)(exportedCount(result)) + assertResult(1)(exportDir.listFiles().count(_.getName.endsWith(".commit"))) + } + } + + test("Test Call export_instants Procedure with desc ordering") { + withTempDir { tmp => + val tableName = generateTableName + createCowTable(tableName, s"${tmp.getCanonicalPath}/$tableName") + + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + spark.sql(s"insert into $tableName select 2, 'a2', 20, 2000") + spark.sql(s"insert into $tableName select 3, 'a3', 30, 3000") + + val exportDir = newExportDir(tmp, "export_desc") + val result = spark.sql( + s"""call export_instants(table => '$tableName', + | local_folder => '${exportDir.getCanonicalPath}', desc => true)""".stripMargin).collect() + + // The desc branch reverses the active instants and exports all three commits to disk. + assertResult(3)(exportedCount(result)) + assertResult(3)(exportDir.listFiles().count(_.getName.endsWith(".commit"))) + } + } + + test("Test Call export_instants Procedure filters by action") { + withTempDir { tmp => + val tableName = generateTableName + createCowTable(tableName, s"${tmp.getCanonicalPath}/$tableName") + + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + spark.sql(s"insert into $tableName select 2, 'a2', 20, 2000") - // insert data to table + // Restricting to an action that is not present exports nothing. + val cleanDir = newExportDir(tmp, "export_clean_only") + val cleanResult = spark.sql( + s"""call export_instants(table => '$tableName', + | local_folder => '${cleanDir.getCanonicalPath}', actions => 'clean')""".stripMargin).collect() + assertResult(0)(exportedCount(cleanResult)) + assertResult(0)(cleanDir.listFiles().count(_.getName.endsWith(".clean"))) + + // Restricting to the commit action exports exactly the commit instants. + val commitDir = newExportDir(tmp, "export_commit_only") + val commitResult = spark.sql( + s"""call export_instants(table => '$tableName', + | local_folder => '${commitDir.getCanonicalPath}', actions => 'commit')""".stripMargin).collect() + assertResult(2)(exportedCount(commitResult)) + assertResult(2)(commitDir.listFiles().count(_.getName.endsWith(".commit"))) + } + } + + test("Test Call export_instants Procedure with an invalid local folder") { + withTempDir { tmp => + val tableName = generateTableName + createCowTable(tableName, s"${tmp.getCanonicalPath}/$tableName") spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") - val result = spark.sql(s"""call export_instants(table => '$tableName', local_folder => '${tmp.getCanonicalPath}/$tableName')""").limit(1).collect() - assertResult(1) { - result.length - } + val notADir = new File(tmp, "does_not_exist").getCanonicalPath + checkExceptionContain( + s"""call export_instants(table => '$tableName', local_folder => '$notADir')""")( + "is not a valid local directory") } } } From ab746b31fd3e6873aee99072f1b9795834ac7103 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 12:04:52 +0800 Subject: [PATCH 138/255] refactor(sync): use SLF4J parameterized logging instead of string concatenation (#19189) Convert log string concatenation to {} placeholders in HiveSyncTool and QueryBasedDDLExecutor. Mechanical and behavior-preserving. (cherry picked from commit 905fcdda42ac6147594b73c222bb960e540cfb22) --- .../src/main/java/org/apache/hudi/hive/HiveSyncTool.java | 2 +- .../java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 2a130d8ee7972..fc07512c353b1 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -555,7 +555,7 @@ private boolean syncPartitions(String tableName, List partitionE List touchPartitions = config.getBoolean(META_SYNC_TOUCH_PARTITIONS_ENABLED) ? filterPartitions(partitionEventList, PartitionEventType.TOUCH) : Collections.emptyList(); if (!touchPartitions.isEmpty()) { - log.info("Touch Partitions " + touchPartitions); + log.info("Touch Partitions {}", touchPartitions); syncClient.touchPartitionsToTable(tableName, touchPartitions); } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java index 7f776f2f7a04d..a4e3ee3091543 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java @@ -205,10 +205,10 @@ public String getPartitionClause(String partition) { @Override public void touchPartitionsToTable(String tableName, List touchPartitions) { if (touchPartitions.isEmpty()) { - log.info("No partitions to touch for " + tableName); + log.info("No partitions to touch for {}", tableName); return; } - log.info("Touching partitions " + touchPartitions.size() + " on " + tableName); + log.info("Touching partitions {} on {}", touchPartitions.size(), tableName); List sqls = constructPartitionAlterStatements(tableName, touchPartitions, PartitionAlterType.TOUCH); for (String sql : sqls) { runSQL(sql); From 5241c4860c9aa9219ceae5f60535eed67b9959ad Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 12:10:45 +0800 Subject: [PATCH 139/255] refactor(client): use SLF4J parameterized logging instead of string concatenation (#19155) * refactor(client): use SLF4J parameterized logging instead of string concatenation Convert `+`-concatenated log messages to SLF4J `{}` placeholders across hudi-client (client-common, java-client, spark-client, flink-client). The rewrite is behaviour-preserving: each concatenated expression becomes a placeholder argument in order, and a trailing throwable is kept as the last argument so its stack trace is still recorded. This also avoids building the message string eagerly when the log level is disabled. * refactor(client): drop redundant .toString() in parameterized logs With {} templating SLF4J calls toString() lazily on the argument, so the explicit calls were redundant. Removed in TimelineArchiverV1, HoodieCompactionPlanGenerator and the matching TestLegacyArchivedMetaEntryReader log. Addresses review nits on #19155. * refactor(client): wrap long merge log statement for readability Keep the format string and its arguments on separate lines in HoodieBinaryCopyHandle.write() instead of one ~200-char line, per review. (cherry picked from commit 865a554be39e98bc9af0d1f2c4e29a99c401ecb4) --- .../hudi/async/AsyncClusteringService.java | 2 +- .../apache/hudi/async/AsyncCompactService.java | 6 +++--- .../apache/hudi/async/HoodieAsyncService.java | 2 +- .../client/BaseHoodieTableServiceClient.java | 2 +- .../hudi/client/CompactionAdminClient.java | 5 ++--- .../HoodieTableServiceManagerClient.java | 2 +- .../selector/BootstrapRegexModeSelector.java | 2 +- .../embedded/EmbeddedTimelineService.java | 2 +- .../versioning/v1/TimelineArchiverV1.java | 10 +++++----- ...ntFileWritesConflictResolutionStrategy.java | 3 +-- .../DirectMarkerTransactionManager.java | 10 ++++------ ...ntFileWritesConflictResolutionStrategy.java | 6 ++---- .../lock/FileSystemBasedLockProvider.java | 2 +- .../client/transaction/lock/LockManager.java | 2 +- .../hudi/client/utils/TransactionUtils.java | 3 +-- .../validator/StreamingOffsetValidator.java | 2 +- .../FileMetadataWriteStatusConverter.java | 2 +- .../bucket/ConsistentBucketIndexUtils.java | 4 ++-- .../hudi/index/bucket/HoodieBucketIndex.java | 2 +- .../org/apache/hudi/io/BaseCreateHandle.java | 4 ++-- .../io/ExternalFileClusteringWriteHandle.java | 2 +- .../org/apache/hudi/io/HoodieAppendHandle.java | 2 +- .../apache/hudi/io/HoodieBinaryCopyHandle.java | 10 +++++----- .../main/java/org/apache/hudi/io/IOUtils.java | 2 +- ...ckedTableMetadataWriterTableVersionSix.java | 2 +- .../org/apache/hudi/table/HoodieTable.java | 6 +++--- .../CommitBasedClusteringPlanStrategy.java | 6 +++--- .../PartitionAwareClusteringPlanStrategy.java | 10 +++++----- .../commit/BaseCommitActionExecutor.java | 8 ++++---- .../ScheduleCompactionActionExecutor.java | 4 ++-- .../HoodieCompactionPlanGenerator.java | 2 +- .../index/AbstractIndexingCatchupTask.java | 2 +- .../restore/BaseRestoreActionExecutor.java | 4 ++-- .../rollback/BaseRollbackActionExecutor.java | 6 +++--- .../CopyOnWriteRollbackActionExecutor.java | 6 +++--- .../rollback/ListingBasedRollbackStrategy.java | 2 +- .../MergeOnReadRollbackActionExecutor.java | 8 ++++---- .../TimelineServerBasedWriteMarkers.java | 6 ++---- .../table/upgrade/FiveToSixUpgradeHandler.java | 4 ++-- .../hudi/table/upgrade/UpgradeDowngrade.java | 2 +- .../TestLegacyArchivedMetaEntryReader.java | 4 ++-- .../action/rollback/TestRollbackHelper.java | 2 +- .../testutils/HoodieWriteableTestTable.java | 2 +- .../client/HoodieFlinkTableServiceClient.java | 4 ++-- .../org/apache/hudi/io/FlinkCreateHandle.java | 2 +- .../storage/row/HoodieRowDataCreateHandle.java | 4 ++-- .../FlinkPartitionTTLActionExecutor.java | 2 +- .../commit/BaseJavaCommitActionExecutor.java | 4 ++-- .../action/commit/JavaUpsertPartitioner.java | 18 ++++++++---------- .../BaseJavaDeltaCommitActionExecutor.java | 4 ++-- .../client/TestJavaHoodieBackedMetadata.java | 12 ++++++------ .../TestJavaCopyOnWriteActionExecutor.java | 2 +- .../testutils/HoodieJavaClientTestHarness.java | 2 +- ...kBinaryCopyClusteringExecutionStrategy.java | 2 +- ...xternalFileClusteringExecutionStrategy.java | 2 +- .../validator/SparkPreCommitValidator.java | 2 +- .../SqlQueryEqualityPreCommitValidator.java | 8 ++++---- .../SqlQueryInequalityPreCommitValidator.java | 8 ++++---- ...SqlQuerySingleResultPreCommitValidator.java | 4 ++-- .../io/storage/row/HoodieRowCreateHandle.java | 2 +- .../BaseBootstrapMetadataHandler.java | 2 +- .../SparkBootstrapCommitActionExecutor.java | 8 +++----- ...estHoodieSparkEngineDynamicRepartition.java | 2 +- .../hudi/io/TestHoodieTimelineArchiver.java | 2 +- .../TestTimelineServerBasedWriteMarkers.java | 4 ++-- 65 files changed, 131 insertions(+), 144 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java index 2bcd851208fb2..2e2588ed28e4f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java @@ -71,7 +71,7 @@ protected Pair startService() { return Pair.of(CompletableFuture.allOf(IntStream.range(0, maxConcurrentClustering).mapToObj(i -> CompletableFuture.supplyAsync(() -> { try { // Set Compactor Pool Name for allowing users to prioritize compaction - log.info("Setting pool name for clustering to " + CLUSTERING_POOL_NAME); + log.info("Setting pool name for clustering to {}", CLUSTERING_POOL_NAME); context.setProperty(EngineProperty.CLUSTERING_POOL_NAME, CLUSTERING_POOL_NAME); while (!isShutdownRequested()) { final String instant = fetchNextAsyncServiceInstant(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java index 52088d8d683e6..6298c01e4ded8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java @@ -71,16 +71,16 @@ protected Pair startService() { return Pair.of(CompletableFuture.allOf(IntStream.range(0, maxConcurrentCompaction).mapToObj(i -> CompletableFuture.supplyAsync(() -> { try { // Set Compactor Pool Name for allowing users to prioritize compaction - log.info("Setting pool name for compaction to " + COMPACT_POOL_NAME); + log.info("Setting pool name for compaction to {}", COMPACT_POOL_NAME); context.setProperty(EngineProperty.COMPACTION_POOL_NAME, COMPACT_POOL_NAME); while (!isShutdownRequested()) { final String instantTime = fetchNextAsyncServiceInstant(); if (null != instantTime) { - log.info("Starting Compaction for instant " + instantTime); + log.info("Starting Compaction for instant {}", instantTime); compactor.compact(instantTime); - log.info("Finished Compaction for instant " + instantTime); + log.info("Finished Compaction for instant {}", instantTime); } } log.info("Compactor shutting down properly!!"); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java index 917156af89a05..4f207b026e882 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java @@ -186,7 +186,7 @@ public void waitTillPendingAsyncServiceInstantsReducesTo(int numPending) throws * @param instantTime {@link String} to enqueue. */ public void enqueuePendingAsyncServiceInstant(String instantTime) { - log.info("Enqueuing new pending table service instant: " + instantTime); + log.info("Enqueuing new pending table service instant: {}", instantTime); pendingInstants.add(instantTime); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index 01105c8bd8d76..739e11933a38b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -1432,7 +1432,7 @@ private Option delegateToTableServiceManager(TableServiceType tableServi case CLEAN: return tableServiceManagerClient.executeClean(); default: - log.info("Not supported delegate to table service manager, tableServiceType : " + tableServiceType.getAction()); + log.info("Not supported delegate to table service manager, tableServiceType : {}", tableServiceType.getAction()); return Option.empty(); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java index 32be4b2741390..6abe10c6cca75 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java @@ -299,13 +299,12 @@ private List runRenamingOps(HoodieTableMetaClient metaClient, context.setJobStatus(this.getClass().getSimpleName(), "Execute unschedule operations: " + config.getTableName()); return context.map(renameActions, lfPair -> { try { - log.info("RENAME " + lfPair.getLeft().getPath() + " => " + lfPair.getRight().getPath()); + log.info("RENAME {} => {}", lfPair.getLeft().getPath(), lfPair.getRight().getPath()); renameLogFile(metaClient, lfPair.getLeft(), lfPair.getRight()); return new RenameOpResult(lfPair, true, Option.empty()); } catch (IOException e) { log.error("Error renaming log file", e); - log.error("\n\n\n***NOTE Compaction is in inconsistent state. Try running \"compaction repair " - + lfPair.getLeft().getDeltaCommitTime() + "\" to recover from failure ***\n\n\n"); + log.error("\n\n\n***NOTE Compaction is in inconsistent state. Try running \"compaction repair {}\" to recover from failure ***\n\n\n", lfPair.getLeft().getDeltaCommitTime()); return new RenameOpResult(lfPair, false, Option.of(e)); } }, parallelism); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java index 4f13034c89df8..ec7dd1b30b0e8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java @@ -93,7 +93,7 @@ private String executeRequest(String requestPath, Map queryParam queryParameters.forEach(builder::addParameter); String url = builder.toString(); - log.info("Sending request to table management service : (" + url + ")"); + log.info("Sending request to table management service : ({})", url); int timeoutMs = this.config.getConnectionTimeoutSec() * 1000; int requestRetryLimit = config.getConnectionRetryLimit(); int connectionRetryDelay = config.getConnectionRetryDelay(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/selector/BootstrapRegexModeSelector.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/selector/BootstrapRegexModeSelector.java index 65fda8e6cf7df..6869016cf8851 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/selector/BootstrapRegexModeSelector.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/selector/BootstrapRegexModeSelector.java @@ -48,7 +48,7 @@ public BootstrapRegexModeSelector(HoodieWriteConfig writeConfig) { this.bootstrapModeOnMatch = writeConfig.getBootstrapModeForRegexMatch(); this.defaultMode = BootstrapMode.FULL_RECORD.equals(bootstrapModeOnMatch) ? BootstrapMode.METADATA_ONLY : BootstrapMode.FULL_RECORD; - log.info("Default Mode :" + defaultMode + ", on Match Mode :" + bootstrapModeOnMatch); + log.info("Default Mode :{}, on Match Mode :{}", defaultMode, bootstrapModeOnMatch); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index 2df9d0940d5b2..7caeab7ead3d9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -97,7 +97,7 @@ static EmbeddedTimelineService getOrStartEmbeddedTimelineService(HoodieEngineCon synchronized (SERVICE_LOCK) { if (RUNNING_SERVICES.containsKey(timelineServiceIdentifier)) { RUNNING_SERVICES.get(timelineServiceIdentifier).addBasePath(writeConfig.getBasePath()); - log.info("Reusing existing embedded timeline server with configuration: " + RUNNING_SERVICES.get(timelineServiceIdentifier).serviceConfig); + log.info("Reusing existing embedded timeline server with configuration: {}", RUNNING_SERVICES.get(timelineServiceIdentifier).serviceConfig); return RUNNING_SERVICES.get(timelineServiceIdentifier); } // if no compatible instance is found, create a new one diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java index bab78324d7ded..579b2a8c6c87d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java @@ -359,7 +359,7 @@ private List getInstantsToArchive() throws IOException { log.info("Not archiving as there is no compaction yet on the metadata table"); instants = Stream.empty(); } else { - log.info("Limiting archiving of instants to latest compaction on metadata table at " + latestCompactionTime.get()); + log.info("Limiting archiving of instants to latest compaction on metadata table at {}", latestCompactionTime.get()); instants = instants.filter(instant -> compareTimestamps(instant.requestedTime(), LESSER_THAN, latestCompactionTime.get())); } @@ -419,7 +419,7 @@ private List getInstantsToArchive() throws IOException { } private boolean deleteArchivedInstants(List archivedInstants, HoodieEngineContext context) throws IOException { - log.info("Deleting instants " + archivedInstants); + log.info("Deleting instants {}", archivedInstants); List pendingInstants = new ArrayList<>(); List completedInstants = new ArrayList<>(); @@ -463,7 +463,7 @@ private boolean deleteArchivedInstants(List archivedInstants, Hoo public void archive(HoodieEngineContext context, List instants) throws HoodieCommitException { try { Schema wrapperSchema = HoodieArchivedMetaEntry.getClassSchema(); - log.info("Wrapper schema " + wrapperSchema.toString()); + log.info("Wrapper schema {}", wrapperSchema); List records = new ArrayList<>(); for (HoodieInstant hoodieInstant : instants) { try { @@ -474,7 +474,7 @@ public void archive(HoodieEngineContext context, List instants) t } } catch (Exception e) { InstantFileNameGenerator fileNameFactory = new InstantFileNameGeneratorV1(); - log.error("Failed to archive commits, .commit file: " + fileNameFactory.getFileName(hoodieInstant), e); + log.error("Failed to archive commits, .commit file: {}", fileNameFactory.getFileName(hoodieInstant), e); if (this.config.isFailOnTimelineArchivingEnabled()) { throw e; } @@ -489,7 +489,7 @@ public void archive(HoodieEngineContext context, List instants) t private void deleteAnyLeftOverMarkers(HoodieEngineContext context, HoodieInstant instant) { WriteMarkers writeMarkers = WriteMarkersFactory.get(config.getMarkersType(), table, instant.requestedTime()); if (writeMarkers.deleteMarkerDir(context, config.getMarkersDeleteParallelism())) { - log.info("Cleaned up left over marker directory for instant :" + instant); + log.info("Cleaned up left over marker directory for instant :{}", instant); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java index 54112abd75eb2..01b071f714a17 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java @@ -51,8 +51,7 @@ public boolean hasConflict(ConcurrentOperation thisOperation, ConcurrentOperatio Set intersection = new HashSet<>(partitionBucketIdSetForFirstInstant); intersection.retainAll(partitionBucketIdSetForSecondInstant); if (!intersection.isEmpty()) { - log.info("Found conflicting writes between first operation = " + thisOperation - + ", second operation = " + otherOperation + " , intersecting bucket ids " + intersection); + log.info("Found conflicting writes between first operation = {}, second operation = {} , intersecting bucket ids {}", thisOperation, otherOperation, intersection); return true; } return false; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java index 02b027f12d31f..90c5e963da1e2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java @@ -48,22 +48,20 @@ public DirectMarkerTransactionManager(HoodieWriteConfig config, HoodieStorage st public void beginTransaction(String newTxnOwnerInstantTime, InstantGenerator instantGenerator) { if (isLockRequired) { - LOG.info("Transaction starting for " + newTxnOwnerInstantTime + " and " + filePath); + LOG.info("Transaction starting for {} and {}", newTxnOwnerInstantTime, filePath); lockManager.lock(); reset(changeActionInstant, Option.of(getInstant(newTxnOwnerInstantTime, instantGenerator)), Option.empty()); - LOG.info("Transaction started for " + newTxnOwnerInstantTime + " and " + filePath); + LOG.info("Transaction started for {} and {}", newTxnOwnerInstantTime, filePath); } } public void endTransaction(String currentTxnOwnerInstantTime, InstantGenerator instantGenerator) { if (isLockRequired) { - LOG.info("Transaction ending with transaction owner " + currentTxnOwnerInstantTime - + " for " + filePath); + LOG.info("Transaction ending with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); if (reset(Option.of(getInstant(currentTxnOwnerInstantTime, instantGenerator)), Option.empty(), Option.empty())) { lockManager.unlock(); - LOG.info("Transaction ended with transaction owner " + currentTxnOwnerInstantTime - + " for " + filePath); + LOG.info("Transaction ended with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); } } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java index e2eaa53103036..0d6327272a19f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java @@ -142,8 +142,7 @@ public boolean hasConflict(ConcurrentOperation thisOperation, ConcurrentOperatio Set> intersection = new HashSet<>(partitionAndFileIdsSetForFirstInstant); intersection.retainAll(partitionAndFileIdsSetForSecondInstant); if (!intersection.isEmpty()) { - log.info("Found conflicting writes between first operation = " + thisOperation - + ", second operation = " + otherOperation + " , intersecting file ids " + intersection); + log.info("Found conflicting writes between first operation = {}, second operation = {} , intersecting file ids {}", thisOperation, otherOperation, intersection); return true; } return false; @@ -163,8 +162,7 @@ private boolean isRollbackConflict(ConcurrentOperation thisOperation, Concurrent String rolledbackCommit = otherOperation.getRolledbackCommit(); String thisCommitTimestamp = thisOperation.getInstantTimestamp(); if (rolledbackCommit != null && rolledbackCommit.equals(thisCommitTimestamp)) { - log.error("Found rollback conflict: rollback operation " + otherOperation - + " is rolling back commit " + thisCommitTimestamp + " created by operation " + thisOperation); + log.error("Found rollback conflict: rollback operation {} is rolling back commit {} created by operation {}", otherOperation, thisCommitTimestamp, thisOperation); return true; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java index fa7fde5175083..235f254cd914e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java @@ -169,7 +169,7 @@ private boolean checkIfExpired() { return true; } } catch (IOException | HoodieIOException e) { - log.error(generateLogStatement(LockState.ALREADY_RELEASED) + " failed to get lockFile's modification time", e); + log.error("{} failed to get lockFile's modification time", generateLogStatement(LockState.ALREADY_RELEASED), e); } return false; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java index 21eb5da615758..6be1ebd7c911a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java @@ -109,7 +109,7 @@ public void unlock() { public synchronized LockProvider getLockProvider() { // Perform lazy initialization of lock provider only if needed if (lockProvider == null) { - log.info("LockProvider " + writeConfig.getLockProviderClass()); + log.info("LockProvider {}", writeConfig.getLockProviderClass()); // Try to load lock provider with HoodieLockMetrics constructor first Class[] metricsConstructorTypes = {LockConfiguration.class, StorageConfiguration.class, HoodieLockMetrics.class}; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java index 6b5ac8c575aa4..cca0486799fdd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java @@ -89,8 +89,7 @@ public static Option resolveWriteConflictIfAny( try { ConcurrentOperation otherOperation = new ConcurrentOperation(instant, table.getMetaClient()); if (resolutionStrategy.hasConflict(thisOperation, otherOperation)) { - log.info("Conflict encountered between current instant = " + thisOperation + " and instant = " - + otherOperation + ", attempting to resolve it..."); + log.info("Conflict encountered between current instant = {} and instant = {}, attempting to resolve it...", thisOperation, otherOperation); resolutionStrategy.resolveConflict(table, thisOperation, otherOperation); } } catch (IOException io) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java index 40e7a4f1635f9..0313d57c30c71 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/StreamingOffsetValidator.java @@ -210,7 +210,7 @@ protected void validateOffsetConsistency(long offsetDiff, long recordsWritten, l previousCheckpoint, currentCheckpoint); if (failurePolicy == ValidationFailurePolicy.WARN_LOG) { - log.warn(errorMsg + " (failure policy is WARN_LOG, commit will proceed)"); + log.warn("{} (failure policy is WARN_LOG, commit will proceed)", errorMsg); } else { throw new HoodieValidationException(errorMsg); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java index 51ac55cd5dc4d..8e96b28adc3b0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java @@ -60,7 +60,7 @@ public FileMetadataWriteStatusConverter(HoodieTable hoodieTable, Hoo */ public WriteStatus convert(String parquetFile, String partitionPath, Map executionConfigs) throws IOException { - LOG.info("Creating write status for parquet file " + parquetFile); + LOG.info("Creating write status for parquet file {}", parquetFile); WriteStatus writeStatus = (WriteStatus) ReflectionUtils.loadClass(this.writeConfig.getWriteStatusClassName(), this.hoodieTable.shouldTrackSuccessRecords(), this.writeConfig.getWriteStatusFailureFraction(), this.hoodieTable.isMetadataTable()); StoragePath parquetFilePath = new StoragePath(parquetFile); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java index 5d02de2cbcfd3..b522a77af83c6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java @@ -175,7 +175,7 @@ public static Option loadMetadata(HoodieTable t } catch (FileNotFoundException e) { return Option.empty(); } catch (IOException e) { - log.error("Error when loading hashing metadata, partition: " + partition, e); + log.error("Error when loading hashing metadata, partition: {}", partition, e); throw new HoodieIndexException("Error while loading hashing metadata", e); } } @@ -258,7 +258,7 @@ private static Option loadMetadataFromGivenFile } catch (FileNotFoundException e) { return Option.empty(); } catch (IOException e) { - log.error("Error when loading hashing metadata, for path: " + metaFile.getPath().getName(), e); + log.error("Error when loading hashing metadata, for path: {}", metaFile.getPath().getName(), e); throw new HoodieIndexException("Error while loading hashing metadata", e); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java index 7b9c28b462169..be61454fae36c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieBucketIndex.java @@ -58,7 +58,7 @@ public HoodieBucketIndex(HoodieWriteConfig config) { this.numBuckets = config.getBucketIndexNumBuckets(); this.indexKeyFields = KeyGenUtils.getIndexKeyFields(config.getBucketIndexHashField()); - log.info("Use bucket index, numBuckets = " + numBuckets + ", indexFields: " + indexKeyFields); + log.info("Use bucket index, numBuckets = {}, indexFields: {}", numBuckets, indexKeyFields); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java index 8144ae4c2f859..e24da085a613a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java @@ -117,7 +117,7 @@ protected void doWrite(HoodieRecord record, HoodieSchema schema, TypedProperties // record successful. record.deflate(); } catch (Throwable t) { - log.error("Error writing record " + record, t); + log.error("Error writing record {}", record, t); if (!config.getIgnoreWriteFailed()) { throw new HoodieException(t.getMessage(), t); } @@ -178,7 +178,7 @@ public IOType getIOType() { */ @Override public List close() { - log.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten); + log.info("Closing the file {} as we are done with all the records {}", writeStatus.getFileId(), recordsWritten); try { if (isClosed()) { // Handle has already been closed diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java index 9c9a5a3f0ba1b..f947ff9bc2b96 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java @@ -62,7 +62,7 @@ public ExternalFileClusteringWriteHandle(HoodieWriteConfig config, String instan // Create inProgress marker file createMarkerFile(partitionPath, path.getName()); - LOG.info("New ExternalFileClusteringWriteHandle for partition :" + partitionPath + " with fileId " + fileId); + LOG.info("New ExternalFileClusteringWriteHandle for partition :{} with fileId {}", partitionPath, fileId); } /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index b6389d9122f07..b179bd3ea5bf2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -260,7 +260,7 @@ private void init(HoodieRecord record) { ? getInstantTimeForLogFile(record) : deltaWriteStat.getPrevCommit(); this.writer = createLogWriter(instantTime, fileSliceOpt); } catch (Exception e) { - log.error("Error in update task at commit " + instantTime, e); + log.error("Error in update task at commit {}", instantTime, e); writeStatus.setGlobalError(e); throw new HoodieUpsertException("Failed to initialize HoodieAppendHandle for FileId: " + fileId + " on commit " + instantTime + " on storage path " + hoodieTable.getMetaClient().getBasePath() + "/" + partitionPath, e); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBinaryCopyHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBinaryCopyHandle.java index 94a86f1f94762..fa7baab2af150 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBinaryCopyHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBinaryCopyHandle.java @@ -70,7 +70,7 @@ private MessageType getWriteSchema(HoodieWriteConfig config, List i try { ParquetUtils parquetUtils = new ParquetUtils(); MessageType fileSchema = parquetUtils.readMessageType(table.getStorage(), inputFiles.get(0)); - log.info("Binary copy schema evolution disabled. Using schema from input file: " + inputFiles.get(0)); + log.info("Binary copy schema evolution disabled. Using schema from input file: {}", inputFiles.get(0)); return fileSchema; } catch (Exception e) { log.error("Failed to read schema from input file", e); @@ -109,8 +109,8 @@ public HoodieBinaryCopyHandle( } public void write() { - log.info("Start to merge source files " + this.inputFiles + " into target file: " + this.path - + ". Please pay attention that we will not rolling files based on max-file-size config during binary copy."); + log.info("Start to merge source files {} into target file: {}. Please pay attention that we will not rolling files based on max-file-size config during binary copy.", + this.inputFiles, this.path); HoodieTimer timer = HoodieTimer.start(); long records = 0; try { @@ -123,12 +123,12 @@ public void write() { this.recordsWritten = records; this.insertRecordsWritten = records; } - log.info("Finish rewriting " + this.path + ". Using " + timer.endTimer() + " mills"); + log.info("Finish rewriting {}. Using {} mills", this.path, timer.endTimer()); } @Override public List close() { - log.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten); + log.info("Closing the file {} as we are done with all the records {}", writeStatus.getFileId(), recordsWritten); try { this.writer.close(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java index e67cc98e50176..778feffa51f88 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java @@ -122,7 +122,7 @@ public static Iterator> runMerge(HoodieMergeHandle // TODO(vc): This needs to be revisited if (mergeHandle.getPartitionPath() == null) { - log.info("Upsert Handle has partition path as null " + mergeHandle.getOldFilePath() + ", " + mergeHandle.getWriteStatuses()); + log.info("Upsert Handle has partition path as null {}, {}", mergeHandle.getOldFilePath(), mergeHandle.getWriteStatuses()); } return Collections.singletonList(mergeHandle.close()).iterator(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java index 2d4717a80054b..eb09e9a12b7ee 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java @@ -237,7 +237,7 @@ public void update(HoodieRollbackMetadata rollbackMetadata, String instantTime) String rollbackInstantTime = createRollbackTimestamp(instantTime); if (metadataMetaClient.getActiveTimeline().containsInstant(deltaCommitInstant)) { - LOG.info("Rolling back MDT deltacommit " + commitToRollbackInstantTime); + LOG.info("Rolling back MDT deltacommit {}", commitToRollbackInstantTime); if (!getWriteClient().rollback(commitToRollbackInstantTime, rollbackInstantTime)) { throw new HoodieMetadataException("Failed to rollback deltacommit at " + commitToRollbackInstantTime); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java index a857ef4417d6b..f0d6d746bd3f6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -757,7 +757,7 @@ private void deleteInvalidFilesByPartitions(HoodieEngineContext context, Map { final HoodieStorage storage = metaClient.getStorage(); - log.info("Deleting invalid data file=" + partitionFilePair); + log.info("Deleting invalid data file={}", partitionFilePair); // Delete try { StoragePath pathToDelete = new StoragePath(partitionFilePair.getValue()); @@ -828,7 +828,7 @@ void reconcileAgainstMarkers(HoodieEngineContext context, throw new HoodieDuplicateDataFileDetectedException("Duplicate data files detected " + invalidDataPaths); } - log.info("Removing duplicate files created due to task retries before committing. Paths=" + invalidDataPaths); + log.info("Removing duplicate files created due to task retries before committing. Paths={}", invalidDataPaths); Map>> invalidPathsByPartition = invalidDataPaths.stream() .map(dp -> Pair.of(new StoragePath(basePath, dp).getParent().toString(), @@ -1146,7 +1146,7 @@ public void deleteMetadataIndexIfNecessary() { Stream.of(MetadataPartitionType.getValidValues()).forEach(partitionType -> { if (shouldDeleteMetadataPartition(partitionType)) { try { - log.info("Deleting metadata partition because it is disabled in writer: " + partitionType.name()); + log.info("Deleting metadata partition because it is disabled in writer: {}", partitionType.name()); if (metadataPartitionExists(metaClient.getBasePath(), context, partitionType.getPartitionPath())) { deleteMetadataPartition(metaClient.getBasePath(), context, partitionType.getPartitionPath()); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java index 1c1a7d7ccb3ab..d825ac471e683 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java @@ -86,7 +86,7 @@ public Option generateClusteringPlan() { LOG.error("earliest commit to cluster is not specified"); return Option.empty(); } - LOG.info("Earliest commit to cluster (exclusive): " + earliestCommit); + LOG.info("Earliest commit to cluster (exclusive): {}", earliestCommit); HoodieTimeline commitTimeline = metaClient.getCommitsTimeline().findInstantsAfter(earliestCommit).filterCompletedInstants(); // For each completed commit, invoke getFileSlicesEligibleForCommitBasedClustering @@ -247,7 +247,7 @@ private CommitFiles getFileSlicesEligibleForCommitBasedClustering(HoodieInstant try { commitMetadata = TimelineUtils.getCommitMetadata(instant, metaClient.getActiveTimeline()); } catch (IOException e) { - LOG.error("Failed to read commit metadata for instant: " + instant, e); + LOG.error("Failed to read commit metadata for instant: {}", instant, e); throw new HoodieException("Failed to read commit metadata for instant: " + instant, e); } } else { @@ -285,7 +285,7 @@ private CommitFiles getFileSlicesEligibleForCommitBasedClustering(HoodieInstant try { pathInfo = storage.getPathInfo(path); } catch (Exception e) { - LOG.error("Could not get PathInfo for file path: " + path, e); + LOG.error("Could not get PathInfo for file path: {}", path, e); throw new HoodieException("Could not get PathInfo for file path: " + path, e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java index 3d49c5f406e15..fc24e7be26a6a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java @@ -82,15 +82,15 @@ protected Pair, Boolean> buildClusteringGroupsForP // check if max size is reached and create new group, if needed. if (totalSizeSoFar + currentSize > writeConfig.getClusteringMaxBytesInGroup() && !currentGroup.isEmpty()) { int numOutputGroups = getNumberOfOutputFileGroups(totalSizeSoFar, writeConfig.getClusteringTargetFileMaxBytes()); - log.info("Adding one clustering group " + totalSizeSoFar + " max bytes: " - + writeConfig.getClusteringMaxBytesInGroup() + " num input slices: " + currentGroup.size() + " output groups: " + numOutputGroups); + log.info("Adding one clustering group {} max bytes: {} num input slices: {} output groups: {}", + totalSizeSoFar, writeConfig.getClusteringMaxBytesInGroup(), currentGroup.size(), numOutputGroups); fileSliceGroups.add(Pair.of(currentGroup, numOutputGroups)); currentGroup = new ArrayList<>(); totalSizeSoFar = 0; // if fileSliceGroups's size reach the max group, stop loop if (fileSliceGroups.size() >= writeConfig.getClusteringMaxNumGroups()) { - log.info("Having generated the maximum number of groups : " + writeConfig.getClusteringMaxNumGroups()); + log.info("Having generated the maximum number of groups : {}", writeConfig.getClusteringMaxNumGroups()); partialScheduled = true; break; } @@ -104,8 +104,8 @@ protected Pair, Boolean> buildClusteringGroupsForP if (!currentGroup.isEmpty()) { int numOutputGroups = getNumberOfOutputFileGroups(totalSizeSoFar, writeConfig.getClusteringTargetFileMaxBytes()); - log.info("Adding final clustering group " + totalSizeSoFar + " max bytes: " - + writeConfig.getClusteringMaxBytesInGroup() + " num input slices: " + currentGroup.size() + " output groups: " + numOutputGroups); + log.info("Adding final clustering group {} max bytes: {} num input slices: {} output groups: {}", + totalSizeSoFar, writeConfig.getClusteringMaxBytesInGroup(), currentGroup.size(), numOutputGroups); fileSliceGroups.add(Pair.of(currentGroup, numOutputGroups)); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java index ff9fcb8666926..c7e7f0287ac17 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java @@ -188,7 +188,7 @@ protected void completeCommit(HoodieWriteMetadata result) { initializeLastCompletedTnxAndPendingInstants(); } autoCommit(result); - log.info("Completed commit for " + instantTime); + log.info("Completed commit for {}", instantTime); } protected void autoCommit(HoodieWriteMetadata result) { @@ -216,7 +216,7 @@ protected void autoCommit(HoodieWriteMetadata result) { protected void commit(HoodieWriteMetadata result, List writeStats) { String actionType = getCommitActionType(); - log.info("Committing " + instantTime + ", action Type " + actionType + ", operation Type " + operationType); + log.info("Committing {}, action Type {}, operation Type {}", instantTime, actionType, operationType); result.setCommitted(true); result.setWriteStats(writeStats); // Finalize write @@ -234,7 +234,7 @@ protected void commit(HoodieWriteMetadata result, List write activeTimeline.saveAsComplete(false, table.getMetaClient().createNewInstant(State.INFLIGHT, actionType, instantTime), Option.of(metadata), completedInstant -> table.getMetaClient().getTableFormat().commit(metadata, completedInstant, table.getContext(), table.getMetaClient(), table.getViewManager())); - log.info("Committed " + instantTime); + log.info("Committed {}", instantTime); result.setCommitMetadata(Option.of(metadata)); // update cols to Index as applicable HoodieColumnStatsIndexUtils.updateColsToIndex(table, config, metadata, actionType, @@ -308,7 +308,7 @@ protected HoodieWriteMetadata> executeClustering(HoodieC writeMetadata.setWriteStatuses(statuses); - log.debug("Create place holder commit metadata for clustering with instant time " + instantTime); + log.debug("Create place holder commit metadata for clustering with instant time {}", instantTime); HoodieCommitMetadata commitMetadata = CommitUtils.buildMetadata(Collections.emptyList(), Collections.emptyMap(), extraMetadata, operationType, schema.get().toString(), getCommitActionType()); writeMetadata.setCommitMetadata(Option.of(commitMetadata)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java index 298beb15b7e0b..6f03b69dcb04a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java @@ -124,11 +124,11 @@ public Option execute() { @Nullable private HoodieCompactionPlan scheduleCompaction() { - log.info("Checking if compaction needs to be run on " + config.getBasePath()); + log.info("Checking if compaction needs to be run on {}", config.getBasePath()); // judge if we need to compact according to num delta commits and time elapsed boolean compactable = needCompact(config.getInlineCompactTriggerStrategy()); if (compactable) { - log.info("Generating compaction plan for merge on read table " + config.getBasePath()); + log.info("Generating compaction plan for merge on read table {}", config.getBasePath()); try { context.setJobStatus(this.getClass().getSimpleName(), "Compaction: generating compaction plan"); return planGenerator.generateCompactionPlan(instantTime); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java index 381cf86f28a07..657b82da7a9bd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java @@ -47,7 +47,7 @@ public HoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engi BaseTableServicePlanActionExecutor executor) { super(table, engineContext, writeConfig, executor); this.compactionStrategy = writeConfig.getCompactionStrategy(); - log.info("Compaction Strategy used is: " + compactionStrategy.toString()); + log.info("Compaction Strategy used is: {}", compactionStrategy); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/AbstractIndexingCatchupTask.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/AbstractIndexingCatchupTask.java index c14df6312ecc8..2ab39cd305342 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/AbstractIndexingCatchupTask.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/AbstractIndexingCatchupTask.java @@ -107,7 +107,7 @@ public void run() { try { // we need take a lock here as inflight writer could also try to update the timeline transactionManager.beginStateChange(Option.of(instant), Option.empty()); - log.info("Updating metadata table for instant: " + instant); + log.info("Updating metadata table for instant: {}", instant); switch (instant.getAction()) { case HoodieTimeline.COMMIT_ACTION: case HoodieTimeline.DELTA_COMMIT_ACTION: diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java index 9ad9159ecef64..1be128c5bde78 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java @@ -86,7 +86,7 @@ public HoodieRestoreMetadata execute() { instantsToRollback.forEach(instant -> { instantToMetadata.put(instant.requestedTime(), Collections.singletonList(rollbackInstant(instant))); - log.info("Deleted instant " + instant); + log.info("Deleted instant {}", instant); }); return finishRestore(instantToMetadata, @@ -143,7 +143,7 @@ private HoodieRestoreMetadata finishRestore(Map doRollbackAndGetStats(HoodieRollbackPlan hoodieRollbackPlan) { @@ -235,7 +235,7 @@ public List doRollbackAndGetStats(HoodieRollbackPlan hoodieR try { List stats = executeRollback(hoodieRollbackPlan); - log.info("Rolled back inflight instant " + instantTimeToRollback); + log.info("Rolled back inflight instant {}", instantTimeToRollback); if (!isPendingCompaction) { rollBackIndex(); } @@ -289,7 +289,7 @@ protected void finishRollback(HoodieInstant inflightInstant, HoodieRollbackMetad // when skipLocking is true, the caller should have already held the lock. table.getActiveTimeline().transitionRollbackInflightToComplete(false, inflightInstant, rollbackMetadata, completedInstant -> table.getMetaClient().getTableFormat().completedRollback(completedInstant, table.getContext(), table.getMetaClient(), table.getViewManager())); - log.info("Rollback of Commits " + rollbackMetadata.getCommitsRollback() + " is complete"); + log.info("Rollback of Commits {} is complete", rollbackMetadata.getCommitsRollback()); } } finally { if (enableLocking) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java index d3f3662896965..5b97a4ef9e153 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java @@ -66,7 +66,7 @@ protected List executeRollback(HoodieRollbackPlan hoodieRoll HoodieActiveTimeline activeTimeline = table.getActiveTimeline(); if (instantToRollback.isCompleted()) { - log.info("Unpublishing instant " + instantToRollback); + log.info("Unpublishing instant {}", instantToRollback); table.getMetaClient().getTableFormat().rollback(instantToRollback, table.getContext(), table.getMetaClient(), table.getViewManager()); // Revert the completed instant to inflight in native format. resolvedInstant = activeTimeline.revertToInflight(instantToRollback); @@ -88,13 +88,13 @@ protected List executeRollback(HoodieRollbackPlan hoodieRoll // deleting the timeline file if (!resolvedInstant.isRequested()) { // delete all the data files for this commit - log.info("Clean out all base files generated for commit: " + resolvedInstant); + log.info("Clean out all base files generated for commit: {}", resolvedInstant); stats = executeRollback(resolvedInstant, hoodieRollbackPlan); } dropBootstrapIndexIfNeeded(instantToRollback); - log.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer()); + log.info("Time(in ms) taken to finish rollback {}", rollbackTimer.endTimer()); return stats; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java index ff28dc99c5862..433c769381dbb 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java @@ -215,7 +215,7 @@ public List getRollbackRequests(HoodieInstant instantToRo return hoodieRollbackRequests.stream(); }, numPartitions); } catch (Exception e) { - log.error("Generating rollback requests failed for " + instantToRollback.requestedTime(), e); + log.error("Generating rollback requests failed for {}", instantToRollback.requestedTime(), e); throw new HoodieRollbackException("Generating rollback requests failed for " + instantToRollback.requestedTime(), e); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java index 1373d9679e02b..bfe783779746a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java @@ -60,11 +60,11 @@ public MergeOnReadRollbackActionExecutor(HoodieEngineContext context, protected List executeRollback(HoodieRollbackPlan hoodieRollbackPlan) { HoodieTimer rollbackTimer = HoodieTimer.start(); - log.info("Rolling back instant " + instantToRollback); + log.info("Rolling back instant {}", instantToRollback); // Atomically un-publish all non-inflight commits if (instantToRollback.isCompleted()) { - log.info("Un-publishing instant " + instantToRollback + ", deleteInstants=" + deleteInstants); + log.info("Un-publishing instant {}, deleteInstants={}", instantToRollback, deleteInstants); resolvedInstant = table.getActiveTimeline().revertToInflight(instantToRollback); // reload meta-client to reflect latest timeline status table.getMetaClient().reloadActiveTimeline(); @@ -81,13 +81,13 @@ protected List executeRollback(HoodieRollbackPlan hoodieRoll // For Requested State (like failure during index lookup), there is nothing to do rollback other than // deleting the timeline file if (!resolvedInstant.isRequested()) { - log.info("Unpublished " + resolvedInstant); + log.info("Unpublished {}", resolvedInstant); allRollbackStats = executeRollback(instantToRollback, hoodieRollbackPlan); } dropBootstrapIndexIfNeeded(resolvedInstant); - log.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer()); + log.info("Time(in ms) taken to finish rollback {}", rollbackTimer.endTimer()); return allRollbackStats; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java index b31fdc94178af..1b98803274737 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java @@ -133,8 +133,7 @@ protected Option create(String partitionPath, String fileName, IOTy Map paramsMap = getConfigMap(partitionPath, markerFileName, false); boolean success = executeCreateMarkerRequest(paramsMap, partitionPath, markerFileName); - log.info("[timeline-server-based] Created marker file " + partitionPath + "/" + markerFileName - + " in " + timer.endTimer() + " ms"); + log.info("[timeline-server-based] Created marker file {}/{} in {} ms", partitionPath, markerFileName, timer.endTimer()); if (success) { return Option.of(new StoragePath(FSUtils.constructAbsolutePath(markerDirPath, partitionPath), markerFileName)); } else { @@ -151,8 +150,7 @@ public Option createWithEarlyConflictDetection(String partitionPath boolean success = executeCreateMarkerRequest(paramsMap, partitionPath, markerFileName); - log.info("[timeline-server-based] Created marker file with early conflict detection " + partitionPath + "/" + markerFileName - + " in " + timer.endTimer() + " ms"); + log.info("[timeline-server-based] Created marker file with early conflict detection {}/{} in {} ms", partitionPath, markerFileName, timer.endTimer()); if (success) { return Option.of(new StoragePath(FSUtils.constructAbsolutePath(markerDirPath, partitionPath), markerFileName)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java index 62a668a1acc6a..c7089175d664a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java @@ -64,12 +64,12 @@ private void deleteCompactionRequestedFileFromAuxiliaryFolder(HoodieTable table) compactionTimeline.getInstantsAsStream().forEach( deleteInstant -> { - log.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath()); + log.info("Deleting instant {} in auxiliary meta path {}", deleteInstant, metaClient.getMetaAuxiliaryPath()); StoragePath metaFile = new StoragePath(metaClient.getMetaAuxiliaryPath(), factory.getFileName(deleteInstant)); try { if (metaClient.getStorage().exists(metaFile)) { metaClient.getStorage().deleteFile(metaFile); - log.info("Deleted instant file in auxiliary meta path : " + metaFile); + log.info("Deleted instant file in auxiliary meta path : {}", metaFile); } } catch (IOException e) { throw new HoodieUpgradeDowngradeException(HoodieTableVersion.FIVE.versionCode(), HoodieTableVersion.SIX.versionCode(), true, e); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java index 0d17a75a7da4b..febb815ad34fd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java @@ -208,7 +208,7 @@ public void run(HoodieTableVersion toVersion, String instantTime) { // Perform the actual upgrade/downgrade; this has to be idempotent, for now. - log.info("Attempting to move table from version " + fromVersion + " to " + toVersion); + log.info("Attempting to move table from version {} to {}", fromVersion, toVersion); Map tablePropsToAdd = new Hashtable<>(); Set tablePropsToRemove = new HashSet<>(); if (isUpgrade) { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java index ac6e7df560381..7bab6f5a19a1c 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java @@ -117,13 +117,13 @@ private HoodieLogFormat.Writer openWriter(HoodieTableMetaClient metaClient) { public void archive(HoodieTableMetaClient metaClient, List instants) throws HoodieCommitException { try (HoodieLogFormat.Writer writer = openWriter(metaClient)) { Schema wrapperSchema = HoodieArchivedMetaEntry.getClassSchema(); - log.info("Wrapper schema " + wrapperSchema.toString()); + log.info("Wrapper schema {}", wrapperSchema); List records = new ArrayList<>(); for (HoodieInstant hoodieInstant : instants) { try { records.add(convertToAvroRecord(hoodieInstant, metaClient)); } catch (Exception e) { - log.error("Failed to archive commits, .commit file: " + INSTANT_FILE_NAME_GENERATOR.getFileName(hoodieInstant), e); + log.error("Failed to archive commits, .commit file: {}", INSTANT_FILE_NAME_GENERATOR.getFileName(hoodieInstant), e); throw e; } } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java index 705aec57e66da..1f0f801d89b70 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java @@ -341,7 +341,7 @@ private void assertFailedDeletion(RollbackHelper rollbackHelper, HoodieEngineCon fail("Should not have reached here"); } catch (HoodieException e) { if (!(e.getCause() instanceof HoodieIOException)) { - log.error("Expected HoodieIOException to be thrown, but found " + e.getCause() + ", w/ error msg " + e.getCause().getMessage()); + log.error("Expected HoodieIOException to be thrown, but found {}, w/ error msg {}", e.getCause(), e.getCause().getMessage()); } assertTrue(e.getCause() instanceof HoodieIOException); assertTrue(e.getCause().getMessage().contains("Failing to delete file during rollback execution failed : " + expectedFileToFailOnDeletion)); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java index f9802b1634761..6a922c690b56e 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java @@ -111,7 +111,7 @@ public StoragePath withInserts(String partition, String fileId, List> execute() { if (expiredPartitions.isEmpty()) { return emptyResult; } - log.info("Partition ttl find the following expired partitions to delete: " + String.join(",", expiredPartitions)); + log.info("Partition ttl find the following expired partitions to delete: {}", String.join(",", expiredPartitions)); return new FlinkAutoCommitActionExecutor(new FlinkDeletePartitionCommitActionExecutor<>(context, config, table, instantTime, expiredPartitions)).execute(); } catch (HoodieDeletePartitionPendingTableServiceException deletePartitionPendingTableServiceException) { log.info("Partition is under table service, do nothing, call delete partition next time."); diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java index 43e5dd260561d..7746bdf3be7ee 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java @@ -94,7 +94,7 @@ public HoodieWriteMetadata> execute(List> inpu WorkloadProfile workloadProfile = new WorkloadProfile(buildProfile(inputRecords), table.getIndex().canIndexLogFiles()); - log.info("Input workload profile :" + workloadProfile); + log.info("Input workload profile :{}", workloadProfile); final Partitioner partitioner = getPartitioner(workloadProfile); try { saveWorkloadProfileMetadataToInflight(workloadProfile, instantTime); @@ -236,7 +236,7 @@ public Iterator> handleUpdate(String partitionPath, String fil throws IOException { // This is needed since sometimes some buckets are never picked in getPartition() and end up with 0 records if (!recordItr.hasNext()) { - log.info("Empty partition with fileId => " + fileId); + log.info("Empty partition with fileId => {}", fileId); return Collections.singletonList((List) Collections.EMPTY_LIST).iterator(); } // these are updates diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPartitioner.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPartitioner.java index 15010d21a0d62..9b86e6ccb8068 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPartitioner.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPartitioner.java @@ -93,9 +93,8 @@ public JavaUpsertPartitioner(WorkloadProfile workloadProfile, HoodieEngineContex assignUpdates(workloadProfile); assignInserts(workloadProfile, context); - log.info("Total Buckets :" + totalBuckets + ", buckets info => " + bucketInfoMap + ", \n" - + "Partition to insert buckets => " + partitionPathToInsertBucketInfos + ", \n" - + "UpdateLocations mapped to buckets =>" + updateLocationToBucket); + log.info("Total Buckets :{}, buckets info => {}, \nPartition to insert buckets => {}, \nUpdateLocations mapped to buckets =>{}", + totalBuckets, bucketInfoMap, partitionPathToInsertBucketInfos, updateLocationToBucket); } private void assignUpdates(WorkloadProfile profile) { @@ -132,7 +131,7 @@ private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) long averageRecordSize = averageBytesPerRecord(table.getMetaClient().getActiveTimeline().getCommitAndReplaceTimeline().filterCompletedInstants(), config); - log.info("AvgRecordSize => " + averageRecordSize); + log.info("AvgRecordSize => {}", averageRecordSize); Map> partitionSmallFilesMap = getSmallFilesForPartitions(new ArrayList(partitionPaths), context); @@ -145,7 +144,7 @@ private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) List smallFiles = partitionSmallFilesMap.getOrDefault(partitionPath, new ArrayList<>()); this.smallFiles.addAll(smallFiles); - log.info("For partitionPath : " + partitionPath + " Small Files => " + smallFiles); + log.info("For partitionPath : {} Small Files => {}", partitionPath, smallFiles); long totalUnassignedInserts = pStat.getNumInserts(); List bucketNumbers = new ArrayList<>(); @@ -160,10 +159,10 @@ private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) int bucket; if (updateLocationToBucket.containsKey(smallFile.location.getFileId())) { bucket = updateLocationToBucket.get(smallFile.location.getFileId()); - log.info("Assigning " + recordsToAppend + " inserts to existing update bucket " + bucket); + log.info("Assigning {} inserts to existing update bucket {}", recordsToAppend, bucket); } else { bucket = addUpdateBucket(partitionPath, smallFile.location.getFileId()); - log.info("Assigning " + recordsToAppend + " inserts to new update bucket " + bucket); + log.info("Assigning {} inserts to new update bucket {}", recordsToAppend, bucket); } if (profile.hasOutputWorkLoadStats()) { outputWorkloadStats.addInserts(smallFile.location, recordsToAppend); @@ -182,8 +181,7 @@ private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) } int insertBuckets = (int) Math.ceil((1.0 * totalUnassignedInserts) / insertRecordsPerBucket); - log.info("After small file assignment: unassignedInserts => " + totalUnassignedInserts - + ", totalInsertBuckets => " + insertBuckets + ", recordsPerBucket => " + insertRecordsPerBucket); + log.info("After small file assignment: unassignedInserts => {}, totalInsertBuckets => {}, recordsPerBucket => {}", totalUnassignedInserts, insertBuckets, insertRecordsPerBucket); for (int b = 0; b < insertBuckets; b++) { bucketNumbers.add(totalBuckets); if (b < insertBuckets - 1) { @@ -210,7 +208,7 @@ private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) currentCumulativeWeight += bkt.weight; insertBuckets.add(new InsertBucketCumulativeWeightPair(bkt, currentCumulativeWeight)); } - log.info("Total insert buckets for partition path " + partitionPath + " => " + insertBuckets); + log.info("Total insert buckets for partition path {} => {}", partitionPath, insertBuckets); partitionPathToInsertBucketInfos.put(partitionPath, insertBuckets); } if (profile.hasOutputWorkLoadStats()) { diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseJavaDeltaCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseJavaDeltaCommitActionExecutor.java index 42be024f167da..ec25bcabd6ec0 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseJavaDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseJavaDeltaCommitActionExecutor.java @@ -69,10 +69,10 @@ public Partitioner getUpsertPartitioner(WorkloadProfile profile) { @Override public Iterator> handleUpdate(String partitionPath, String fileId, Iterator> recordItr) throws IOException { - log.info("Merging updates for commit " + instantTime + " for file " + fileId); + log.info("Merging updates for commit {} for file {}", instantTime, fileId); if (!table.getIndex().canIndexLogFiles() && partitioner != null && partitioner.getSmallFileIds().contains(fileId)) { - log.info("Small file corrections for updates for commit " + instantTime + " for file " + fileId); + log.info("Small file corrections for updates for commit {} for file {}", instantTime, fileId); return super.handleUpdate(partitionPath, fileId, recordItr); } else { HoodieAppendHandle appendHandle = new HoodieAppendHandle<>(config, instantTime, table, diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index feda056472dc8..52b5563ac93ef 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -1840,7 +1840,7 @@ public void testMultiWriterForDoubleLocking() throws Exception { // Ensure all commits were synced to the Metadata Table HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable(); - log.warn("total commits in metadata table " + metadataMetaClient.getActiveTimeline().getCommitsTimeline().countInstants()); + log.warn("total commits in metadata table {}", metadataMetaClient.getActiveTimeline().getCommitsTimeline().countInstants()); // 6 commits and 2 cleaner commits. assertEquals(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants().countInstants(), 8); @@ -2832,17 +2832,17 @@ private void validateMetadata(HoodieJavaWriteClient testClient, Option i if ((fsFileNames.size() != metadataFilenames.size()) || (!fsFileNames.equals(metadataFilenames))) { - log.info("*** File system listing = " + Arrays.toString(fsFileNames.toArray())); - log.info("*** Metadata listing = " + Arrays.toString(metadataFilenames.toArray())); + log.info("*** File system listing = {}", Arrays.toString(fsFileNames.toArray())); + log.info("*** Metadata listing = {}", Arrays.toString(metadataFilenames.toArray())); for (String fileName : fsFileNames) { if (!metadataFilenames.contains(fileName)) { - log.error(partition + "FsFilename " + fileName + " not found in Meta data"); + log.error("{}FsFilename {} not found in Meta data", partition, fileName); } } for (String fileName : metadataFilenames) { if (!fsFileNames.contains(fileName)) { - log.error(partition + "Metadata file " + fileName + " not found in original FS"); + log.error("{}Metadata file {} not found in original FS", partition, fileName); } } } @@ -2922,7 +2922,7 @@ private void validateMetadata(HoodieJavaWriteClient testClient, Option i }); // TODO: include validation for record_index partition here. - log.info("Validation time=" + timer.endTimer()); + log.info("Validation time={}", timer.endTimer()); } } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java index 8fc8b480a3f0d..a0998cbdd0310 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java @@ -366,7 +366,7 @@ public void testFileSizeUpsertRecords() throws Exception { int counts = 0; for (File file : Paths.get(basePath, "2016/01/31").toFile().listFiles()) { if (file.getName().endsWith(table.getBaseFileExtension()) && FSUtils.getCommitTime(file.getName()).equals(instantTime)) { - log.info(file.getName() + "-" + file.length()); + log.info("{}-{}", file.getName(), file.length()); counts++; } } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java index 353be1549081b..5d5d15e19d9e6 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java @@ -353,7 +353,7 @@ public void validateMetadata(HoodieTestTable testTable, List inflightCom runFullValidation(writeConfig, metadataTableBasePath, engineContext); } - log.info("Validation time=" + timer.endTimer()); + log.info("Validation time={}", timer.endTimer()); } protected void validateFilesPerPartition(HoodieTestTable testTable, diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkBinaryCopyClusteringExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkBinaryCopyClusteringExecutionStrategy.java index f68579e580d7b..efabfeaf5d4ed 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkBinaryCopyClusteringExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkBinaryCopyClusteringExecutionStrategy.java @@ -95,7 +95,7 @@ public HoodieWriteMetadata> performClustering( JavaSparkContext engineContext = HoodieSparkEngineContext.getSparkContext(getEngineContext()); TaskContextSupplier taskContextSupplier = getEngineContext().getTaskContextSupplier(); JavaRDD groupInfoJavaRDD = engineContext.parallelize(clusteringGroupInfos, clusteringGroupInfos.size()); - log.info("number of partitions for clustering " + groupInfoJavaRDD.getNumPartitions()); + log.info("number of partitions for clustering {}", groupInfoJavaRDD.getNumPartitions()); JavaRDD writeStatusRDD = groupInfoJavaRDD .mapPartitions(clusteringOps -> { Iterable clusteringOpsIterable = () -> clusteringOps; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java index be5c8aaa01ef5..0abcec9c9bee4 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java @@ -87,7 +87,7 @@ protected List performClusteringForGroup(ReaderContextFactory re try { getHoodieTable().getStorage().deleteFile(writeHandler.getPath()); } catch (Exception deleteEx) { - LOG.warn("Failed to clean up partial output file: " + writeHandler.getPath(), deleteEx); + LOG.warn("Failed to clean up partial output file: {}", writeHandler.getPath(), deleteEx); } throw new HoodieClusteringException("Failed to transform file: " + dataFilePathStr, e); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java index b06e80bf63922..de3ffbf6ae63e 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java @@ -96,7 +96,7 @@ public void validate(String instantTime, HoodieWriteMetadata writeResult, Dat throw new RuntimeException(e); } finally { long duration = timer.endTimer(); - log.info(getClass() + " validator took " + duration + " ms" + ", metrics on? " + getWriteConfig().isMetricsOn()); + log.info("{} validator took {} ms, metrics on? {}", getClass(), duration, getWriteConfig().isMetricsOn()); publishRunStats(instantTime, duration); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java index 9959baad30a2c..d5884aaabada8 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java @@ -57,16 +57,16 @@ protected void validateUsingQuery(String query, String prevTableSnapshot, String try { prevRows = executeSqlQuery( sqlContext, query, prevTableSnapshot, "previous state").cache(); - log.info("Total rows in prevRows " + prevRows.count()); + log.info("Total rows in prevRows {}", prevRows.count()); newRows = executeSqlQuery( sqlContext, query, newTableSnapshot, "new state").cache(); - log.info("Total rows in newRows " + newRows.count()); + log.info("Total rows in newRows {}", newRows.count()); printAllRowsIfDebugEnabled(prevRows); printAllRowsIfDebugEnabled(newRows); boolean areDatasetsEqual = prevRows.intersect(newRows).count() == prevRows.count(); - log.info("Completed Equality Validation, datasets equal? " + areDatasetsEqual); + log.info("Completed Equality Validation, datasets equal? {}", areDatasetsEqual); if (!areDatasetsEqual) { - log.error("query validation failed. See stdout for sample query results. Query: " + query); + log.error("query validation failed. See stdout for sample query results. Query: {}", query); System.out.println("Expected result (sample records only):"); prevRows.show(); System.out.println("Actual result (sample records only):"); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java index f0aae541d3d3e..0b22b540a79b8 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java @@ -54,16 +54,16 @@ protected String getQueryConfigName() { protected void validateUsingQuery(String query, String prevTableSnapshot, String newTableSnapshot, SQLContext sqlContext) { Dataset prevRows = executeSqlQuery( sqlContext, query, prevTableSnapshot, "previous state").cache(); - log.info("Total rows in prevRows " + prevRows.count()); + log.info("Total rows in prevRows {}", prevRows.count()); Dataset newRows = executeSqlQuery( sqlContext, query, newTableSnapshot, "new state").cache(); - log.info("Total rows in newRows " + newRows.count()); + log.info("Total rows in newRows {}", newRows.count()); printAllRowsIfDebugEnabled(prevRows); printAllRowsIfDebugEnabled(newRows); boolean areDatasetsEqual = prevRows.intersect(newRows).count() == prevRows.count(); - log.info("Completed Inequality Validation, datasets equal? " + areDatasetsEqual); + log.info("Completed Inequality Validation, datasets equal? {}", areDatasetsEqual); if (areDatasetsEqual) { - log.error("query validation failed. See stdout for sample query results. Query: " + query); + log.error("query validation failed. See stdout for sample query results. Query: {}", query); System.out.println("Expected query results to be different, but they are same. Result (sample records only):"); prevRows.show(); throw new HoodieValidationException("Query validation failed for '" + query diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java index 47f961b88b94f..ef5278f02d3d5 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java @@ -66,11 +66,11 @@ protected void validateUsingQuery(String query, String prevTableSnapshot, String } Object result = newRows.get(0).apply(0); if (result == null || !expectedResult.equals(result.toString())) { - log.error("Mismatch query result. Expected: " + expectedResult + " got " + result + " on Query: " + query); + log.error("Mismatch query result. Expected: {} got {} on Query: {}", expectedResult, result, query); throw new HoodieValidationException("Query validation failed for '" + query + "'. Expected " + expectedResult + " row(s), Found " + result); } else { - log.info("Query validation successful. Expected: " + expectedResult + " got " + result); + log.info("Query validation successful. Expected: {} got {}", expectedResult, result); } } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java index 0222506f56faf..c6d20eefb5823 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java @@ -194,7 +194,7 @@ private void writeRow(InternalRow row) { ? HoodieRecordDelegate.create(recordKey.toString(), partitionPath.toString(), null, newRecordLocation) : null; writeStatus.markSuccess(recordDelegate, Option.empty()); } catch (Exception t) { - log.error("Error writing record " + row, t); + log.error("Error writing record {}", row, t); if (!writeConfig.getIgnoreWriteFailed()) { throw new HoodieException(t.getMessage(), t); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java index c0babd7248739..6e24f34fa1c81 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java @@ -65,7 +65,7 @@ public BootstrapWriteStatus runMetadataBootstrap(String srcPartitionPath, String .collect(Collectors.toList()); HoodieSchema recordKeySchema = HoodieSchemaUtils.generateProjectionSchema(schema, recordKeyColumns); - LOG.info("Schema to be used for reading record keys: " + recordKeySchema); + LOG.info("Schema to be used for reading record keys: {}", recordKeySchema); executeBootstrap(bootstrapHandle, sourceFilePath, keyGenerator, partitionPath, recordKeySchema); } catch (Exception e) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java index f611e71d43f39..eb61b2699b371 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java @@ -204,14 +204,12 @@ protected void commit(HoodieWriteMetadata> result) { HoodieTableMetaClient metaClient = table.getMetaClient(); try (BootstrapIndex.IndexWriter indexWriter = BootstrapIndex.getBootstrapIndex(metaClient) .createWriter(metaClient.getTableConfig().getBootstrapBasePath().get())) { - log.info("Starting to write bootstrap index for source " + config.getBootstrapSourceBasePath() + " in table " - + config.getBasePath()); + log.info("Starting to write bootstrap index for source {} in table {}", config.getBootstrapSourceBasePath(), config.getBasePath()); indexWriter.begin(); bootstrapSourceAndStats.forEach((key, value) -> indexWriter.appendNextPartition(key, value.stream().map(Pair::getKey).collect(Collectors.toList()))); indexWriter.finish(); - log.info("Finished writing bootstrap index for source " + config.getBootstrapSourceBasePath() + " in table " - + config.getBasePath()); + log.info("Finished writing bootstrap index for source {} in table {}", config.getBootstrapSourceBasePath(), config.getBasePath()); } commit(result, bootstrapSourceAndStats.values().stream() .flatMap(f -> f.stream().map(Pair::getValue)).collect(Collectors.toList())); @@ -278,7 +276,7 @@ private Map>>> listAndPr log.info("Fetching Bootstrap Schema !!"); HoodieBootstrapSchemaProvider sourceSchemaProvider = new HoodieSparkBootstrapSchemaProvider(config); bootstrapSchema = sourceSchemaProvider.getBootstrapSchema(context, folders).toString(); - log.info("Bootstrap Schema :" + bootstrapSchema); + log.info("Bootstrap Schema :{}", bootstrapSchema); BootstrapModeSelector selector = (BootstrapModeSelector) ReflectionUtils.loadClass(config.getBootstrapModeSelectorClass(), config); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java index 217b2c4dd6a5c..86bd48f20ac4f 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java @@ -210,7 +210,7 @@ private static void validateRepartitionedRDDProperties( } catch (AssertionError e) { logRDDContent("Original RDD", originalRdd); logRDDContent("Repartitioned RDD", repartitionedRdd); - LOG.error("Validation failed: " + e.getMessage(), e); + LOG.error("Validation failed: {}", e.getMessage(), e); throw e; // rethrow to fail the test } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java index 5725c571c8dc3..450a1b1d6b418 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java @@ -940,7 +940,7 @@ private static CompletableFuture allOfTerminateOnFailure(List { if (!jobFailed.getAndSet(true)) { - log.warn("One of the job failed. Cancelling all other futures. " + ex.getCause() + ", " + ex.getMessage()); + log.warn("One of the job failed. Cancelling all other futures. {}, {}", ex.getCause(), ex.getMessage()); int secondCounter = 0; while (secondCounter < futures.size()) { if (secondCounter != finalCounter) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java index e2d769ece032a..9a29e7093fcd1 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java @@ -74,7 +74,7 @@ public void setup() throws IOException { this.markerFolderPath = new StoragePath(metaClient.getMarkerFolderPath("000")); restartServerAndClient(0); - log.info("Connecting to Timeline Server :" + timelineService.getServerPort()); + log.info("Connecting to Timeline Server :{}", timelineService.getServerPort()); } @AfterEach @@ -108,7 +108,7 @@ void verifyMarkersInFileSystem(boolean isTablePartitioned) throws IOException { @EnumSource(value = FileSystemViewStorageType.class) public void testCreationWithTimelineServiceRetries(FileSystemViewStorageType storageType) throws Exception { restartServerAndClient(0, storageType); - log.info("Connecting to Timeline Server :" + timelineService.getServerPort()); + log.info("Connecting to Timeline Server :{}", timelineService.getServerPort()); // Validate marker creation/ deletion work without any failures in the timeline service. createSomeMarkers(true); assertTrue(storage.exists(markerFolderPath)); From 81fff774a5bdce7c6c4b372d40513552f7a84e7c Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 5 Jul 2026 23:49:56 +0800 Subject: [PATCH 140/255] refactor(io): use SLF4J parameterized logging instead of string concatenation (#19188) * refactor(io): use SLF4J parameterized logging instead of string concatenation Convert log/LOG string concatenation to {} placeholders in Registry and ReflectionUtils. Mechanical and behavior-preserving. * refactor(io): log context and stack trace in ReflectionUtils error paths Per review, pass the caught exception as the trailing throwable so SLF4J logs the full stack trace, and put the actual in-scope context in the placeholder: package name for the getResources IOException, and the resource URL for the toURI URISyntaxException. (cherry picked from commit 2ab2c0a38952a51e5ba9c8813b5e768c36798818) --- .../main/java/org/apache/hudi/common/metrics/Registry.java | 5 ++--- .../java/org/apache/hudi/common/util/ReflectionUtils.java | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hudi-io/src/main/java/org/apache/hudi/common/metrics/Registry.java b/hudi-io/src/main/java/org/apache/hudi/common/metrics/Registry.java index 7373481c44aa4..65f4496597c7d 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/metrics/Registry.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/metrics/Registry.java @@ -98,13 +98,12 @@ static Registry getRegistryOfClass(String tableName, String registryName, String Registry registry = REGISTRY_MAP.computeIfAbsent(key, k -> { String registryFullName = tableName.isEmpty() ? registryName : tableName + "." + registryName; Registry r = (Registry) ReflectionUtils.loadClass(clazz, registryFullName); - LOG.info("Created a new registry " + r); + LOG.info("Created a new registry {}", r); return r; }); if (!registry.getClass().getName().equals(clazz)) { - LOG.error("Registry with name " + registryName + " already exists with a different class " + registry.getClass().getName() - + " than the requested class " + clazz); + LOG.error("Registry with name {} already exists with a different class {} than the requested class {}", registryName, registry.getClass().getName(), clazz); } return registry; } diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java index 49c3e6c9dea3c..cb4cbc4290f89 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java @@ -136,7 +136,7 @@ public static Stream getTopLevelClassesInClasspath(Class clazz) { try { resources = classLoader.getResources(path); } catch (IOException e) { - log.error("Unable to fetch Resources in package " + e.getMessage()); + log.error("Unable to fetch Resources in package {}", packageName, e); } List directories = new ArrayList<>(); while (Objects.requireNonNull(resources).hasMoreElements()) { @@ -144,7 +144,7 @@ public static Stream getTopLevelClassesInClasspath(Class clazz) { try { directories.add(new File(resource.toURI())); } catch (URISyntaxException e) { - log.error("Unable to get " + e.getMessage()); + log.error("Unable to get URI for {}", resource, e); } } List classes = new ArrayList<>(); From dc7eff474f167f46b76924c93df1f7c2344fff45 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 7 Jul 2026 15:40:05 +0700 Subject: [PATCH 141/255] test(trino): de-flake testRecordLevelFileSkipping by setting the record-index wait timeout (#19213) testRecordLevelFileSkipping enables the record-level index but sets withColumnStatsTimeout("10s") - a no-op here, since column stats is disabled in this test - and never sets the record-index wait timeout, so the async RLI load falls back to the 2s default (HudiConfig#recordIndexWaitTimeout). Under CI load the RLI is not ready in 2s, so file skipping is incomplete and the query scans 2 splits instead of 1 (expected: 1 but was: 2). Replace the no-op withColumnStatsTimeout("10s") with withRecordIndexTimeout("10s"), matching the sibling RLI tests testRLIWithColumnNameUsingUppercaseLetters and testMultiKeyRLIWithColumnNameUsingUppercaseLetters and the earlier de-flake #13869. Test-only. Co-authored-by: Vova Kolmakov (cherry picked from commit aefabf79f707fe5930f4dfc02b7952c19296af00) --- .../src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java index f49933d3fb986..0e19f2a051771 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java @@ -801,7 +801,7 @@ public void testRecordLevelFileSkipping(ResourceHudiTablesInitializer.TestingTab .withRecordLevelIndexEnabled(true) .withSecondaryIndexEnabled(false) .withPartitionStatsIndexEnabled(false) - .withColumnStatsTimeout("10s") + .withRecordIndexTimeout("10s") .build(); MaterializedResult totalRes = getQueryRunner().execute(session, "SELECT * FROM " + table); MaterializedResult prunedRes = getQueryRunner().execute(session, "SELECT * FROM " + table From 82c8fd9249da3ee93eb1c7345efc5f3235c83975 Mon Sep 17 00:00:00 2001 From: ericyuan915 <77124531+ericyuan915@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:18:50 -0700 Subject: [PATCH 142/255] fix(flink): avoid AIOOBE in NestedColumnReader across the read batch boundary (#19210) NestedColumnReader#readRow collapses a present row whose children are all null into a NULL row. That loop iterated up to rowPosition.getPositionsCount(), which on a full, non-final vectorized batch is one larger than the materialized child vectors: the Dremel level stream carries a one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the batch, and #getLevelDelegation keeps that trailing level for the next batch). Reading a nested ROW column from a COW base file with more rows than the 2048-row batch (RecordIterators.DEFAULT_BATCH_SIZE) therefore indexed one element past a shorter child vector and threw ArrayIndexOutOfBoundsException. Clamp the collapse loop to the shortest vector it indexes (the row vector and every child), unwrapping ParquetDecimalVector (a non-AbstractHeapVector DECIMAL child) via a new vectorLength helper; the phantom trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch at num). Applied identically to all five flink modules (1.18.x/1.19.x/1.20.x/2.0.x/2.1.x). Adds integration test ITTestHoodieDataSource#testParquetNestedRowExceedingReadBatch covering both a heap-vector nested row and an isolated decimal-only nested row across the batch boundary. closes #19208 (cherry picked from commit ee3692cf3142c293b2ec36e520c252936a8c7af3) --- .../hudi/table/ITTestHoodieDataSource.java | 119 ++++++++++++++++++ .../cow/vector/reader/NestedColumnReader.java | 33 ++++- .../cow/vector/reader/NestedColumnReader.java | 33 ++++- .../cow/vector/reader/NestedColumnReader.java | 33 ++++- .../cow/vector/reader/NestedColumnReader.java | 33 ++++- .../cow/vector/reader/NestedColumnReader.java | 33 ++++- 6 files changed, 279 insertions(+), 5 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 148e14f30b2f7..c5e38bde48838 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -120,6 +120,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertLinesMatch; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -2374,6 +2375,124 @@ void testParquetArrayMapOfRowTypes(String operation) { assertRowsEqualsUnordered(expected, result); } + @Test + void testParquetNestedRowExceedingReadBatch() { + // Regression for NestedColumnReader#readRow throwing ArrayIndexOutOfBoundsException when a COW + // base file holds more rows than the 2048-row vectorized read batch + // (RecordIterators.DEFAULT_BATCH_SIZE) and a nested ROW column is read. On a full, non-final + // batch the Dremel level stream carries a one-record lookahead, so NestedPositionUtil + // #calculateRowOffsets returns positionsCount = batchSize + 1 = 2049 while the materialized + // child column vectors are sized to their value count = 2048. The Hudi-specific null-row-collapse + // loop iterates to positionsCount and reads child.isNullAt(2048), one past a length-2048 vector. + // + // Two conditions are both required to surface it, and drove this schema and data: + // 1. The bad index is only reached through AbstractHeapVector#isNullAt, which short-circuits to + // false without touching isNull[] when the vector has no nulls. So a child vector must + // actually carry a null. Odd-id rows therefore store a present ROW with all-null children + // (row(null, ...)); the row stays present (its own isNullAt(2048) short-circuits) but the + // child leaf vectors get noNulls=false and overrun at the phantom index. Half the rows are + // null-children so the first full batch is guaranteed to contain them regardless of how + // bulk_insert orders keys. + // 2. The nullable leaves must be *direct* children of the collapsed row. A sub-row child would + // be renewed to positionsCount (length 2049) and not overrun, so the two nested rows are + // top-level columns: f_scalar row(f0 int, f1 varchar(10)) covers heap-vector children, and + // f_dec row(d decimal(10, 2)) covers a decimal child, whose ParquetDecimalVector is not an + // AbstractHeapVector and must be unwrapped by NestedColumnReader#vectorLength. + // See ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes for the collapse behaviour. + TableEnvironment tableEnv = batchTableEnv; + + // More rows than one 2048-row read batch, so the first batch is full and non-final -- that is + // what makes the level stream carry the trailing lookahead that overshoots the vectors. The + // rows are generated by cross joining two small VALUES lists rather than a single 2000+-row + // VALUES literal: Calcite plans the latter pathologically slowly (minutes to hours), while two + // ~50-element lists plan instantly and the row count is simply their product. + final int outer = 43; + final int inner = 50; + final int numRows = outer * inner; // 2150 > 2048 + + String hoodieTableDDL = sql("t1") + .field("f_int int") + .field("f_scalar row(f0 int, f1 varchar(10))") + .field("f_dec row(d decimal(10, 2))") + .pkField("f_int") + .noPartition() + .option(FlinkOptions.PATH, tempFile.getAbsolutePath()) + .option(FlinkOptions.OPERATION, "bulk_insert") + // Single write task => all rows land in one base file, so one read split crosses the + // 2048-row batch boundary. + .option(FlinkOptions.WRITE_TASKS, 1) + .end(); + tableEnv.executeSql(hoodieTableDDL); + + // id = blk * inner + pos is unique over blk in [0, outer), pos in [0, inner) => 0 .. numRows-1. + // Both nested rows stay present; even ids get populated leaves, odd ids get all-null leaves + // (which the reader collapses back to a NULL row). Each ROW is cast to its named type so the + // query output type matches the sink column exactly. + String insert = "insert into t1 select\n" + + " g.id,\n" + + " cast(row(\n" + + " case when mod(g.id, 2) = 0 then g.id else cast(null as int) end,\n" + + " case when mod(g.id, 2) = 0 then concat('v', cast(g.id as varchar)) else cast(null as varchar(10)) end\n" + + " ) as row),\n" + + " cast(row(\n" + + " case when mod(g.id, 2) = 0 then cast(g.id as decimal(10, 2)) else cast(null as decimal(10, 2)) end\n" + + " ) as row)\n" + + "from (\n" + + " select blk.b * " + inner + " + pos.p as id\n" + + " from (values " + valuesList(outer) + ") as blk(b)\n" + + " cross join (values " + valuesList(inner) + ") as pos(p)\n" + + ") g"; + execInsertSql(tableEnv, insert); + + List result = CollectionUtil.iterableToList( + () -> tableEnv.sqlQuery("select * from t1").execute().collect()); + + // The read completes (no AIOOBE across the batch boundary) and every row is returned. Without + // the fix the vectorized read throws while materializing the first full batch, so this fails. + assertEquals(numRows, result.size()); + + // bulk_insert does not preserve order, so index by pk. + Map byId = new HashMap<>(); + for (Row r : result) { + byId.put((Integer) r.getField(0), r); + } + // Populated rows (even id) round-trip both nested rows -- one from the first (full) batch and + // one with a large id past the boundary. + assertPopulatedRow(byId.get(0), 0); + assertPopulatedRow(byId.get(numRows - 2), numRows - 2); + // All-null-children rows (odd id) collapse both nested rows back to NULL, including a large id. + assertCollapsedRow(byId.get(1)); + assertCollapsedRow(byId.get(numRows - 1)); + } + + /** Builds the VALUES row list {@code (0), (1), ..., (n-1)} for the generator cross join. */ + private static String valuesList(int n) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < n; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append('(').append(i).append(')'); + } + return sb.toString(); + } + + /** Asserts the row keyed by an even {@code id} round-trips its populated nested rows. */ + private static void assertPopulatedRow(Row row, int id) { + assertNotNull(row, "row with pk " + id + " was not read back"); + Row scalar = (Row) row.getField(1); + assertEquals(id, scalar.getField(0)); + assertEquals("v" + id, scalar.getField(1)); + assertNotNull(((Row) row.getField(2)).getField(0)); // decimal leaf present, not null + } + + /** Asserts the row keyed by an odd {@code id} had both all-null nested rows collapsed to NULL. */ + private static void assertCollapsedRow(Row row) { + assertNotNull(row, "expected an odd-id row to be read back"); + assertNull(row.getField(1)); // f_scalar collapsed to null + assertNull(row.getField(2)); // f_dec collapsed to null + } + @ParameterizedTest @ValueSource(strings = {"insert", "upsert", "bulk_insert"}) void testParquetNullChildColumnsRowTypes(String operation) { diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 2e0bd744cab1c..ac94292c3315f 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; import org.apache.hudi.table.format.cow.vector.position.RowPosition; @@ -156,7 +157,21 @@ private Tuple2 readRow( // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. - int rowCount = rowPosition.getPositionsCount(); + // positionsCount comes from the Dremel definition/repetition level stream + // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a + // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the + // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch), + // so positionsCount can be one larger than the materialized vector lengths. When inside==true + // the row vector is renewed to positionsCount but its children are sized to their value count; + // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way + // to positionsCount can read one element past a shorter vector and throw + // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom + // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch + // at num). + int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen()); + for (WritableColumnVector child : finalChildrenVectors) { + rowCount = Math.min(rowCount, vectorLength(child)); + } for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { continue; @@ -272,6 +287,22 @@ private Tuple2 readPrimitive( return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); } + /** + * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed + * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link + * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link + * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code + * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code + * AbstractHeapVector} in all cases. + */ + private static int vectorLength(ColumnVector vector) { + ColumnVector storage = + vector instanceof ParquetDecimalVector + ? ((ParquetDecimalVector) vector).getVector() + : vector; + return ((AbstractHeapVector) storage).getLen(); + } + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { if (nullFlags[index]) { diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 2e0bd744cab1c..ac94292c3315f 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; import org.apache.hudi.table.format.cow.vector.position.RowPosition; @@ -156,7 +157,21 @@ private Tuple2 readRow( // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. - int rowCount = rowPosition.getPositionsCount(); + // positionsCount comes from the Dremel definition/repetition level stream + // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a + // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the + // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch), + // so positionsCount can be one larger than the materialized vector lengths. When inside==true + // the row vector is renewed to positionsCount but its children are sized to their value count; + // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way + // to positionsCount can read one element past a shorter vector and throw + // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom + // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch + // at num). + int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen()); + for (WritableColumnVector child : finalChildrenVectors) { + rowCount = Math.min(rowCount, vectorLength(child)); + } for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { continue; @@ -272,6 +287,22 @@ private Tuple2 readPrimitive( return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); } + /** + * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed + * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link + * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link + * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code + * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code + * AbstractHeapVector} in all cases. + */ + private static int vectorLength(ColumnVector vector) { + ColumnVector storage = + vector instanceof ParquetDecimalVector + ? ((ParquetDecimalVector) vector).getVector() + : vector; + return ((AbstractHeapVector) storage).getLen(); + } + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { if (nullFlags[index]) { diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 7ec70490dc45b..60575f148cc45 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; import org.apache.hudi.table.format.cow.vector.position.RowPosition; @@ -155,7 +156,21 @@ private Tuple2 readRow( // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. - int rowCount = rowPosition.getPositionsCount(); + // positionsCount comes from the Dremel definition/repetition level stream + // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a + // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the + // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch), + // so positionsCount can be one larger than the materialized vector lengths. When inside==true + // the row vector is renewed to positionsCount but its children are sized to their value count; + // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way + // to positionsCount can read one element past a shorter vector and throw + // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom + // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch + // at num). + int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen()); + for (WritableColumnVector child : finalChildrenVectors) { + rowCount = Math.min(rowCount, vectorLength(child)); + } for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { continue; @@ -271,6 +286,22 @@ private Tuple2 readPrimitive( return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); } + /** + * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed + * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link + * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link + * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code + * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code + * AbstractHeapVector} in all cases. + */ + private static int vectorLength(ColumnVector vector) { + ColumnVector storage = + vector instanceof ParquetDecimalVector + ? ((ParquetDecimalVector) vector).getVector() + : vector; + return ((AbstractHeapVector) storage).getLen(); + } + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { if (nullFlags[index]) { diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 7ec70490dc45b..60575f148cc45 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; import org.apache.hudi.table.format.cow.vector.position.RowPosition; @@ -155,7 +156,21 @@ private Tuple2 readRow( // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. - int rowCount = rowPosition.getPositionsCount(); + // positionsCount comes from the Dremel definition/repetition level stream + // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a + // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the + // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch), + // so positionsCount can be one larger than the materialized vector lengths. When inside==true + // the row vector is renewed to positionsCount but its children are sized to their value count; + // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way + // to positionsCount can read one element past a shorter vector and throw + // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom + // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch + // at num). + int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen()); + for (WritableColumnVector child : finalChildrenVectors) { + rowCount = Math.min(rowCount, vectorLength(child)); + } for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { continue; @@ -271,6 +286,22 @@ private Tuple2 readPrimitive( return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); } + /** + * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed + * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link + * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link + * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code + * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code + * AbstractHeapVector} in all cases. + */ + private static int vectorLength(ColumnVector vector) { + ColumnVector storage = + vector instanceof ParquetDecimalVector + ? ((ParquetDecimalVector) vector).getVector() + : vector; + return ((AbstractHeapVector) storage).getLen(); + } + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { if (nullFlags[index]) { diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java index 00abceeee4fab..7be03dacc2183 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.table.format.cow.vector.HeapArrayVector; import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector; import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector; +import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector; import org.apache.hudi.table.format.cow.vector.position.CollectionPosition; import org.apache.hudi.table.format.cow.vector.position.LevelDelegation; import org.apache.hudi.table.format.cow.vector.position.RowPosition; @@ -158,7 +159,21 @@ private Tuple2 readRow( // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes. - int rowCount = rowPosition.getPositionsCount(); + // positionsCount comes from the Dremel definition/repetition level stream + // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a + // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the + // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch), + // so positionsCount can be one larger than the materialized vector lengths. When inside==true + // the row vector is renewed to positionsCount but its children are sized to their value count; + // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way + // to positionsCount can read one element past a shorter vector and throw + // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom + // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch + // at num). + int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen()); + for (WritableColumnVector child : finalChildrenVectors) { + rowCount = Math.min(rowCount, vectorLength(child)); + } for (int j = 0; j < rowCount; j++) { if (heapRowVector.isNullAt(j)) { continue; @@ -274,6 +289,22 @@ private Tuple2 readPrimitive( return Tuple2.of(reader.getLevelDelegation(), writableColumnVector); } + /** + * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed + * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link + * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link + * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code + * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code + * AbstractHeapVector} in all cases. + */ + private static int vectorLength(ColumnVector vector) { + ColumnVector storage = + vector instanceof ParquetDecimalVector + ? ((ParquetDecimalVector) vector).getVector() + : vector; + return ((AbstractHeapVector) storage).getLen(); + } + private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) { for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) { if (nullFlags[index]) { From ac17e61c683e7e53ccf1e1232671a1fafdafa088 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 8 Jul 2026 18:57:59 +0800 Subject: [PATCH 143/255] =?UTF-8?q?test(integ-test):=20add=20Testcontainer?= =?UTF-8?q?s=20E2E=20for=20VECTOR/BLOB/VARIANT=20Hive=E2=80=A6=20(#19203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(integ-test): add Testcontainers E2E for VECTOR/BLOB/VARIANT Hive sync Testcontainers-based E2E coverage that a Spark write of Hudi's custom logical types produces a Hive-syncable schema queryable through a real Hive metastore. ITTestCustomTypeHiveSync exercises VECTOR and BLOB (SQL + DataFrame) plus Spark 4.0 VARIANT (Spark4-gated), asserting write -> hive-sync -> Hive DESCRIBE/SHOW PARTITIONS/SELECT round-trips. Runs on both a Spark 3.5.3 (JDK 11) and Spark 4.0.2 (JDK 17) stack against Hive 2.3.10, via a new integration-tests-hive-sync CI job. The integ2 package manages its own docker stack and is excluded from the legacy in-job compose demo to avoid host-port collisions. Scoped decomposition of #18535 (Hive-sync half only): the Trino read E2E and two bundled production features (blob partial-struct write, Lance VARIANT support) are deferred to follow-up PRs. * test(integ-test): address review comments on testcontainers harness - delete unused-and-broken HiveService.runQuery (execInContainer has no shell, so the added literal quotes would reach hive -e and break parsing) - fix assertStdOutContains javadoc: asserts exactly once, not at least once - gate integration-tests-hive-sync CI job on the changes job's relevance output, matching the legacy integration-tests job - trim unused harness API: sparkAdhoc2, unused Containers/Paths/Network constants, CommandResult.expectToFail/assertExitCodeIs/assertStdErrContains, HiveService.executeFile/copyFile, SparkService.copyFile, CommandExecutor.copyFileToContainer - register compose temp files for deleteOnExit so the temp dir actually deletes; add EOF newline; fix ContainerProvider javadoc; rely on root dependencyManagement for the managed testcontainers artifact * test(integ-test): use jdbc hive_sync mode for VARIANT SQL fixture The VARIANT SQL fixture was the only one syncing via hms mode while retaining a jdbcurl (used only by jdbc mode). Align it with every other fixture, including the DataFrame VARIANT fixture that syncs the identical VariantType, which all use jdbc. * test(integ-test): move VERBOSE_HIVECONFS into SystemProps Keep all verbose-mode config in one place. VERBOSE_HIVECONFS was the only constant at the outer-class level; group it with SystemProps.HIVE_VERBOSE, which its javadoc already references. (cherry picked from commit 3015eb944083272459040dc5da5755e683674cb5) --- .github/workflows/bot.yml | 76 +++++ ...pose_hadoop284_hive2310_spark353_amd64.yml | 76 ----- ...pose_hadoop284_hive2310_spark353_arm64.yml | 72 ----- docker/demo/sparksql-blob-type-df.commands | 108 +++++++ docker/demo/sparksql-blob-type-sql.commands | 126 +++++++++ docker/demo/sparksql-variant-type-df.commands | 80 ++++++ .../demo/sparksql-variant-type-sql.commands | 89 ++++++ docker/demo/sparksql-vector-type-df.commands | 90 ++++++ docker/demo/sparksql-vector-type-sql.commands | 90 ++++++ docker/hoodie/hadoop/hive_base/entrypoint.sh | 2 +- docker/hoodie/hadoop/sparkmaster/master.sh | 2 +- hudi-integ-test/pom.xml | 48 +++- .../testcontainers/ContainerProvider.java | 37 +++ .../ITTestBaseTestcontainers.java | 236 ++++++++++++++++ .../ITTestCustomTypeHiveSync.java | 266 ++++++++++++++++++ .../testcontainers/TestcontainersConfig.java | 110 ++++++++ .../command/CommandExecutor.java | 87 ++++++ .../testcontainers/command/CommandResult.java | 137 +++++++++ .../testcontainers/service/HiveService.java | 72 +++++ .../testcontainers/service/SparkService.java | 71 +++++ 20 files changed, 1724 insertions(+), 151 deletions(-) create mode 100644 docker/demo/sparksql-blob-type-df.commands create mode 100644 docker/demo/sparksql-blob-type-sql.commands create mode 100644 docker/demo/sparksql-variant-type-df.commands create mode 100644 docker/demo/sparksql-variant-type-sql.commands create mode 100644 docker/demo/sparksql-vector-type-df.commands create mode 100644 docker/demo/sparksql-vector-type-sql.commands create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ContainerProvider.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestCustomTypeHiveSync.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandExecutor.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandResult.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/HiveService.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/SparkService.java diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index fb7dd760c5058..851b53e786181 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1251,6 +1251,82 @@ jobs: rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS + integration-tests-hive-sync: + # Testcontainers-based E2E hive sync coverage for Hudi's custom logical types + # (VECTOR, BLOB) and the Spark 4.0 VARIANT type. Runs the integ2 testcontainers + # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on both a + # Spark 3.5.3 and Spark 4.0.2 stack. + runs-on: ubuntu-latest + needs: changes + strategy: + fail-fast: false + matrix: + include: + - sparkProfile: 'spark3.5' + scalaProfile: '-Dscala-2.12 -Dscala.binary.version=2.12' + flinkProfile: 'flink1.20' + jdkVersion: '11' + composePrefix: 'docker-compose_hadoop284_hive2310_spark353' + sparkAdhocImage: 'apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest' + - sparkProfile: 'spark4.0' + scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13' + flinkProfile: 'flink1.20' + jdkVersion: '17' + composePrefix: 'docker-compose_hadoop340_hive2310_spark402' + sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest' + steps: + - if: needs.changes.outputs.relevant == 'true' + uses: actions/checkout@v5 + - name: Set up JDK ${{ matrix.jdkVersion }} + if: needs.changes.outputs.relevant == 'true' + uses: actions/setup-java@v5 + with: + java-version: ${{ matrix.jdkVersion }} + distribution: 'temurin' + architecture: x64 + cache: maven + - name: Free disk space + if: needs.changes.outputs.relevant == 'true' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + docker system prune --all --force --volumes + - name: Pre-pull compose images (fails fast if not published) + if: needs.changes.outputs.relevant == 'true' + env: + SPARK_ADHOC_IMAGE: ${{ matrix.sparkAdhocImage }} + run: | + # Surface missing Spark 4.0.2 images before the 15-minute Maven install. + # The remaining images in the compose stack are pulled by docker-compose at + # test time. + docker pull "$SPARK_ADHOC_IMAGE" + - name: Build and install Hudi artifacts + if: needs.changes.outputs.relevant == 'true' + env: + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + SCALA_PROFILE: ${{ matrix.scalaProfile }} + run: + mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS + - name: Run integ2 testcontainers suite + if: needs.changes.outputs.relevant == 'true' + env: + SPARK_PROFILE: ${{ matrix.sparkProfile }} + SCALA_PROFILE: ${{ matrix.scalaProfile }} + COMPOSE_PREFIX: ${{ matrix.composePrefix }} + run: | + # -DskipITs=false overrides the spark4.0 profile's skipITs=true default + # (see root pom.xml). Without it, failsafe skips all ITs on the spark4.0 matrix row. + mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests \ + -pl hudi-integ-test \ + -DskipITs=false \ + -Ddocker.compose.skip=true \ + -Dit.test='ITTestCustomTypeHiveSync' \ + -Dspark.docker.compose.prefix=$COMPOSE_PREFIX \ + $MVN_ARGS + build-spark-java17: runs-on: ubuntu-latest strategy: diff --git a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml b/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml index 1d84417b378af..c302a3c95cca8 100644 --- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml +++ b/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml @@ -186,78 +186,6 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes - presto-coordinator-1: - container_name: presto-coordinator-1 - hostname: presto-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest - platform: linux/amd64 - ports: - - "8090:8090" - environment: - - PRESTO_JVM_MAX_HEAP=512M - - PRESTO_QUERY_MAX_MEMORY=1GB - - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB - - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB - - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB - - TERM=xterm - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: coordinator - - presto-worker-1: - container_name: presto-worker-1 - hostname: presto-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest - platform: linux/amd64 - depends_on: [ "presto-coordinator-1" ] - environment: - - PRESTO_JVM_MAX_HEAP=512M - - PRESTO_QUERY_MAX_MEMORY=1GB - - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB - - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB - - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB - - TERM=xterm - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: worker - - trino-coordinator-1: - container_name: trino-coordinator-1 - hostname: trino-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-trinocoordinator_368:latest - platform: linux/amd64 - ports: - - "8091:8091" - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-coordinator-1 - - trino-worker-1: - container_name: trino-worker-1 - hostname: trino-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-trinoworker_368:latest - platform: linux/amd64 - depends_on: [ "trino-coordinator-1" ] - ports: - - "8092:8092" - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-worker-1 - graphite: container_name: graphite hostname: graphite @@ -287,8 +215,6 @@ services: - "hiveserver" - "hive-metastore-postgresql" - "namenode" - - "presto-coordinator-1" - - "trino-coordinator-1" volumes: - ${HUDI_WS}:/var/hoodie/ws @@ -311,8 +237,6 @@ services: - "hiveserver" - "hive-metastore-postgresql" - "namenode" - - "presto-coordinator-1" - - "trino-coordinator-1" volumes: - ${HUDI_WS}:/var/hoodie/ws diff --git a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml b/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml index b995859c80222..a9f042155d300 100644 --- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml +++ b/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml @@ -179,74 +179,6 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes - presto-coordinator-1: - container_name: presto-coordinator-1 - hostname: presto-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest - ports: - - "8090:8090" - environment: - - PRESTO_JVM_MAX_HEAP=512M - - PRESTO_QUERY_MAX_MEMORY=1GB - - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB - - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB - - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB - - TERM=xterm - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: coordinator - - presto-worker-1: - container_name: presto-worker-1 - hostname: presto-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest - depends_on: [ "presto-coordinator-1" ] - environment: - - PRESTO_JVM_MAX_HEAP=512M - - PRESTO_QUERY_MAX_MEMORY=1GB - - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB - - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB - - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB - - TERM=xterm - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: worker - - trino-coordinator-1: - container_name: trino-coordinator-1 - hostname: trino-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-trinocoordinator_368:latest - ports: - - "8091:8091" - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-coordinator-1 - - trino-worker-1: - container_name: trino-worker-1 - hostname: trino-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-trinoworker_368:latest - depends_on: [ "trino-coordinator-1" ] - ports: - - "8092:8092" - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-worker-1 - graphite: container_name: graphite hostname: graphite @@ -275,8 +207,6 @@ services: - "hiveserver" - "hive-metastore-postgresql" - "namenode" - - "presto-coordinator-1" - - "trino-coordinator-1" volumes: - ${HUDI_WS}:/var/hoodie/ws @@ -298,8 +228,6 @@ services: - "hiveserver" - "hive-metastore-postgresql" - "namenode" - - "presto-coordinator-1" - - "trino-coordinator-1" volumes: - ${HUDI_WS}:/var/hoodie/ws diff --git a/docker/demo/sparksql-blob-type-df.commands b/docker/demo/sparksql-blob-type-df.commands new file mode 100644 index 0000000000000..28e0663d74e94 --- /dev/null +++ b/docker/demo/sparksql-blob-type-df.commands @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Row, SaveMode} + +// BLOB is a struct with fields: type (string), data (binary, nullable), reference (struct, nullable) +val blobMetadata = new MetadataBuilder().putString("hudi_type", "BLOB").build() + +val referenceType = StructType(Seq( + StructField("external_path", StringType, nullable = false), + StructField("offset", LongType, nullable = true), + StructField("length", LongType, nullable = true), + StructField("managed", BooleanType, nullable = false) +)) + +val blobType = StructType(Seq( + StructField("type", StringType, nullable = false), + StructField("data", BinaryType, nullable = true), + StructField("reference", referenceType, nullable = true) +)) + +val schema = StructType(Seq( + StructField("id", LongType, nullable = false), + StructField("name", StringType), + StructField("blob_data", blobType, nullable = false, metadata = blobMetadata), + StructField("dt", StringType) +)) + +// Shared Hive-sync + write options factored out so the upsert and delete writes +// reuse the exact same sync configuration as the initial Overwrite. +// Body is wrapped in { } so the spark-shell REPL keeps the chained .option(...) +// calls attached to the def. Without braces, writer.format("hudi") on the first +// body line parses as a complete expression, the REPL closes the def there, and +// the remaining .option(...) lines get dot-applied to `sc` (SparkContext). +def applyWriteOpts(writer: org.apache.spark.sql.DataFrameWriter[Row]): org.apache.spark.sql.DataFrameWriter[Row] = { + writer.format("hudi") + .option("hoodie.table.name", "blob_test_df") + .option("hoodie.datasource.write.recordkey.field", "id") + .option("hoodie.datasource.write.precombine.field", "name") + .option("hoodie.datasource.write.partitionpath.field", "dt") + .option("hoodie.datasource.hive_sync.enable", "true") + .option("hoodie.datasource.hive_sync.database", "default") + .option("hoodie.datasource.hive_sync.table", "blob_test_df") + .option("hoodie.datasource.hive_sync.jdbcurl", "jdbc:hive2://hiveserver:10000/") + .option("hoodie.datasource.hive_sync.mode", "jdbc") + .option("hoodie.datasource.hive_sync.partition_fields", "dt") + .option("hoodie.datasource.hive_sync.partition_extractor_class", + "org.apache.hudi.hive.MultiPartKeysValueExtractor") + .option("hoodie.datasource.hive_sync.username", "hive") + .option("hoodie.datasource.hive_sync.password", "hive") +} + +// Seed two rows in dt='2024-01-01' via Overwrite. +val seed = Seq( + Row(1L, "file1", Row("INLINE", "hello world".getBytes, null), "2024-01-01"), + Row(2L, "file2", Row("INLINE", "test data".getBytes, null), "2024-01-01") +) +val seedDf = spark.createDataFrame(spark.sparkContext.parallelize(seed), schema) +applyWriteOpts(seedDf.write).mode(SaveMode.Overwrite).save("/user/hive/warehouse/blob_test_df") +spark.sql("select id, name, blob_data.type, dt from blob_test_df order by id").show(10, false) +println("BLOB_DF_INSERT_SUCCESS") + +// Append-upsert: id=2 value changes, id=3 new in a new partition dt='2024-01-02'. +// precombine=name must strictly increase for the matched row so Hudi keeps the +// new value (alphabetical 'file2-updated' > 'file2'). +val upsertRows = Seq( + Row(2L, "file2-updated", Row("INLINE", "updated payload".getBytes, null), "2024-01-01"), + Row(3L, "file3", Row("INLINE", "brand new".getBytes, null), "2024-01-02") +) +val upsertDf = spark.createDataFrame(spark.sparkContext.parallelize(upsertRows), schema) +applyWriteOpts(upsertDf.write) + .option("hoodie.datasource.write.operation", "upsert") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/blob_test_df") +spark.sql("select id, name, blob_data.type, dt from blob_test_df order by id").show(10, false) +println("BLOB_DF_UPSERT_SUCCESS") + +// Delete id=3 via operation=delete. The DF still needs a schema-compatible BLOB +// column (Hudi uses only the record key/partition for delete, the payload is +// ignored but must deserialize). +val deleteRows = Seq( + Row(3L, "file3", Row("INLINE", "ignored".getBytes, null), "2024-01-02") +) +val deleteDf = spark.createDataFrame(spark.sparkContext.parallelize(deleteRows), schema) +applyWriteOpts(deleteDf.write) + .option("hoodie.datasource.write.operation", "delete") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/blob_test_df") +spark.sql("select id, name, blob_data.type, dt from blob_test_df order by id").show(10, false) +println("BLOB_DF_DELETE_SUCCESS") + +println("BLOB_DF_TEST_SUCCESS") diff --git a/docker/demo/sparksql-blob-type-sql.commands b/docker/demo/sparksql-blob-type-sql.commands new file mode 100644 index 0000000000000..8a03b1d16f305 --- /dev/null +++ b/docker/demo/sparksql-blob-type-sql.commands @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// E2E test: BLOB type with Hive sync via SQL CREATE TABLE path. +// BLOB is parsed by HoodieSpark3_5ExtendedSqlAstBuilder into a struct +// >. +spark.sql(""" + CREATE TABLE blob_test ( + id LONG, + name STRING, + blob_data BLOB, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/blob_test' + TBLPROPERTIES ( + 'primaryKey' = 'id', + 'preCombineField' = 'name', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'blob_test', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +// Seed two rows in dt='2024-01-01'. Use OUT_OF_LINE here so subsequent UPDATE/MERGE +// can mutate to a different reference without changing the struct shape between ops +// (avoids mixing INLINE and OUT_OF_LINE branches in a single test). +spark.sql(""" + INSERT INTO blob_test VALUES + (1, 'file1', named_struct( + 'type', 'OUT_OF_LINE', + 'data', cast(null as binary), + 'reference', named_struct( + 'external_path', 'blobs/seed-1', + 'offset', 0L, + 'length', 11L, + 'managed', false)), '2024-01-01'), + (2, 'file2', named_struct( + 'type', 'OUT_OF_LINE', + 'data', cast(null as binary), + 'reference', named_struct( + 'external_path', 'blobs/seed-2', + 'offset', 0L, + 'length', 9L, + 'managed', false)), '2024-01-01') +""") +spark.sql("select id, name, blob_data.type, dt from blob_test").show(10, false) +println("BLOB_SQL_INSERT_SUCCESS") + +// UPDATE exercises the BLOB metadata re-attach on the UPDATE write path. +// Per RFC-100 external_path and managed are non-null. +spark.sql(""" + UPDATE blob_test + SET blob_data = named_struct( + 'type', 'OUT_OF_LINE', + 'data', cast(null as binary), + 'reference', named_struct( + 'external_path', 'blobs/updated-1', + 'offset', 10L, + 'length', 100L, + 'managed', true)), + name = 'file1-updated' + WHERE id = 1 +""") +spark.sql("select id, name, blob_data.reference.external_path from blob_test where id = 1").show(10, false) +println("BLOB_SQL_UPDATE_SUCCESS") + +// MERGE exercises both MATCHED (UPDATE SET) and NOT MATCHED (INSERT of a new +// row into a new partition dt='2024-01-02'). The USING CTE strips the hudi_type +// metadata from the struct column, so this specifically validates the reattach +// path for MERGE. +spark.sql(""" + MERGE INTO blob_test t + USING ( + SELECT 2L AS id, 'file2-merged' AS name, named_struct( + 'type', 'OUT_OF_LINE', + 'data', cast(null as binary), + 'reference', named_struct( + 'external_path', 'blobs/merged-2', + 'offset', 20L, + 'length', 200L, + 'managed', true)) AS blob_data, '2024-01-01' AS dt + UNION ALL + SELECT 3L AS id, 'file3' AS name, named_struct( + 'type', 'OUT_OF_LINE', + 'data', cast(null as binary), + 'reference', named_struct( + 'external_path', 'blobs/inserted-3', + 'offset', 300L, + 'length', 30L, + 'managed', false)) AS blob_data, '2024-01-02' AS dt + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET t.name = s.name, t.blob_data = s.blob_data + WHEN NOT MATCHED THEN INSERT (id, name, blob_data, dt) VALUES (s.id, s.name, s.blob_data, s.dt) +""") +spark.sql("select id, name, blob_data.reference.external_path, dt from blob_test order by id").show(10, false) +println("BLOB_SQL_MERGE_SUCCESS") + +// DELETE the NOT MATCHED row we just inserted, returning to 2 rows total. +spark.sql("DELETE FROM blob_test WHERE id = 3") +spark.sql("select id, name, blob_data.type, dt from blob_test order by id").show(10, false) +println("BLOB_SQL_DELETE_SUCCESS") + +println("BLOB_SQL_TEST_SUCCESS") diff --git a/docker/demo/sparksql-variant-type-df.commands b/docker/demo/sparksql-variant-type-df.commands new file mode 100644 index 0000000000000..dbbeabdad039c --- /dev/null +++ b/docker/demo/sparksql-variant-type-df.commands @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.spark.sql.{DataFrame, SaveMode} + +// E2E test: VARIANT type with Hive sync via DataFrame API path. +// Tests that the HiveSyncTool correctly maps VARIANT to struct in Hive. + +// Body is wrapped in { } so the spark-shell REPL keeps the chained .option(...) +// calls attached to the def. Without braces, writer.format("hudi") on the first +// body line parses as a complete expression, the REPL closes the def there, and +// the remaining .option(...) lines get dot-applied to `sc` (SparkContext). +def applyWriteOpts(writer: org.apache.spark.sql.DataFrameWriter[_]): org.apache.spark.sql.DataFrameWriter[_] = { + writer.format("hudi") + .option("hoodie.table.name", "variant_test_df") + .option("hoodie.datasource.write.recordkey.field", "id") + .option("hoodie.datasource.write.precombine.field", "name") + .option("hoodie.datasource.write.partitionpath.field", "dt") + .option("hoodie.datasource.hive_sync.enable", "true") + .option("hoodie.datasource.hive_sync.database", "default") + .option("hoodie.datasource.hive_sync.table", "variant_test_df") + .option("hoodie.datasource.hive_sync.jdbcurl", "jdbc:hive2://hiveserver:10000/") + .option("hoodie.datasource.hive_sync.mode", "jdbc") + .option("hoodie.datasource.hive_sync.partition_fields", "dt") + .option("hoodie.datasource.hive_sync.partition_extractor_class", + "org.apache.hudi.hive.MultiPartKeysValueExtractor") + .option("hoodie.datasource.hive_sync.username", "hive") + .option("hoodie.datasource.hive_sync.password", "hive") +} + +// Seed two rows in dt='2024-01-01' via Overwrite. +val seedDf: DataFrame = spark.sql(""" + SELECT 1L AS id, 'row1' AS name, parse_json('{"key":"value1"}') AS variant_data, '2024-01-01' AS dt + UNION ALL + SELECT 2L AS id, 'row2' AS name, parse_json('{"key":"value2"}') AS variant_data, '2024-01-01' AS dt +""") +applyWriteOpts(seedDf.write).mode(SaveMode.Overwrite).save("/user/hive/warehouse/variant_test_df") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test_df order by id").show(10, false) +println("VARIANT_DF_INSERT_SUCCESS") + +// Append-upsert: id=2 mutated, id=3 new in dt='2024-01-02'. +val upsertDf: DataFrame = spark.sql(""" + SELECT 2L AS id, 'row2-updated' AS name, parse_json('{"key":"value2-updated"}') AS variant_data, '2024-01-01' AS dt + UNION ALL + SELECT 3L AS id, 'row3' AS name, parse_json('{"key":"value3"}') AS variant_data, '2024-01-02' AS dt +""") +applyWriteOpts(upsertDf.write) + .option("hoodie.datasource.write.operation", "upsert") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/variant_test_df") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test_df order by id").show(10, false) +println("VARIANT_DF_UPSERT_SUCCESS") + +// Delete id=3 via operation=delete. +val deleteDf: DataFrame = spark.sql(""" + SELECT 3L AS id, 'row3' AS name, parse_json('{}') AS variant_data, '2024-01-02' AS dt +""") +applyWriteOpts(deleteDf.write) + .option("hoodie.datasource.write.operation", "delete") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/variant_test_df") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test_df order by id").show(10, false) +println("VARIANT_DF_DELETE_SUCCESS") + +println("VARIANT_DF_TEST_SUCCESS") diff --git a/docker/demo/sparksql-variant-type-sql.commands b/docker/demo/sparksql-variant-type-sql.commands new file mode 100644 index 0000000000000..c21a62674ef34 --- /dev/null +++ b/docker/demo/sparksql-variant-type-sql.commands @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// E2E test: VARIANT type with Hive sync via SQL CREATE TABLE path. +// Tests that VariantType is mapped to struct in Hive. +spark.sql(""" + CREATE TABLE variant_test ( + id LONG, + name STRING, + variant_data VARIANT, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/variant_test' + TBLPROPERTIES ( + 'primaryKey' = 'id', + 'preCombineField' = 'name', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'variant_test', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql(""" + INSERT INTO variant_test VALUES + (1, 'row1', parse_json('{"key":"value1"}'), '2024-01-01'), + (2, 'row2', parse_json('{"key":"value2"}'), '2024-01-01') +""") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test order by id").show(10, false) +println("VARIANT_SQL_INSERT_SUCCESS") + +// UPDATE against a VariantType column. Hudi's V1 writer accepts VariantType +// writes on Spark 4 (see fix f395eea4b4a9); UPDATE exercises the same writer +// path with castIfNeeded applied to the assignment. +spark.sql(""" + UPDATE variant_test + SET variant_data = parse_json('{"key":"value1-updated"}'), + name = 'row1-updated' + WHERE id = 1 +""") +spark.sql("select id, name, cast(variant_data as string) from variant_test where id = 1").show(10, false) +println("VARIANT_SQL_UPDATE_SUCCESS") + +// MERGE exercises both MATCHED (UPDATE) and NOT MATCHED (INSERT into new +// partition dt='2024-01-02'). +spark.sql(""" + MERGE INTO variant_test t + USING ( + SELECT 2L AS id, 'row2-merged' AS name, + parse_json('{"key":"value2-merged"}') AS variant_data, + '2024-01-01' AS dt + UNION ALL + SELECT 3L AS id, 'row3' AS name, + parse_json('{"key":"value3"}') AS variant_data, + '2024-01-02' AS dt + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET t.name = s.name, t.variant_data = s.variant_data + WHEN NOT MATCHED THEN INSERT (id, name, variant_data, dt) VALUES (s.id, s.name, s.variant_data, s.dt) +""") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test order by id").show(10, false) +println("VARIANT_SQL_MERGE_SUCCESS") + +spark.sql("DELETE FROM variant_test WHERE id = 3") +spark.sql("select id, name, cast(variant_data as string), dt from variant_test order by id").show(10, false) +println("VARIANT_SQL_DELETE_SUCCESS") + +println("VARIANT_SQL_TEST_SUCCESS") diff --git a/docker/demo/sparksql-vector-type-df.commands b/docker/demo/sparksql-vector-type-df.commands new file mode 100644 index 0000000000000..ef71771be2953 --- /dev/null +++ b/docker/demo/sparksql-vector-type-df.commands @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Row, SaveMode} + +// Create schema with VECTOR metadata + partition column +val metadata = new MetadataBuilder().putString("hudi_type", "VECTOR(3)").build() +val schema = StructType(Seq( + StructField("id", LongType, nullable = false), + StructField("name", StringType), + StructField("embedding", ArrayType(FloatType, containsNull = false), nullable = false, metadata = metadata), + StructField("dt", StringType) +)) + +// Body is wrapped in { } so the spark-shell REPL keeps the chained .option(...) +// calls attached to the def. Without braces, writer.format("hudi") on the first +// body line parses as a complete expression, the REPL closes the def there, and +// the remaining .option(...) lines get dot-applied to `sc` (SparkContext). +def applyWriteOpts(writer: org.apache.spark.sql.DataFrameWriter[Row]): org.apache.spark.sql.DataFrameWriter[Row] = { + writer.format("hudi") + .option("hoodie.table.name", "vector_test_df") + .option("hoodie.datasource.write.recordkey.field", "id") + .option("hoodie.datasource.write.precombine.field", "name") + .option("hoodie.datasource.write.partitionpath.field", "dt") + .option("hoodie.datasource.hive_sync.enable", "true") + .option("hoodie.datasource.hive_sync.database", "default") + .option("hoodie.datasource.hive_sync.table", "vector_test_df") + .option("hoodie.datasource.hive_sync.jdbcurl", "jdbc:hive2://hiveserver:10000/") + .option("hoodie.datasource.hive_sync.mode", "jdbc") + .option("hoodie.datasource.hive_sync.partition_fields", "dt") + .option("hoodie.datasource.hive_sync.partition_extractor_class", + "org.apache.hudi.hive.MultiPartKeysValueExtractor") + .option("hoodie.datasource.hive_sync.username", "hive") + .option("hoodie.datasource.hive_sync.password", "hive") +} + +// Seed two rows in dt='2024-01-01' via Overwrite. +val seed = Seq( + Row(1L, "doc1", Seq(0.1f, 0.2f, 0.3f), "2024-01-01"), + Row(2L, "doc2", Seq(0.4f, 0.5f, 0.6f), "2024-01-01") +) +val seedDf = spark.createDataFrame(spark.sparkContext.parallelize(seed), schema) +applyWriteOpts(seedDf.write).mode(SaveMode.Overwrite).save("/user/hive/warehouse/vector_test_df") +spark.sql("select id, name, embedding, dt from vector_test_df order by id").show(10, false) +println("VECTOR_DF_INSERT_SUCCESS") + +// Append-upsert: id=2 mutated, id=3 new in dt='2024-01-02'. New preCombine +// value (doc2-updated) sorts strictly after the seed (doc2). +val upsertRows = Seq( + Row(2L, "doc2-updated", Seq(0.41f, 0.51f, 0.61f), "2024-01-01"), + Row(3L, "doc3", Seq(0.7f, 0.8f, 0.9f), "2024-01-02") +) +val upsertDf = spark.createDataFrame(spark.sparkContext.parallelize(upsertRows), schema) +applyWriteOpts(upsertDf.write) + .option("hoodie.datasource.write.operation", "upsert") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/vector_test_df") +spark.sql("select id, name, embedding, dt from vector_test_df order by id").show(10, false) +println("VECTOR_DF_UPSERT_SUCCESS") + +// Delete id=3 via operation=delete. The DF still needs a schema-compatible +// VECTOR payload (Hudi uses only the record key/partition for delete). +val deleteRows = Seq( + Row(3L, "doc3", Seq(0.0f, 0.0f, 0.0f), "2024-01-02") +) +val deleteDf = spark.createDataFrame(spark.sparkContext.parallelize(deleteRows), schema) +applyWriteOpts(deleteDf.write) + .option("hoodie.datasource.write.operation", "delete") + .mode(SaveMode.Append) + .save("/user/hive/warehouse/vector_test_df") +spark.sql("select id, name, embedding, dt from vector_test_df order by id").show(10, false) +println("VECTOR_DF_DELETE_SUCCESS") + +println("VECTOR_DF_TEST_SUCCESS") diff --git a/docker/demo/sparksql-vector-type-sql.commands b/docker/demo/sparksql-vector-type-sql.commands new file mode 100644 index 0000000000000..e87736a4be2e2 --- /dev/null +++ b/docker/demo/sparksql-vector-type-sql.commands @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// E2E test: VECTOR type with Hive sync via SQL CREATE TABLE path. +// Runs the full INSERT -> UPDATE -> MERGE -> DELETE lifecycle to exercise +// the custom-type metadata re-attach path on every write command. +spark.sql(""" + CREATE TABLE vector_test ( + id LONG, + name STRING, + embedding VECTOR(3), + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/vector_test' + TBLPROPERTIES ( + 'primaryKey' = 'id', + 'preCombineField' = 'name', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'vector_test', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql(""" + INSERT INTO vector_test VALUES + (1, 'doc1', array(cast(0.1 as float), cast(0.2 as float), cast(0.3 as float)), '2024-01-01'), + (2, 'doc2', array(cast(0.4 as float), cast(0.5 as float), cast(0.6 as float)), '2024-01-01') +""") +spark.sql("select id, name, embedding, dt from vector_test order by id").show(10, false) +println("VECTOR_SQL_INSERT_SUCCESS") + +// UPDATE goes through castIfNeeded on the VECTOR column; without the reattach +// path it would fail schema compat with MISSING_UNION_BRANCH. +spark.sql(""" + UPDATE vector_test + SET embedding = array(cast(0.9 as float), cast(0.8 as float), cast(0.7 as float)), + name = 'doc1-updated' + WHERE id = 1 +""") +spark.sql("select id, name, embedding from vector_test where id = 1").show(10, false) +println("VECTOR_SQL_UPDATE_SUCCESS") + +// MERGE exercises both MATCHED (UPDATE) and NOT MATCHED (INSERT into new +// partition dt='2024-01-02'). The USING CTE does not carry the hudi_type +// marker, so this specifically exercises the reattach path for MERGE. +spark.sql(""" + MERGE INTO vector_test t + USING ( + SELECT 2L AS id, 'doc2-merged' AS name, + array(cast(0.41 as float), cast(0.51 as float), cast(0.61 as float)) AS embedding, + '2024-01-01' AS dt + UNION ALL + SELECT 3L AS id, 'doc3' AS name, + array(cast(0.7 as float), cast(0.8 as float), cast(0.9 as float)) AS embedding, + '2024-01-02' AS dt + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET t.name = s.name, t.embedding = s.embedding + WHEN NOT MATCHED THEN INSERT (id, name, embedding, dt) VALUES (s.id, s.name, s.embedding, s.dt) +""") +spark.sql("select id, name, embedding, dt from vector_test order by id").show(10, false) +println("VECTOR_SQL_MERGE_SUCCESS") + +spark.sql("DELETE FROM vector_test WHERE id = 3") +spark.sql("select id, name, dt from vector_test order by id").show(10, false) +println("VECTOR_SQL_DELETE_SUCCESS") + +println("VECTOR_SQL_TEST_SUCCESS") diff --git a/docker/hoodie/hadoop/hive_base/entrypoint.sh b/docker/hoodie/hadoop/hive_base/entrypoint.sh index a3df5e6cf4d79..f048114ff45c4 100644 --- a/docker/hoodie/hadoop/hive_base/entrypoint.sh +++ b/docker/hoodie/hadoop/hive_base/entrypoint.sh @@ -131,4 +131,4 @@ do wait_for_it ${i} done -exec $@ +exec "$@" diff --git a/docker/hoodie/hadoop/sparkmaster/master.sh b/docker/hoodie/hadoop/sparkmaster/master.sh index 9409cbc0c12e0..e437f68ddc005 100644 --- a/docker/hoodie/hadoop/sparkmaster/master.sh +++ b/docker/hoodie/hadoop/sparkmaster/master.sh @@ -29,4 +29,4 @@ export SPARK_HOME=/opt/spark ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out cd /opt/spark/bin && /opt/spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \ - --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out + --host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml index 4910031b5a477..f2f2633847ba5 100644 --- a/hudi-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -42,12 +42,13 @@ jersey-container-servlet-core     + com.github.docker-java docker-java - 3.1.2 + 3.3.6 test @@ -391,6 +392,19 @@ trino-jdbc + + org.testcontainers + testcontainers + test + + + + org.testcontainers + junit-jupiter + + ${testcontainers.version} + test + @@ -518,6 +532,38 @@ + + integration-tests + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + ${skipITs} + + **/IT*.java + + + **/integ2/** + + + ${dynamodb-local.endpoint} + ${surefire-log4j.file} + + false + + + + + m1-mac diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ContainerProvider.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ContainerProvider.java new file mode 100644 index 0000000000000..0283f24e6b676 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ContainerProvider.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers; + +import org.testcontainers.containers.ContainerState; + +/** + * Interface for providing access to containers by service name. + * This abstraction allows different implementations (e.g., ComposeContainer, individual containers). + */ +public interface ContainerProvider { + + /** + * Get a container by its service name. + * + * @param serviceName the full compose service instance name as Testcontainers resolves it (e.g. "adhoc-1-1") + * @return the container state + * @throws IllegalStateException if container not found + */ + ContainerState getContainer(String serviceName); +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java new file mode 100644 index 0000000000000..51fdd57b4bfbc --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers; + +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.integ2.testcontainers.service.HiveService; +import org.apache.hudi.integ2.testcontainers.service.SparkService; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Containers; +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Network; +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.SystemProps; +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Timeouts; + +/** + * Base test class for integration tests using Testcontainers with Docker Compose. + * Uses a "Service" pattern, where each external service (Hive, Spark, etc.) is + * represented by a dedicated service object. This provides better separation of + * concerns and a cleaner API for tests. + */ +@Slf4j +@Testcontainers +public abstract class ITTestBaseTestcontainers implements ContainerProvider { + + protected static ComposeContainer environment; + + // Service objects for interacting with different components + protected HiveService hive; + protected SparkService sparkAdhoc1; + + @BeforeAll + public static void setupDockerCompose() { + String composeFilePath = createProcessedComposeFilePath(getDockerComposeFilePath()); + String hudiWorkspace = getHudiWorkspace(); + + log.info("Starting Docker Compose environment"); + log.info("Compose file: {}", composeFilePath); + log.info("HUDI_WS: {}", hudiWorkspace); + + environment = new ComposeContainer(new File(composeFilePath)) + .withEnv("HUDI_WS", hudiWorkspace) + .withExposedService(Containers.SPARK_MASTER, Network.SPARK_MASTER_WEB_UI_PORT, + Wait.forListeningPort().forPorts(Network.SPARK_MASTER_WEB_UI_PORT) + .withStartupTimeout(Timeouts.CONTAINER_STARTUP)) + .withStartupTimeout(Timeouts.CONTAINER_STARTUP); + environment.start(); + + log.info("Docker Compose environment started successfully"); + log.info("All containers verified and running"); + } + + /** + * Tear down the compose stack between test classes. The docker-compose files publish + * host ports directly (zookeeper 2181, spark 7077, …), so leaving one + * stack up would make the next class's `@BeforeAll` collide on those host ports. + * Testcontainers' Ryuk reaper only fires at JVM shutdown, which is too late when + * failsafe reuses a JVM across classes. + */ + @AfterAll + public static void tearDownDockerCompose() { + if (environment != null) { + log.info("Stopping Docker Compose environment"); + try { + environment.stop(); + } finally { + environment = null; + } + } + } + + /** + * Initialize service objects. Should be called in @BeforeEach or constructor of test class. + */ + protected void initializeServices() { + this.hive = new HiveService(this); + this.sparkAdhoc1 = new SparkService(this, Containers.ADHOC_1); + } + + /** + * Skips the test unless the active docker-compose prefix points to a Spark 4.x stack. + * Use for tests that rely on Spark 4.0+ only features (e.g. VARIANT type). + */ + protected static void assumeSpark4Compose() { + Assumptions.assumeTrue(isSpark4Compose(), + "Test requires a Spark 4.x compose stack; active prefix is '" + + System.getProperty(SystemProps.COMPOSE_PREFIX, SystemProps.DEFAULT_COMPOSE_PREFIX) + "'"); + } + + /** + * Non-assumption variant of {@link #assumeSpark4Compose()}: returns {@code true} when the + * active compose prefix points at a Spark 4.x stack. Use in {@code @BeforeAll} seeding + * to conditionally run Spark 4-only fixtures (e.g. VARIANT) without aborting the whole + * test class on a Spark 3.5 run. + */ + protected static boolean isSpark4Compose() { + String composePrefix = System.getProperty(SystemProps.COMPOSE_PREFIX, SystemProps.DEFAULT_COMPOSE_PREFIX); + return composePrefix.contains(SystemProps.SPARK_4_PREFIX_TOKEN); + } + + /** + * Waits for HDFS namenode to be ready by retrying the safemode wait command. + * The namenode may take some time to start after Docker Compose reports containers as running. + */ + protected void waitForHdfs() throws Exception { + for (int i = 1; i <= Timeouts.HDFS_MAX_RETRIES; i++) { + try { + sparkAdhoc1.executeShellCommand("hdfs dfsadmin -safemode wait").expectToSucceed(); + log.info("HDFS namenode is ready"); + return; + } catch (Throwable e) { + if (i == Timeouts.HDFS_MAX_RETRIES) { + throw new RuntimeException( + "HDFS namenode did not become ready after " + Timeouts.HDFS_MAX_RETRIES + " retries", e); + } + log.info("Waiting for HDFS namenode to be ready (attempt {}/{})", i, Timeouts.HDFS_MAX_RETRIES); + Thread.sleep(Timeouts.HDFS_RETRY_INTERVAL.toMillis()); + } + } + } + + /** + * Get a container by service name from the Docker Compose environment. + * Docker Compose appends _1 suffix to service names. + * Implements ContainerProvider interface. + */ + @Override + public ContainerState getContainer(String serviceName) { + try { + return environment.getContainerByServiceName(serviceName) + .orElseThrow(() -> new IllegalStateException("Container not found: " + serviceName)); + } catch (IllegalStateException e) { + log.error("Failed to get container: {}", serviceName, e); + throw e; + } + } + + private static String getHudiWorkspace() { + String projectDir = System.getProperty("user.dir"); + return new File(projectDir, "..").getAbsolutePath(); + } + + private static String getDockerComposeFilePath() { + String projectDir = System.getProperty("user.dir"); + String os = System.getProperty("os.name").toLowerCase(); + String arch = System.getProperty("os.arch").toLowerCase(); + String composePrefix = System.getProperty(SystemProps.COMPOSE_PREFIX, SystemProps.DEFAULT_COMPOSE_PREFIX); + + // Determine which compose file to use based on OS and architecture + boolean isMacArm64 = os.contains("mac") && arch.contains("aarch64"); + String archSuffix = isMacArm64 ? "_arm64" : "_amd64"; + File dockerComposeFile = new File(projectDir, + TestcontainersConfig.Paths.COMPOSE_DIR + composePrefix + archSuffix + ".yml"); + if (!dockerComposeFile.isFile() || !dockerComposeFile.exists()) { + throw new HoodieException(String.format("%s does not exist", dockerComposeFile.getAbsolutePath())); + } + return dockerComposeFile.getAbsolutePath(); + } + + private static String getHadoopEnvFilePath() { + return new File(System.getProperty("user.dir"), "../docker/compose/hadoop.env").getAbsolutePath(); + } + + /** + * Reads the original docker-compose file, removes all 'container_name' directives, and returns a temporary file containing the modified content. Including Testcontainers in the docker-compose file + * will cause ContainerLaunchExceptions to be thrown. + *

    + * Any env_file will normalize/canonicalize to the temporary directory used. This function will make a copy of hadoop.env into the temporary directory to ensure that no error is thrown. + */ + private static String createProcessedComposeFilePath(String composeFile) { + try { + // Read all bytes from the file and convert to a String + byte[] bytes = Files.readAllBytes(Paths.get(composeFile)); + String originalContent = new String(bytes, StandardCharsets.UTF_8); + + // Use a regular expression to find and remove all lines containing 'container_name' + String modifiedContent = originalContent.replaceAll("(?m)^\\s*container_name:.*$", ""); + + // Create a temporary file to hold our modified configuration + Path tempDir = Files.createTempDirectory("hudi-test-compose-"); + + // Delete the temp dir at JVM exit. File.delete() cannot remove a non-empty directory, + // and deleteOnExit runs LIFO, so the files below are registered too: they are deleted + // first, leaving the dir empty for its own delete. + tempDir.toFile().deleteOnExit(); + + // Write the modified content to the temporary file as a byte array. + File tempDockerComposeFile = new File(tempDir.toFile(), "docker-compose.yml"); + Files.write(tempDockerComposeFile.toPath(), modifiedContent.getBytes(StandardCharsets.UTF_8)); + tempDockerComposeFile.deleteOnExit(); + + // tempDir is used as the working directory, docker-compose will look for hadoop.env in the SAME temp directory + // Copy hadoop.env into the SAME temp directory + Path destHadoopEnvPath = tempDir.resolve("hadoop.env"); + Path originalHadoopEnvPath = Paths.get(getHadoopEnvFilePath()).toAbsolutePath().normalize(); + Files.copy(originalHadoopEnvPath, destHadoopEnvPath, StandardCopyOption.REPLACE_EXISTING); + destHadoopEnvPath.toFile().deleteOnExit(); + + // Return the temporary file for Testcontainers to use + return tempDockerComposeFile.toPath().toAbsolutePath().toString(); + } catch (IOException e) { + throw new RuntimeException("Failed to process the docker-compose file", e); + } + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestCustomTypeHiveSync.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestCustomTypeHiveSync.java new file mode 100644 index 0000000000000..3b3cbc5e5e0a0 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestCustomTypeHiveSync.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Paths; + +/** + * End-to-end Hive sync coverage for Hudi's custom logical types (VECTOR, BLOB) and the + * Spark 4.0 VARIANT type, running against a real Hive metastore via the Testcontainers + * harness. + * + * Each type is exercised through both paths: + * - SQL CREATE TABLE (`*-sql.commands`) - table name `_test` + * - DataFrame writer API (`*-df.commands`) - table name `_test_df` + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestCustomTypeHiveSync extends ITTestBaseTestcontainers { + + // BLOB + private static final String BLOB_SQL_TEST_BASE_PATH = "/user/hive/warehouse/blob_test"; + private static final String BLOB_DF_TEST_BASE_PATH = "/user/hive/warehouse/blob_test_df"; + private static final String SPARKSQL_BLOB_TYPE_SQL_COMMANDS = Paths.DEMO_DIR + "/sparksql-blob-type-sql.commands"; + private static final String SPARKSQL_BLOB_TYPE_DF_COMMANDS = Paths.DEMO_DIR + "/sparksql-blob-type-df.commands"; + + // VARIANT + private static final String VARIANT_SQL_TEST_BASE_PATH = "/user/hive/warehouse/variant_test"; + private static final String VARIANT_DF_TEST_BASE_PATH = "/user/hive/warehouse/variant_test_df"; + private static final String SPARKSQL_VARIANT_TYPE_SQL_COMMANDS = Paths.DEMO_DIR + "/sparksql-variant-type-sql.commands"; + private static final String SPARKSQL_VARIANT_TYPE_DF_COMMANDS = Paths.DEMO_DIR + "/sparksql-variant-type-df.commands"; + + // VECTOR + private static final String VECTOR_SQL_TEST_BASE_PATH = "/user/hive/warehouse/vector_test"; + private static final String VECTOR_DF_TEST_BASE_PATH = "/user/hive/warehouse/vector_test_df"; + private static final String SPARKSQL_VECTOR_TYPE_SQL_COMMANDS = Paths.DEMO_DIR + "/sparksql-vector-type-sql.commands"; + private static final String SPARKSQL_VECTOR_TYPE_DF_COMMANDS = Paths.DEMO_DIR + "/sparksql-vector-type-df.commands"; + + // All test paths cleaned in a single `hdfs dfs -rm -R -f` call; -f silently skips + // non-existent paths so tests that don't create every table don't fail teardown. + private static final String CLEANUP_PATHS_JOINED = String.join(" ", + BLOB_SQL_TEST_BASE_PATH, BLOB_DF_TEST_BASE_PATH, + VARIANT_SQL_TEST_BASE_PATH, VARIANT_DF_TEST_BASE_PATH, + VECTOR_SQL_TEST_BASE_PATH, VECTOR_DF_TEST_BASE_PATH); + + /** + * Run idempotent demo setup once per test class instead of once per @Test. The script + * (mkdir -p, cp, copyFromLocal -f, chmod +x) is safe to run a single time and saves + * one shell exec round-trip per test. Requires @TestInstance(PER_CLASS) so this method + * can be a non-static instance method that uses sparkAdhoc1. + */ + @BeforeAll + public void setupOnce() throws Exception { + initializeServices(); + waitForHdfs(); + sparkAdhoc1.executeShellCommand("/bin/bash " + Paths.DEMO_SETUP).expectToSucceed(); + } + + @AfterEach + public void clean() throws Exception { + sparkAdhoc1.executeShellCommand("hdfs dfs -rm -R -f " + CLEANUP_PATHS_JOINED) + .expectToSucceed(); + } + + // ---------- BLOB ---------- + + @Test + public void testBlobTypeWithHiveSyncSQL() throws Exception { + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_SQL_INSERT_SUCCESS") + .assertStdOutContainsLine("BLOB_SQL_UPDATE_SUCCESS") + .assertStdOutContainsLine("BLOB_SQL_MERGE_SUCCESS") + .assertStdOutContainsLine("BLOB_SQL_DELETE_SUCCESS") + .assertStdOutContainsLine("BLOB_SQL_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.blob_test") + .expectToSucceed() + .assertStdOutContains("blob_data"); + + // MERGE added dt=2024-01-02; DELETE removed the row but kept the partition metadata. + hive.execute("SHOW PARTITIONS default.blob_test") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + // Post-DELETE final row count is 2 (id=1 updated, id=2 merged; id=3 deleted). + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.blob_test") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + + // Project a nested struct field through the Hive serde to verify the BLOB + // struct layout round-trips correctly (count() and DESCRIBE do not). + hive.execute( + "SELECT concat('BLOB_TYPE=', blob_data.type) FROM default.blob_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("BLOB_TYPE=OUT_OF_LINE"); + } + + @Test + public void testBlobTypeWithHiveSyncDataFrameAPI() throws Exception { + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_DF_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_DF_INSERT_SUCCESS") + .assertStdOutContainsLine("BLOB_DF_UPSERT_SUCCESS") + .assertStdOutContainsLine("BLOB_DF_DELETE_SUCCESS") + .assertStdOutContainsLine("BLOB_DF_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.blob_test_df") + .expectToSucceed() + .assertStdOutContains("blob_data"); + + hive.execute("SHOW PARTITIONS default.blob_test_df") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.blob_test_df") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + + // DF path seed used the INLINE struct branch and id=1 is never mutated, + // so projecting blob_data.type through the Hive serde should return INLINE. + hive.execute( + "SELECT concat('BLOB_TYPE=', blob_data.type) FROM default.blob_test_df WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("BLOB_TYPE=INLINE"); + } + + // ---------- VARIANT (Spark 4.x only) ---------- + + @Test + public void testVariantTypeWithHiveSyncSQL() throws Exception { + assumeSpark4Compose(); + sparkAdhoc1.executeSQLFile(SPARKSQL_VARIANT_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VARIANT_SQL_INSERT_SUCCESS") + .assertStdOutContainsLine("VARIANT_SQL_UPDATE_SUCCESS") + .assertStdOutContainsLine("VARIANT_SQL_MERGE_SUCCESS") + .assertStdOutContainsLine("VARIANT_SQL_DELETE_SUCCESS") + .assertStdOutContainsLine("VARIANT_SQL_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.variant_test") + .expectToSucceed() + .assertStdOutContains("variant_data"); + + hive.execute("SHOW PARTITIONS default.variant_test") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + // count(*) does not deserialize the variant column, so it is safe even if + // the Hive serde can't project the variant payload. + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.variant_test") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + } + + @Test + public void testVariantTypeWithHiveSyncDataFrameAPI() throws Exception { + assumeSpark4Compose(); + sparkAdhoc1.executeSQLFile(SPARKSQL_VARIANT_TYPE_DF_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VARIANT_DF_INSERT_SUCCESS") + .assertStdOutContainsLine("VARIANT_DF_UPSERT_SUCCESS") + .assertStdOutContainsLine("VARIANT_DF_DELETE_SUCCESS") + .assertStdOutContainsLine("VARIANT_DF_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.variant_test_df") + .expectToSucceed() + .assertStdOutContains("variant_data"); + + hive.execute("SHOW PARTITIONS default.variant_test_df") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.variant_test_df") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + } + + // ---------- VECTOR ---------- + + @Test + public void testVectorTypeWithHiveSyncSQL() throws Exception { + sparkAdhoc1.executeSQLFile(SPARKSQL_VECTOR_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VECTOR_SQL_INSERT_SUCCESS") + .assertStdOutContainsLine("VECTOR_SQL_UPDATE_SUCCESS") + .assertStdOutContainsLine("VECTOR_SQL_MERGE_SUCCESS") + .assertStdOutContainsLine("VECTOR_SQL_DELETE_SUCCESS") + .assertStdOutContainsLine("VECTOR_SQL_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.vector_test") + .expectToSucceed() + .assertStdOutContains("embedding") + .assertStdOutContains("binary"); + + hive.execute("SHOW PARTITIONS default.vector_test") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.vector_test") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + + // VECTOR(3) is stored on disk as fixed_len_byte_array(12) and mapped to + // Hive BINARY (per RFC-99). length() on the binary column confirms the + // bytes round-trip through the Hive serde at the correct width: 3 floats + // x 4 bytes each = 12. Any flip to array would return 3 instead. + hive.execute( + "SELECT concat('VEC_LEN=', length(embedding)) FROM default.vector_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("VEC_LEN=12"); + } + + @Test + public void testVectorTypeWithHiveSyncDataFrameAPI() throws Exception { + sparkAdhoc1.executeSQLFile(SPARKSQL_VECTOR_TYPE_DF_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VECTOR_DF_INSERT_SUCCESS") + .assertStdOutContainsLine("VECTOR_DF_UPSERT_SUCCESS") + .assertStdOutContainsLine("VECTOR_DF_DELETE_SUCCESS") + .assertStdOutContainsLine("VECTOR_DF_TEST_SUCCESS"); + + hive.execute("DESCRIBE default.vector_test_df") + .expectToSucceed() + .assertStdOutContains("embedding") + .assertStdOutContains("binary"); + + hive.execute("SHOW PARTITIONS default.vector_test_df") + .expectToSucceed() + .assertStdOutContains("dt=2024-01-01") + .assertStdOutContains("dt=2024-01-02"); + + hive.execute("SELECT concat('HIVE_COUNT=', count(*)) FROM default.vector_test_df") + .expectToSucceed() + .assertStdOutContains("HIVE_COUNT=2"); + + hive.execute( + "SELECT concat('VEC_LEN=', length(embedding)) FROM default.vector_test_df WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("VEC_LEN=12"); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java new file mode 100644 index 0000000000000..ad403ff6cccd6 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers; + +import java.time.Duration; +import java.util.List; + +/** + * Central configuration for the integ2 Testcontainers harness. Every constant that + * describes "how this harness expects the compose environment to look" belongs here: + * container service names, container-side paths, network endpoints, timeouts, and + * the keys / defaults of system properties that tune the harness. + * + *

    Per-test fixture data (table paths, per-test {@code .commands} scripts) does + * not belong here, those stay in the relevant test class, though they should build + * their common prefix from {@link Paths#DEMO_DIR} / {@link Paths#WS_ROOT} rather + * than inlining the path literal. + */ +public final class TestcontainersConfig { + + private TestcontainersConfig() { + } + + /** Docker Compose service names. Must match the compose YAML verbatim. */ + public static final class Containers { + public static final String HIVESERVER = "hiveserver"; + public static final String SPARK_MASTER = "sparkmaster"; + // Testcontainers appends the replica index, so the adhoc services resolve as "-1". + public static final String ADHOC_1 = "adhoc-1-1"; + + private Containers() { + } + } + + /** Paths inside the containers (absolute) and the host-side compose directory. */ + public static final class Paths { + public static final String WS_ROOT = "/var/hoodie/ws"; + public static final String DEMO_DIR = WS_ROOT + "/docker/demo"; + public static final String DEMO_SETUP = DEMO_DIR + "/setup_demo_container.sh"; + public static final String HIVE_TARGET = WS_ROOT + "/docker/hoodie/hadoop/hive_base/target"; + public static final String SPARK_BUNDLE = HIVE_TARGET + "/hoodie-spark-bundle.jar"; + public static final String HADOOP_CONF_DIR = "/etc/hadoop"; + /** Host-side, relative to the hudi-integ-test module working directory. */ + public static final String COMPOSE_DIR = "../docker/compose/"; + + private Paths() { + } + } + + /** Network endpoints the harness exposes to tests. */ + public static final class Network { + public static final int SPARK_MASTER_WEB_UI_PORT = 8080; + + private Network() { + } + } + + /** Waits and timeouts used by the harness. */ + public static final class Timeouts { + public static final Duration CONTAINER_STARTUP = Duration.ofMinutes(5); + public static final int HDFS_MAX_RETRIES = 12; + public static final Duration HDFS_RETRY_INTERVAL = Duration.ofSeconds(10); + + private Timeouts() { + } + } + + /** System-property keys and their defaults (read via {@link System#getProperty}). */ + public static final class SystemProps { + public static final String COMPOSE_PREFIX = "spark.docker.compose.prefix"; + public static final String DEFAULT_COMPOSE_PREFIX = "docker-compose_hadoop340_hive2310_spark402"; + /** Substring present in compose prefixes that run Spark 4.x (e.g. "...spark402"). */ + public static final String SPARK_4_PREFIX_TOKEN = "spark4"; + + /** + * Flip to {@code true} (e.g. {@code -Dhudi.integ.hive.verbose=true}) to route + * Hive logs to the console so exception stack traces show up in test output. + */ + public static final String HIVE_VERBOSE = "hudi.integ.hive.verbose"; + + /** + * Hiveconf entries applied only when {@link #HIVE_VERBOSE} is enabled. + * See {@code HiveService#execute} for the per-flag rationale. + */ + public static final List VERBOSE_HIVECONFS = + List.of("hive.root.logger=INFO,console", + "hive.exec.mode.local.auto=false", + "hive.log.explain.output=true", + "hive.server2.logging.operation.verbose=true"); + + private SystemProps() { + } + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandExecutor.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandExecutor.java new file mode 100644 index 0000000000000..f2f01a006c1ab --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandExecutor.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.command; + +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +/** + * A utility class for executing commands within a given Testcontainer. + * This class is stateless regarding the command being executed but is tied to a specific container. + */ +@Slf4j +public final class CommandExecutor { + + private final ContainerState container; + + public CommandExecutor(ContainerState container) { + this.container = container; + } + + /** + * Execute a command in the executor's container. + */ + public CommandResult executeCommand(String... command) throws Exception { + String containerIdentifier = getContainerIdentifier(container); + String commandStr = String.join(" ", command); + + log.info("==> [{}] Executing: {}", containerIdentifier, commandStr); + + long startTime = System.currentTimeMillis(); + Container.ExecResult result = container.execInContainer(command); + long duration = System.currentTimeMillis() - startTime; + + int exitCode = result.getExitCode(); + log.info("<== [{}] Exit code: {} ({}ms)", containerIdentifier, exitCode, duration); + + if (exitCode != 0) { + log.error("STDOUT:\n{}", result.getStdout()); + log.error("STDERR:\n{}", result.getStderr()); + } else if (log.isDebugEnabled()) { + log.debug("STDOUT:\n{}", result.getStdout()); + if (!result.getStderr().isEmpty()) { + log.debug("STDERR:\n{}", result.getStderr()); + } + } + + return new CommandResult(result); + } + + /** + * Execute a shell command string. + */ + public CommandResult executeCommandString(String cmd) throws Exception { + String[] cmdArray = {"/bin/bash", "-c", cmd}; + return executeCommand(cmdArray); + } + + /** + * Get a readable identifier for the container. + */ + private String getContainerIdentifier(ContainerState container) { + String containerName = container.getContainerInfo().getName(); + if (containerName != null && !containerName.isEmpty()) { + // Container names start with '/', so remove it + String cleanName = containerName.startsWith("/") ? containerName.substring(1) : containerName; + return cleanName + ":" + container.getContainerId().substring(0, 8); + } + return container.getContainerId().substring(0, 12); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandResult.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandResult.java new file mode 100644 index 0000000000000..8e65e9aebcbaf --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandResult.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.command; + +import lombok.AllArgsConstructor; +import org.testcontainers.containers.Container; + +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * A dedicated class to hold the result of a command execution and provide + * fluent assertion methods for cleaner tests. + */ +@AllArgsConstructor +public class CommandResult { + + // Spark 4.0 (Scala 2.13) spark-shell runs under a dumb terminal when stdin + // is piped from a file, which flushes the next `scala> ` prompt onto the + // same line as preceding async println output (e.g. `scala> MARKER`). + // Stripping one-or-more leading `scala>\s+` prefixes normalizes those lines + // back to the bare sentinel while leaving Spark 3.5 output (sentinel already + // on its own line) unchanged. + private static final Pattern REPL_PROMPT_PREFIX = Pattern.compile("^(scala>\\s+)+"); + + private final String stdout; + private final String stderr; + private final int exitCode; + + public CommandResult(Container.ExecResult execResult) { + this.stdout = execResult.getStdout(); + this.stderr = execResult.getStderr(); + this.exitCode = execResult.getExitCode(); + } + + /** + * Asserts that the command's exit code is 0 (success). + * + * @return The same {@link CommandResult} instance for chaining assertions. + * @throws AssertionError if the exit code is not 0. + */ + public CommandResult expectToSucceed() { + assertEquals(0, exitCode, + String.format("Command failed with exit code %d. Stderr: %s", exitCode, stderr)); + return this; + } + + /** + * Asserts that the standard output contains a specific substring exactly once. + * + * @param expectedSubstring The substring to search for. + * @return The same {@link CommandResult} instance for chaining assertions. + */ + public CommandResult assertStdOutContains(String expectedSubstring) { + return assertStdOutContains(expectedSubstring, 1); + } + + /** + * Asserts that the standard output contains a specific substring an exact number of times. + * + * @param expectedSubstring The substring to search for. + * @param times The exact number of times the substring is expected to appear. + * @return The same {@link CommandResult} instance for chaining assertions. + */ + public CommandResult assertStdOutContains(String expectedSubstring, int times) { + // Normalize whitespace for more robust matching + String stdOutSingleSpaced = stdout.replaceAll("[\\s]+", " ").trim(); + String expectedOutput = expectedSubstring.replaceAll("[\\s]+", " ").trim(); + + int lastIndex = 0; + int count = 0; + while (lastIndex != -1) { + lastIndex = stdOutSingleSpaced.indexOf(expectedOutput, lastIndex); + if (lastIndex != -1) { + count++; + lastIndex += expectedOutput.length(); + } + } + assertEquals(times, count, + String.format("Expected to find substring '%s' %d times, but found %d. Full stdout: %s", + expectedOutput, times, count, stdout)); + return this; + } + + /** + * Asserts that the standard output contains a line exactly equal to the expected value + * (after trimming and after stripping any leading {@code scala> } REPL prompt prefix), + * appearing exactly once. Use this for REPL sentinel markers emitted via + * {@code println(...)}. Handles both Scala 2.12's spark-shell echoing the input line + * back (Spark 3.5) and Scala 2.13's dumb-terminal mode prefixing sentinel output with + * {@code scala> } on the same line (Spark 4.0). + * + * @param expectedLine The exact line content to match (after trim and prompt strip). + * @return The same {@link CommandResult} instance for chaining assertions. + */ + public CommandResult assertStdOutContainsLine(String expectedLine) { + return assertStdOutContainsLine(expectedLine, 1); + } + + /** + * Asserts that the standard output contains a line exactly equal to the expected value + * (after trimming and after stripping any leading {@code scala> } REPL prompt prefix), + * appearing an exact number of times. + * + * @param expectedLine The exact line content to match (after trim and prompt strip). + * @param times The exact number of matching lines expected. + * @return The same {@link CommandResult} instance for chaining assertions. + */ + public CommandResult assertStdOutContainsLine(String expectedLine, int times) { + String expected = expectedLine.trim(); + long count = stdout.lines() + .map(line -> REPL_PROMPT_PREFIX.matcher(line).replaceFirst("").trim()) + .filter(expected::equals) + .count(); + assertEquals(times, count, + String.format("Expected line '%s' %d times, but found %d. Full stdout: %s", + expected, times, count, stdout)); + return this; + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/HiveService.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/HiveService.java new file mode 100644 index 0000000000000..5c5151809a7a2 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/HiveService.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.service; + +import org.apache.hudi.integ2.testcontainers.ContainerProvider; +import org.apache.hudi.integ2.testcontainers.TestcontainersConfig; +import org.apache.hudi.integ2.testcontainers.command.CommandExecutor; +import org.apache.hudi.integ2.testcontainers.command.CommandResult; + +import java.util.ArrayList; +import java.util.List; + +/** + * A service wrapper for the Hive container. + * This class is responsible for all interactions with the Hive service, + * including executing commands and managing files. + */ +public class HiveService { + + private final CommandExecutor executor; + private final boolean verbose; + + public HiveService(ContainerProvider provider) { + this(provider, Boolean.getBoolean(TestcontainersConfig.SystemProps.HIVE_VERBOSE)); + } + + /** + * Visible-for-tests overload so callers can toggle verbose mode without + * setting the system property at JVM start time. + */ + public HiveService(ContainerProvider provider, boolean verbose) { + this.executor = new CommandExecutor(provider.getContainer(TestcontainersConfig.Containers.HIVESERVER)); + this.verbose = verbose; + } + + /** + * Execute a Hive command and return the result. + */ + public CommandResult execute(String hiveCommand) throws Exception { + List hiveCmd = new ArrayList<>(); + hiveCmd.add("hive"); + hiveCmd.add("--hiveconf"); + hiveCmd.add("hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat"); + hiveCmd.add("--hiveconf"); + hiveCmd.add("hive.stats.autogather=false"); + if (verbose) { + for (String kv : TestcontainersConfig.SystemProps.VERBOSE_HIVECONFS) { + hiveCmd.add("--hiveconf"); + hiveCmd.add(kv); + } + } + hiveCmd.add("-e"); + hiveCmd.add(hiveCommand); + return executor.executeCommand(hiveCmd.toArray(new String[0])); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/SparkService.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/SparkService.java new file mode 100644 index 0000000000000..1c105402d6445 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/SparkService.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.service; + +import org.apache.hudi.integ2.testcontainers.ContainerProvider; +import org.apache.hudi.integ2.testcontainers.TestcontainersConfig; +import org.apache.hudi.integ2.testcontainers.command.CommandExecutor; +import org.apache.hudi.integ2.testcontainers.command.CommandResult; + +/** + * A service wrapper for the Spark container. + * This class is responsible for all interactions with the Spark service, + * including executing spark-shell commands. + */ +public class SparkService { + + private final CommandExecutor executor; + + public SparkService(ContainerProvider provider, String containerName) { + this.executor = new CommandExecutor(provider.getContainer(containerName)); + } + + /** + * Execute a Spark SQL command file. + */ + public CommandResult executeSQLFile(String commandFile) throws Exception { + String sparkShellCmd = new StringBuilder() + .append("spark-shell --jars ").append(TestcontainersConfig.Paths.SPARK_BUNDLE) + .append(" --master local[2] --driver-class-path ").append(TestcontainersConfig.Paths.HADOOP_CONF_DIR) + .append(" --conf spark.serializer=org.apache.spark.serializer.KryoSerializer") + .append(" --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog") + .append(" --conf spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension") + .append(" --deploy-mode client --driver-memory 1G --executor-memory 1G --num-executors 1") + // Pipe via stdin instead of `-i`. With `-i`, Spark 4's REPL boots in + // dumb-terminal mode (`WARN jline: Unable to create a system terminal, + // creating a dumb terminal`) and silently runs the script — neither + // input echo nor exception traces reach stdout, so success markers and + // failures alike are invisible to assertStdOutContains. + .append(" < ").append(commandFile) + .toString(); + + return executor.executeCommandString(sparkShellCmd); + } + + /** + * Executes an arbitrary shell command inside the service's container. + * This is useful for hdfs commands, hudi-cli, etc. + * + * @param command The shell command to execute. + * @return The result of the command execution. + */ + public CommandResult executeShellCommand(String command) throws Exception { + return executor.executeCommandString(command); + } +} From c0cde80f8f43c3802b526be14470ffb98df5f93c Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 8 Jul 2026 05:05:02 -0700 Subject: [PATCH 144/255] fix(spark): demote spurious per-write INFO logs to debug (#19170) (cherry picked from commit 1165eb536b019e2f2ad0df2778fd02eae6eeff8f) --- .../scala/org/apache/hudi/HoodieSparkSqlWriter.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index 07058b885b620..9137433d3178b 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -991,7 +991,7 @@ class HoodieSparkSqlWriterInternal { extraPreCommitFn: Option[BiConsumer[HoodieTableMetaClient, HoodieCommitMetadata]] ): (Boolean, HOption[java.lang.String], HOption[java.lang.String]) = { val hasErrors = new AtomicBoolean(false) - log.info("Proceeding to commit the write.") + log.debug("Proceeding to commit the write.") // get extra metadata from props // 1. properties starting with commit metadata key prefix // 2. properties related to checkpoint in spark streaming @@ -1020,7 +1020,7 @@ class HoodieSparkSqlWriterInternal { common.util.Option.empty() } - log.info(s"Compaction Scheduled is $compactionInstant") + log.debug(s"Compaction Scheduled is $compactionInstant") val asyncClusteringEnabled = isAsyncClusteringEnabled(client, parameters) val clusteringInstant: common.util.Option[java.lang.String] = @@ -1030,12 +1030,12 @@ class HoodieSparkSqlWriterInternal { common.util.Option.empty() } - log.info(s"Clustering Scheduled is $clusteringInstant") + log.debug(s"Clustering Scheduled is $clusteringInstant") val metaSyncSuccess = metaSync(spark, HoodieWriterUtils.convertMapToHoodieConfig(parameters), tableInstantInfo.basePath, schema) - log.info(s"Is Async Compaction Enabled ? $asyncCompactionEnabled") + log.debug(s"Is Async Compaction Enabled ? $asyncCompactionEnabled") (commitSuccess && metaSyncSuccess, compactionInstant, clusteringInstant) } else { (false, common.util.Option.empty(), common.util.Option.empty()) @@ -1045,7 +1045,7 @@ class HoodieSparkSqlWriterInternal { private def isAsyncCompactionEnabled(client: SparkRDDWriteClient[_], tableConfig: HoodieTableConfig, parameters: Map[String, String], configuration: Configuration): Boolean = { - log.info(s"Config.inlineCompactionEnabled ? ${client.getConfig.inlineCompactionEnabled}") + log.debug(s"Config.inlineCompactionEnabled ? ${client.getConfig.inlineCompactionEnabled}") (asyncCompactionTriggerFnDefined && !client.getConfig.inlineCompactionEnabled && parameters.get(ASYNC_COMPACT_ENABLE.key).exists(r => r.toBoolean) && tableConfig.getTableType == MERGE_ON_READ) @@ -1053,7 +1053,7 @@ class HoodieSparkSqlWriterInternal { private def isAsyncClusteringEnabled(client: SparkRDDWriteClient[_], parameters: Map[String, String]): Boolean = { - log.info(s"Config.asyncClusteringEnabled ? ${client.getConfig.isAsyncClusteringEnabled}") + log.debug(s"Config.asyncClusteringEnabled ? ${client.getConfig.isAsyncClusteringEnabled}") (asyncClusteringTriggerFnDefined && !client.getConfig.inlineClusteringEnabled && client.getConfig.isAsyncClusteringEnabled) } From 53c2bc71394ade04dbf49855859d8d268f35e287 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 8 Jul 2026 05:30:15 -0700 Subject: [PATCH 145/255] test(client): cover low-coverage small classes across hudi-client (#19224) * test(client): cover low-coverage small classes across hudi-client * test(client): address review nits on tail-sweep coverage tests - Assert content equality instead of reference identity for the candidate-key and data-write-stat accessors, so the tests no longer pin the no-defensive-copy implementation detail. - Document that isKeyInRange fails fast with an NPE when bounds are unset. - Use assertNotEquals for the BloomIndexFileInfo inequality check, matching the sibling equality tests. --------- Co-authored-by: voon (cherry picked from commit 21a931572dfd07d8ba18c4cd64cd721aaf5e3df1) --- .../hudi/client/TestClientStatsPojos.java | 94 +++++++++++++++++++ ...TestBootstrapPartitionPathTranslators.java | 46 +++++++++ .../transaction/lock/TestLockResultEnums.java | 62 ++++++++++++ .../lock/audit/TestAuditOperationState.java | 54 +++++++++++ .../hudi/exception/TestClientExceptions.java | 71 ++++++++++++++ .../bulkinsert/TestBulkInsertSortMode.java | 52 ++++++++++ .../index/bloom/TestBloomIndexFileInfo.java | 79 ++++++++++++++++ .../action/commit/TestBucketTypeAndInfo.java | 72 ++++++++++++++ .../hudi/util/TestOperationConverter.java | 46 +++++++++ ...aBulkInsertInternalPartitionerFactory.java | 53 +++++++++++ .../TestHoodieBloomFilterProbingResult.java | 64 +++++++++++++ .../commit/TestSparkBucketInfoGetter.java | 68 ++++++++++++++ 12 files changed, 761 insertions(+) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestClientStatsPojos.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/bootstrap/translator/TestBootstrapPartitionPathTranslators.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/audit/TestAuditOperationState.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/exception/TestClientExceptions.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertSortMode.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bloom/TestBloomIndexFileInfo.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/commit/TestBucketTypeAndInfo.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitionerFactory.java create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomFilterProbingResult.java create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkBucketInfoGetter.java diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestClientStatsPojos.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestClientStatsPojos.java new file mode 100644 index 0000000000000..f971c0369568b --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestClientStatsPojos.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.common.model.HoodieWriteStat; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the plain client-side stat holders {@link SecondaryIndexStats} and {@link TableWriteStats}. + */ +public class TestClientStatsPojos { + + @Test + void secondaryIndexStatsExposesConstructorValues() { + SecondaryIndexStats deleted = new SecondaryIndexStats("rk1", "sk1", true); + assertEquals("rk1", deleted.getRecordKey()); + assertEquals("sk1", deleted.getSecondaryKeyValue()); + assertTrue(deleted.isDeleted()); + + SecondaryIndexStats live = new SecondaryIndexStats("rk2", "sk2", false); + assertEquals("rk2", live.getRecordKey()); + assertEquals("sk2", live.getSecondaryKeyValue()); + assertFalse(live.isDeleted()); + } + + @Test + void secondaryIndexStatsSetterMutatesRecordKey() { + SecondaryIndexStats stats = new SecondaryIndexStats("rk1", "sk1", false); + stats.setRecordKey("rk2"); + stats.setSecondaryKeyValue("sk2"); + assertEquals("rk2", stats.getRecordKey()); + assertEquals("sk2", stats.getSecondaryKeyValue()); + } + + @Test + void secondaryIndexStatsEqualityUsesAllFields() { + SecondaryIndexStats a = new SecondaryIndexStats("rk", "sk", false); + SecondaryIndexStats same = new SecondaryIndexStats("rk", "sk", false); + SecondaryIndexStats deleteDiffers = new SecondaryIndexStats("rk", "sk", true); + assertEquals(a, same); + assertEquals(a.hashCode(), same.hashCode()); + assertNotEquals(a, deleteDiffers); + } + + @Test + void tableWriteStatsSingleArgDefaultsMetadataToEmpty() { + List dataStats = Collections.singletonList(new HoodieWriteStat()); + TableWriteStats stats = new TableWriteStats(dataStats); + assertEquals(dataStats, stats.getDataTableWriteStats()); + assertTrue(stats.getMetadataTableWriteStats().isEmpty()); + assertFalse(stats.isEmptyDataTableWriteStats()); + } + + @Test + void tableWriteStatsReportsEmptyDataStats() { + TableWriteStats empty = new TableWriteStats(Collections.emptyList()); + assertTrue(empty.isEmptyDataTableWriteStats()); + } + + @Test + void tableWriteStatsRetainsBothLists() { + List dataStats = Arrays.asList(new HoodieWriteStat(), new HoodieWriteStat()); + List metadataStats = Collections.singletonList(new HoodieWriteStat()); + TableWriteStats stats = new TableWriteStats(dataStats, metadataStats); + assertEquals(2, stats.getDataTableWriteStats().size()); + assertEquals(1, stats.getMetadataTableWriteStats().size()); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/bootstrap/translator/TestBootstrapPartitionPathTranslators.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/bootstrap/translator/TestBootstrapPartitionPathTranslators.java new file mode 100644 index 0000000000000..9598b581aa686 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/bootstrap/translator/TestBootstrapPartitionPathTranslators.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.bootstrap.translator; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests the two built-in bootstrap partition-path translators. + */ +public class TestBootstrapPartitionPathTranslators { + + @Test + void identityTranslatorReturnsInputUnchanged() { + IdentityBootstrapPartitionPathTranslator translator = new IdentityBootstrapPartitionPathTranslator(); + assertEquals("2024/01/01", translator.getBootstrapTranslatedPath("2024/01/01")); + // Even already-encoded input is passed through verbatim. + assertEquals("2024%2F01", translator.getBootstrapTranslatedPath("2024%2F01")); + } + + @Test + void decodedTranslatorUriDecodesEscapedPath() { + DecodedBootstrapPartitionPathTranslator translator = new DecodedBootstrapPartitionPathTranslator(); + // %2F decodes to a slash. + assertEquals("2024/01", translator.getBootstrapTranslatedPath("2024%2F01")); + // Paths without escape sequences are unaffected. + assertEquals("region=us", translator.getBootstrapTranslatedPath("region=us")); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java new file mode 100644 index 0000000000000..51e944c214a95 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.transaction.lock; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Tests the small result enums used by the storage-based lock provider. + */ +public class TestLockResultEnums { + + @Test + void lockGetResultCodesAreStable() { + assertEquals(0, LockGetResult.NOT_EXISTS.getCode()); + assertEquals(1, LockGetResult.SUCCESS.getCode()); + assertEquals(2, LockGetResult.UNKNOWN_ERROR.getCode()); + // Codes must be unique so callers can map them one-to-one. + assertEquals(3, LockGetResult.values().length); + } + + @Test + void lockGetResultValueOfRoundTrips() { + for (LockGetResult result : LockGetResult.values()) { + assertSame(result, LockGetResult.valueOf(result.name())); + } + } + + @Test + void lockUpsertResultCodesAreStable() { + assertEquals(0, LockUpsertResult.SUCCESS.getCode()); + assertEquals(1, LockUpsertResult.ACQUIRED_BY_OTHERS.getCode()); + assertEquals(2, LockUpsertResult.UNKNOWN_ERROR.getCode()); + assertEquals(3, LockUpsertResult.THROTTLED.getCode()); + assertEquals(4, LockUpsertResult.values().length); + } + + @Test + void lockUpsertResultValueOfRoundTrips() { + for (LockUpsertResult result : LockUpsertResult.values()) { + assertSame(result, LockUpsertResult.valueOf(result.name())); + } + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/audit/TestAuditOperationState.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/audit/TestAuditOperationState.java new file mode 100644 index 0000000000000..fd9cdf7f44e1f --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/audit/TestAuditOperationState.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.transaction.lock.audit; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the {@link AuditOperationState} lifecycle enum. + */ +public class TestAuditOperationState { + + @Test + void declaresExpectedStatesInOrder() { + assertArrayEquals( + new AuditOperationState[] { + AuditOperationState.START, + AuditOperationState.RENEW, + AuditOperationState.END + }, + AuditOperationState.values()); + } + + @Test + void valueOfResolvesEachState() { + for (AuditOperationState state : AuditOperationState.values()) { + assertSame(state, AuditOperationState.valueOf(state.name())); + } + } + + @Test + void valueOfRejectsUnknownState() { + assertThrows(IllegalArgumentException.class, () -> AuditOperationState.valueOf("PAUSE")); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/exception/TestClientExceptions.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/exception/TestClientExceptions.java new file mode 100644 index 0000000000000..8501aa3774b63 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/exception/TestClientExceptions.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.exception; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the thin client-side exception types that wrap {@link HoodieException}. + */ +public class TestClientExceptions { + + @Test + void keyGeneratorExceptionPreservesMessageAndCause() { + Throwable cause = new IllegalStateException("boom"); + HoodieKeyGeneratorException withCause = new HoodieKeyGeneratorException("bad key", cause); + assertEquals("bad key", withCause.getMessage()); + assertSame(cause, withCause.getCause()); + assertTrue(withCause instanceof HoodieException); + + HoodieKeyGeneratorException messageOnly = new HoodieKeyGeneratorException("no cause"); + assertEquals("no cause", messageOnly.getMessage()); + assertNull(messageOnly.getCause()); + } + + @Test + void compactionExceptionPreservesMessageAndCause() { + Throwable cause = new RuntimeException("io"); + HoodieCompactionException withCause = new HoodieCompactionException("compaction failed", cause); + assertEquals("compaction failed", withCause.getMessage()); + assertSame(cause, withCause.getCause()); + assertTrue(withCause instanceof HoodieException); + + HoodieCompactionException messageOnly = new HoodieCompactionException("compaction failed"); + assertEquals("compaction failed", messageOnly.getMessage()); + assertNull(messageOnly.getCause()); + } + + @Test + void rollbackExceptionPreservesMessageAndCause() { + Throwable cause = new RuntimeException("io"); + HoodieRollbackException withCause = new HoodieRollbackException("rollback failed", cause); + assertEquals("rollback failed", withCause.getMessage()); + assertSame(cause, withCause.getCause()); + assertTrue(withCause instanceof HoodieException); + + HoodieRollbackException messageOnly = new HoodieRollbackException("rollback failed"); + assertEquals("rollback failed", messageOnly.getMessage()); + assertNull(messageOnly.getCause()); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertSortMode.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertSortMode.java new file mode 100644 index 0000000000000..8ff7ef0679d58 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertSortMode.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.execution.bulkinsert; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link BulkInsertSortMode}. The constant names double as config values, so the + * full set is pinned here to catch accidental renames or removals. + */ +public class TestBulkInsertSortMode { + + @Test + void exposesExpectedModes() { + assertEquals( + "NONE,GLOBAL_SORT,PARTITION_SORT,PARTITION_PATH_REPARTITION,PARTITION_PATH_REPARTITION_AND_SORT", + Arrays.stream(BulkInsertSortMode.values()) + .map(Enum::name) + .collect(Collectors.joining(","))); + } + + @Test + void valueOfResolvesEachModeCaseSensitively() { + for (BulkInsertSortMode mode : BulkInsertSortMode.values()) { + assertSame(mode, BulkInsertSortMode.valueOf(mode.name())); + } + assertThrows(IllegalArgumentException.class, () -> BulkInsertSortMode.valueOf("global_sort")); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bloom/TestBloomIndexFileInfo.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bloom/TestBloomIndexFileInfo.java new file mode 100644 index 0000000000000..9f235d1d23370 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bloom/TestBloomIndexFileInfo.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.index.bloom; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link BloomIndexFileInfo} accessors and key-range checks. + */ +public class TestBloomIndexFileInfo { + + @Test + void fileIdOnlyConstructorLeavesRangeUnset() { + BloomIndexFileInfo info = new BloomIndexFileInfo("f1"); + assertEquals("f1", info.getFileId()); + assertNull(info.getMinRecordKey()); + assertNull(info.getMaxRecordKey()); + assertFalse(info.hasKeyRanges()); + } + + @Test + void fullConstructorPopulatesRange() { + BloomIndexFileInfo info = new BloomIndexFileInfo("f1", "key05", "key20"); + assertEquals("key05", info.getMinRecordKey()); + assertEquals("key20", info.getMaxRecordKey()); + assertTrue(info.hasKeyRanges()); + } + + @Test + void isKeyInRangeIsInclusiveOfBounds() { + BloomIndexFileInfo info = new BloomIndexFileInfo("f1", "key05", "key20"); + assertTrue(info.isKeyInRange("key05")); + assertTrue(info.isKeyInRange("key10")); + assertTrue(info.isKeyInRange("key20")); + assertFalse(info.isKeyInRange("key04")); + assertFalse(info.isKeyInRange("key21")); + } + + @Test + void isKeyInRangeRequiresBounds() { + BloomIndexFileInfo noRange = new BloomIndexFileInfo("f1"); + // isKeyInRange does not guard on hasKeyRanges(); with no bounds set it currently + // fails fast with an NPE from Objects.requireNonNull rather than returning false. + assertThrows(NullPointerException.class, () -> noRange.isKeyInRange("key10")); + } + + @Test + void valueSemanticsForEqualsAndHashCode() { + BloomIndexFileInfo a = new BloomIndexFileInfo("f1", "min", "max"); + BloomIndexFileInfo same = new BloomIndexFileInfo("f1", "min", "max"); + BloomIndexFileInfo different = new BloomIndexFileInfo("f2", "min", "max"); + assertEquals(a, same); + assertEquals(a.hashCode(), same.hashCode()); + assertNotEquals(a, different); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/commit/TestBucketTypeAndInfo.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/commit/TestBucketTypeAndInfo.java new file mode 100644 index 0000000000000..537250c00e61c --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/commit/TestBucketTypeAndInfo.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.commit; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link BucketType} and the {@link BucketInfo} value holder that carries it. + */ +public class TestBucketTypeAndInfo { + + @Test + void bucketTypeHasUpdateAndInsert() { + assertEquals(2, BucketType.values().length); + assertSame(BucketType.UPDATE, BucketType.valueOf("UPDATE")); + assertSame(BucketType.INSERT, BucketType.valueOf("INSERT")); + } + + @Test + void bucketInfoGettersReturnConstructorArgs() { + BucketInfo info = new BucketInfo(BucketType.INSERT, "fileId-1", "2024/01/01"); + assertSame(BucketType.INSERT, info.getBucketType()); + assertEquals("fileId-1", info.getFileIdPrefix()); + assertEquals("2024/01/01", info.getPartitionPath()); + } + + @Test + void bucketInfoEqualsAndHashCodeUseAllFields() { + BucketInfo a = new BucketInfo(BucketType.UPDATE, "f1", "p1"); + BucketInfo same = new BucketInfo(BucketType.UPDATE, "f1", "p1"); + BucketInfo differentType = new BucketInfo(BucketType.INSERT, "f1", "p1"); + BucketInfo differentFile = new BucketInfo(BucketType.UPDATE, "f2", "p1"); + BucketInfo differentPartition = new BucketInfo(BucketType.UPDATE, "f1", "p2"); + + assertEquals(a, same); + assertEquals(a.hashCode(), same.hashCode()); + assertNotEquals(a, differentType); + assertNotEquals(a, differentFile); + assertNotEquals(a, differentPartition); + assertNotEquals(a, null); + assertNotEquals(a, "not a bucket info"); + } + + @Test + void bucketInfoToStringContainsFields() { + String rendered = new BucketInfo(BucketType.INSERT, "fileId-9", "part-9").toString(); + assertTrue(rendered.contains("INSERT")); + assertTrue(rendered.contains("fileId-9")); + assertTrue(rendered.contains("part-9")); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java new file mode 100644 index 0000000000000..e8cee32e7f4de --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.util; + +import org.apache.hudi.common.model.WriteOperationType; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link OperationConverter}, the jcommander string-to-enum converter. + */ +public class TestOperationConverter { + + private final OperationConverter converter = new OperationConverter(); + + @Test + void convertsEnumConstantNames() { + assertSame(WriteOperationType.INSERT, converter.convert("INSERT")); + assertSame(WriteOperationType.UPSERT, converter.convert("UPSERT")); + assertSame(WriteOperationType.BULK_INSERT, converter.convert("BULK_INSERT")); + } + + @Test + void rejectsUnknownOperation() { + assertThrows(IllegalArgumentException.class, () -> converter.convert("not_an_operation")); + } +} diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitionerFactory.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitionerFactory.java new file mode 100644 index 0000000000000..708e25d29cba6 --- /dev/null +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitionerFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.execution.bulkinsert; + +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.table.BulkInsertPartitioner; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the sort-mode based selection in {@link JavaBulkInsertInternalPartitionerFactory}. + */ +public class TestJavaBulkInsertInternalPartitionerFactory { + + @Test + void noneModeReturnsNonSortPartitioner() { + BulkInsertPartitioner partitioner = JavaBulkInsertInternalPartitionerFactory.get(BulkInsertSortMode.NONE); + assertInstanceOf(JavaNonSortPartitioner.class, partitioner); + } + + @Test + void globalSortModeReturnsGlobalSortPartitioner() { + BulkInsertPartitioner partitioner = JavaBulkInsertInternalPartitionerFactory.get(BulkInsertSortMode.GLOBAL_SORT); + assertInstanceOf(JavaGlobalSortPartitioner.class, partitioner); + } + + @Test + void unsupportedModeThrows() { + assertThrows(HoodieException.class, + () -> JavaBulkInsertInternalPartitionerFactory.get(BulkInsertSortMode.PARTITION_SORT)); + assertThrows(HoodieException.class, + () -> JavaBulkInsertInternalPartitionerFactory.get(BulkInsertSortMode.PARTITION_PATH_REPARTITION)); + } +} diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomFilterProbingResult.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomFilterProbingResult.java new file mode 100644 index 0000000000000..ea4f663d2d494 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomFilterProbingResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.index.bloom; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link HoodieBloomFilterProbingResult} value holder. + */ +public class TestHoodieBloomFilterProbingResult { + + @Test + void exposesCandidateKeys() { + Set keys = new HashSet<>(); + keys.add("k1"); + keys.add("k2"); + HoodieBloomFilterProbingResult result = new HoodieBloomFilterProbingResult(keys); + assertEquals(keys, result.getCandidateKeys()); + } + + @Test + void supportsEmptyCandidateSet() { + HoodieBloomFilterProbingResult result = + new HoodieBloomFilterProbingResult(Collections.emptySet()); + assertTrue(result.getCandidateKeys().isEmpty()); + } + + @Test + void valueSemanticsForEqualsAndHashCode() { + HoodieBloomFilterProbingResult a = + new HoodieBloomFilterProbingResult(new HashSet<>(Collections.singletonList("k"))); + HoodieBloomFilterProbingResult same = + new HoodieBloomFilterProbingResult(new HashSet<>(Collections.singletonList("k"))); + HoodieBloomFilterProbingResult different = + new HoodieBloomFilterProbingResult(new HashSet<>(Collections.singletonList("other"))); + assertEquals(a, same); + assertEquals(a.hashCode(), same.hashCode()); + assertNotEquals(a, different); + } +} diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkBucketInfoGetter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkBucketInfoGetter.java new file mode 100644 index 0000000000000..670db2c77c653 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkBucketInfoGetter.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.commit; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the list- and map-backed {@link SparkBucketInfoGetter} implementations. + */ +public class TestSparkBucketInfoGetter { + + private static BucketInfo bucket(String fileIdPrefix) { + return new BucketInfo(BucketType.INSERT, fileIdPrefix, "partition"); + } + + @Test + void listBasedGetterIndexesByPosition() { + BucketInfo first = bucket("f0"); + BucketInfo second = bucket("f1"); + List bucketInfoList = Arrays.asList(first, second); + ListBasedSparkBucketInfoGetter getter = new ListBasedSparkBucketInfoGetter(bucketInfoList); + assertSame(first, getter.getBucketInfo(0)); + assertSame(second, getter.getBucketInfo(1)); + } + + @Test + void listBasedGetterRejectsOutOfRangeIndex() { + ListBasedSparkBucketInfoGetter getter = + new ListBasedSparkBucketInfoGetter(Arrays.asList(bucket("f0"))); + assertThrows(IndexOutOfBoundsException.class, () -> getter.getBucketInfo(5)); + } + + @Test + void mapBasedGetterLooksUpByBucketNumber() { + Map bucketInfoMap = new HashMap<>(); + BucketInfo target = bucket("f7"); + bucketInfoMap.put(7, target); + MapBasedSparkBucketInfoGetter getter = new MapBasedSparkBucketInfoGetter(bucketInfoMap); + assertSame(target, getter.getBucketInfo(7)); + // Missing keys resolve to null rather than throwing. + assertNull(getter.getBucketInfo(0)); + } +} From 177f67de7ea9da0c7e96ddbb90a164066740b8ce Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 8 Jul 2026 23:49:56 +0800 Subject: [PATCH 146/255] test(integ-test): add Spark 4.1.1 stack to hive-sync E2E matrix (#19216) Adds a spark4.1 row to the integration-tests-hive-sync job, backed by new docker-compose_hadoop340_hive2310_spark411_{amd64,arm64}.yml files derived from the 4.0.2 pair (image tags and HDFS cluster name only; arm64 comment normalized to ASCII). VARIANT tests run on this row via the existing spark4 compose-prefix token; no Java harness changes. Requires apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-spark{base,master,worker,adhoc}_4.1.1 images to be published to Docker Hub before the new CI row can pass. (cherry picked from commit 1046ec084732e76075ed8b3b677a23a48503b4ae) --- .github/workflows/bot.yml | 12 +- ...pose_hadoop340_hive2310_spark411_amd64.yml | 267 ++++++++++++++++++ ...pose_hadoop340_hive2310_spark411_arm64.yml | 267 ++++++++++++++++++ 3 files changed, 543 insertions(+), 3 deletions(-) create mode 100644 docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml create mode 100644 docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 851b53e786181..e4d3238ffc3fb 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1253,9 +1253,9 @@ jobs: integration-tests-hive-sync: # Testcontainers-based E2E hive sync coverage for Hudi's custom logical types - # (VECTOR, BLOB) and the Spark 4.0 VARIANT type. Runs the integ2 testcontainers - # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on both a - # Spark 3.5.3 and Spark 4.0.2 stack. + # (VECTOR, BLOB) and the Spark 4.0+ VARIANT type. Runs the integ2 testcontainers + # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark 3.5.3, + # 4.0.2, and 4.1.1 stacks. runs-on: ubuntu-latest needs: changes strategy: @@ -1274,6 +1274,12 @@ jobs: jdkVersion: '17' composePrefix: 'docker-compose_hadoop340_hive2310_spark402' sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest' + - sparkProfile: 'spark4.1' + scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13' + flinkProfile: 'flink1.20' + jdkVersion: '17' + composePrefix: 'docker-compose_hadoop340_hive2310_spark411' + sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest' steps: - if: needs.changes.outputs.relevant == 'true' uses: actions/checkout@v5 diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml new file mode 100644 index 0000000000000..15227b126c6b6 --- /dev/null +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + + namenode: + image: apachehudi/hudi-hadoop_3.4.0-namenode:latest + hostname: namenode + container_name: namenode + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + ports: + - "8020:8020" # HDFS NameNode IPC + - "9000:9000" # HDFS NameNode Client + - "9870:9870" # HDFS NameNode Web UI + env_file: + - ./hadoop.env + healthcheck: + test: ["CMD", "curl", "-f", "http://namenode:9870"] + interval: 30s + timeout: 10s + retries: 3 + + datanode1: + image: apachehudi/hudi-hadoop_3.4.0-datanode:latest + container_name: datanode1 + hostname: datanode1 + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + env_file: + - ./hadoop.env + ports: + - "50075:50075" + - "9864:9864" + - "50010:50010" + links: + - "namenode" + - "historyserver" + healthcheck: + test: ["CMD", "curl", "-f", "http://datanode1:9864"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - namenode + + historyserver: + image: apachehudi/hudi-hadoop_3.4.0-history:latest + hostname: historyserver + container_name: historyserver + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "8188:8188" + healthcheck: + test: ["CMD", "curl", "-f", "http://historyserver:8188"] + interval: 30s + timeout: 10s + retries: 3 + env_file: + - ./hadoop.env + volumes: + - historyserver:/hadoop/yarn/timeline + + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). + # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync + # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). + # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + volumes: + - hive-metastore-postgresql:/var/lib/postgresql + hostname: hive-metastore-postgresql + container_name: hive-metastore-postgresql + + hivemetastore: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hivemetastore + container_name: hivemetastore + links: + - "hive-metastore-postgresql" + - "namenode" + env_file: + - ./hadoop.env + command: /opt/hive/bin/hive --service metastore + environment: + - "SERVICE_PRECONDITION=namenode:9870 hive-metastore-postgresql:5432" + ports: + - "9083:9083" + healthcheck: + test: ["CMD", "nc", "-z", "hivemetastore", "9083"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - "hive-metastore-postgresql" + - "namenode" + + hiveserver: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hiveserver + container_name: hiveserver + env_file: + - ./hadoop.env + environment: + - SERVICE_PRECONDITION=hivemetastore:9083 + ports: + - "10000:10000" + depends_on: + - "hivemetastore" + links: + - "hivemetastore" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + zookeeper: + image: 'bitnamilegacy/zookeeper:3.6.4' + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: 'bitnamilegacy/kafka:3.4.1' + hostname: kafkabroker + container_name: kafkabroker + ports: + - "9092:9092" + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + + sparkmaster: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkmaster_4.1.1:latest + hostname: sparkmaster + container_name: sparkmaster + env_file: + - ./hadoop.env + ports: + - "8080:8080" + - "7077:7077" + - "8888:8888" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + - ./notebooks:/opt/workspace/notebooks + environment: + - INIT_DAEMON_STEP=setup_spark + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + spark-worker-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkworker_4.1.1:latest + hostname: spark-worker-1 + container_name: spark-worker-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - "8081:8081" + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + adhoc-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-1 + container_name: adhoc-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - '4040:4040' + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + adhoc-2: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-2 + container_name: adhoc-2 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + minio: + image: 'minio/minio:latest' + hostname: minio + container_name: minio + ports: + - 9090:9090 # server address + - 9091:9091 # console address + volumes: + - minio-data:/data + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=minio123 + - MINIO_DOMAIN=minio + command: server --address ":9090" --console-address ":9091" /data + + mc: + image: minio/mc + container_name: mc + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9090 minio minio123 --api S3v4) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + depends_on: + - minio + +volumes: + namenode: + historyserver: + hive-metastore-postgresql: + minio-data: + +networks: + default: + name: hudi diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml new file mode 100644 index 0000000000000..15227b126c6b6 --- /dev/null +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + + namenode: + image: apachehudi/hudi-hadoop_3.4.0-namenode:latest + hostname: namenode + container_name: namenode + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + ports: + - "8020:8020" # HDFS NameNode IPC + - "9000:9000" # HDFS NameNode Client + - "9870:9870" # HDFS NameNode Web UI + env_file: + - ./hadoop.env + healthcheck: + test: ["CMD", "curl", "-f", "http://namenode:9870"] + interval: 30s + timeout: 10s + retries: 3 + + datanode1: + image: apachehudi/hudi-hadoop_3.4.0-datanode:latest + container_name: datanode1 + hostname: datanode1 + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + env_file: + - ./hadoop.env + ports: + - "50075:50075" + - "9864:9864" + - "50010:50010" + links: + - "namenode" + - "historyserver" + healthcheck: + test: ["CMD", "curl", "-f", "http://datanode1:9864"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - namenode + + historyserver: + image: apachehudi/hudi-hadoop_3.4.0-history:latest + hostname: historyserver + container_name: historyserver + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "8188:8188" + healthcheck: + test: ["CMD", "curl", "-f", "http://historyserver:8188"] + interval: 30s + timeout: 10s + retries: 3 + env_file: + - ./hadoop.env + volumes: + - historyserver:/hadoop/yarn/timeline + + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). + # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync + # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). + # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + volumes: + - hive-metastore-postgresql:/var/lib/postgresql + hostname: hive-metastore-postgresql + container_name: hive-metastore-postgresql + + hivemetastore: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hivemetastore + container_name: hivemetastore + links: + - "hive-metastore-postgresql" + - "namenode" + env_file: + - ./hadoop.env + command: /opt/hive/bin/hive --service metastore + environment: + - "SERVICE_PRECONDITION=namenode:9870 hive-metastore-postgresql:5432" + ports: + - "9083:9083" + healthcheck: + test: ["CMD", "nc", "-z", "hivemetastore", "9083"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - "hive-metastore-postgresql" + - "namenode" + + hiveserver: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hiveserver + container_name: hiveserver + env_file: + - ./hadoop.env + environment: + - SERVICE_PRECONDITION=hivemetastore:9083 + ports: + - "10000:10000" + depends_on: + - "hivemetastore" + links: + - "hivemetastore" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + zookeeper: + image: 'bitnamilegacy/zookeeper:3.6.4' + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: 'bitnamilegacy/kafka:3.4.1' + hostname: kafkabroker + container_name: kafkabroker + ports: + - "9092:9092" + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + + sparkmaster: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkmaster_4.1.1:latest + hostname: sparkmaster + container_name: sparkmaster + env_file: + - ./hadoop.env + ports: + - "8080:8080" + - "7077:7077" + - "8888:8888" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + - ./notebooks:/opt/workspace/notebooks + environment: + - INIT_DAEMON_STEP=setup_spark + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + spark-worker-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkworker_4.1.1:latest + hostname: spark-worker-1 + container_name: spark-worker-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - "8081:8081" + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + adhoc-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-1 + container_name: adhoc-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - '4040:4040' + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + adhoc-2: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-2 + container_name: adhoc-2 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + minio: + image: 'minio/minio:latest' + hostname: minio + container_name: minio + ports: + - 9090:9090 # server address + - 9091:9091 # console address + volumes: + - minio-data:/data + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=minio123 + - MINIO_DOMAIN=minio + command: server --address ":9090" --console-address ":9091" /data + + mc: + image: minio/mc + container_name: mc + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9090 minio minio123 --api S3v4) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + depends_on: + - minio + +volumes: + namenode: + historyserver: + hive-metastore-postgresql: + minio-data: + +networks: + default: + name: hudi From ad88cb3e9109a14b1e27bb59199502d8a898534a Mon Sep 17 00:00:00 2001 From: voonhous Date: Fri, 10 Jul 2026 01:49:49 +0800 Subject: [PATCH 147/255] fix(bundle): shade parquet-variant into common bundle includes for Spark 4.1+ (#19235) Parquet 1.16.0 (Spark 4.1) split VARIANT support into a separate `parquet-variant` module carrying `org.apache.parquet.variant.VariantConverters`. `parquet-avro:1.16.0`, shaded into the spark and utilities bundles and used on the read path, references it, but the shade `` allowlists omitted parquet-variant, so the class was dropped from the bundles and absent at runtime. On Spark 4.1 every read-modify write (UPDATE / MERGE / DELETE) then aborted with NoClassDefFoundError; only INSERT succeeded. This affects any Spark 4.1 bundle that shades parquet-avro: both hoodie-spark-bundle (surfaced by the hive-sync E2E, #19216) and hoodie-utilities-bundle (the HoodieStreamer standalone runtime). Add parquet-variant to the shared "common to all bundles" artifactSet in the root pom that every bundle appends to via combine.children="append". It is a transitive compile dep of parquet-avro for Parquet 1.16.0+, so it shades for Spark 4.1/4.2 and is a no-op for older Spark and bundles that pin pre-1.16 parquet. Verified the rebuilt spark4.1 spark-bundle and utilities-bundle both contain VariantConverters. Fixes apache/hudi#19234 (cherry picked from commit 9df73c747453790e1fea6a8be6d8bac3c37f48ab) --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 1d5d6e17ebfda..ae01441bf9585 100644 --- a/pom.xml +++ b/pom.xml @@ -479,6 +479,9 @@ com.fasterxml.jackson.module:jackson-module-afterburner com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version} + + org.apache.parquet:parquet-variant From 24bd11b2b4cff36947f13719fb31d2106e6b46fa Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 9 Jul 2026 10:51:13 -0700 Subject: [PATCH 148/255] test(spark): add extended SQL parser coverage for index DDL and Hudi column types (#19218) * test(spark): add extended SQL parser coverage for index DDL and Hudi column types * test(spark): strengthen parser coverage assertions per review - BLOB/VECTOR tests now assert the hudi_type field metadata (BLOB, canonical VECTOR(3)) and the mapped dataType (BlobType(), ArrayType(FloatType, false)) instead of only checking the column name exists, so they fail if the custom type maps incorrectly. - Drop the unused create-table setup from the four index-DDL tests; parsePlan is purely syntactic, so the parse is identical whether or not the table exists (noted on the parse helper). * test(spark): fold parser coverage into TestIndexSyntax per review TestExtendedSqlParserCoverage duplicated existing coverage: BLOB/VECTOR create-table + hudi_type assertions live in TestCreateTable and TestBlobDataType, and CREATE/DROP/SHOW INDEX parse+resolution is covered by TestIndexSyntax. The only genuine gap was REFRESH INDEX, which TestIndexSyntax's 'Test Create/Drop/Show/Refresh Index' promised in its title but never exercised. Add the refresh-index parse+analyze assertions there, mirroring the sibling statements, and drop the redundant suite. * test(spark): assert catalog-stored schema retains hudi_type for BLOB/VECTOR The dropped parser-coverage suite read the schema back via catalog.getTableMetadata rather than spark.table; that was the one assertion without an existing home: the CatalogTable persisted by CreateHoodieTableCommand retains the hudi_type field metadata. Pin it in TestCreateTable's BLOB and VECTOR create-table tests. --------- Co-authored-by: voon (cherry picked from commit b22a3d7af37a5c87665477b5e5775307d0f3137d) --- .../apache/spark/sql/hudi/ddl/TestCreateTable.scala | 10 ++++++++++ .../spark/sql/hudi/feature/index/TestIndexSyntax.scala | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala index 88ec4b7fb4c36..ab70edb18b4ef 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala @@ -2064,6 +2064,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase { // Verify structure matches blob schema assertTrue(videoField.dataType.isInstanceOf[StructType]) assertEquals(BlobType(), videoField.dataType) + + // The catalog-stored copy of the schema must retain the hudi_type metadata too + val catalogField = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + .schema.find(_.name == "video").get + assertEquals(HoodieSchemaType.BLOB.name(), catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) } } @@ -2183,6 +2188,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase { assertEquals("VECTOR(128)", embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) assertEquals("document embedding", embeddingField.metadata.getString("comment")) assertEquals(ArrayType(FloatType, containsNull = false), embeddingField.dataType) + + // The catalog-stored copy of the schema must retain the hudi_type metadata too + val catalogField = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + .schema.find(_.name == "embedding").get + assertEquals("VECTOR(128)", catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala index 8fb3fd8648c03..2e186ae5347b4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala @@ -26,7 +26,7 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.hudi.command.{CreateIndexCommand, DropIndexCommand, ShowIndexesCommand} +import org.apache.spark.sql.hudi.command.{CreateIndexCommand, DropIndexCommand, RefreshIndexCommand, ShowIndexesCommand} import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue} @@ -90,6 +90,11 @@ class TestIndexSyntax extends HoodieSparkSqlTestBase { assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].table, databaseName, tableName) assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].indexName) assertResult(true)(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].ignoreIfNotExists) + + logicalPlan = sqlParser.parsePlan(s"refresh index idx_name on $tableName") + resolvedLogicalPlan = analyzer.execute(logicalPlan) + assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].table, databaseName, tableName) + assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].indexName) } } } From 61655026b978ad140c384ec1059c1b44de11fd0d Mon Sep 17 00:00:00 2001 From: vinoth chandar Date: Fri, 10 Jul 2026 01:53:42 -0700 Subject: [PATCH 149/255] fix(spark): read Lance BLOB columns in <=512-row chunks to avoid lance-core FFI abort (#19181) * fix(spark): read Lance BLOB columns in <=512-row chunks to avoid lance-core FFI abort lance-core 4.0.0 aborts the JVM in its Arrow C-stream export (arrow_array::ffi_stream::get_next: "range end index N out of range for slice of length 0") whenever a single readAll stream crosses Lance's internal BLOB page boundary (512 rows); the requested batchSize does not help because Lance re-chunks BLOB columns at 512 internally. As a result CoW Lance tables with BLOB columns could not be read past 512 rows: OUT_OF_LINE reads threw an Arrow "should have as many children as in the schema" error and INLINE CONTENT reads crashed the JVM (SIGABRT). Read BLOB-containing Lance files in <=512-row row-range chunks, issuing a fresh readAll per chunk so each FFI stream stays within a single BLOB page and the buggy second-page export is never reached. LanceRecordIterator gains a chunkedBlobReader(...) factory driven by an ArrowReaderSupplier; SparkLanceReaderBase (SQL/DataFrame read) and HoodieSparkLanceReader (internal CONTENT path) route to it when the projection contains a BLOB field. Non-BLOB reads keep the single streamed reader and are unchanged; the per-row hot path is untouched, with only small fixed per-chunk allocations. Also document hoodie.read.blob.inline.mode option placement (read option for both DataFrame and SQL) and add batch-scale regression tests: INLINE CONTENT, VECTOR + OUT_OF_LINE BLOB at n=100/1000 (crossing the 512-row boundary), with vector top-k, projection/filter, and read_blob byte + SHA-256 checks. * refactor(spark): rename ArrowReaderSupplier to ArrowReaderSequence, use nonEmpty * fix(spark): recurse Lance BLOB chunking detection, pin chunk size to lance-core 4.0.0 Route reads through the chunked path when a BLOB appears at any nesting depth (HoodieSchema.containsBlobType() made public for the internal reader; recursive StructType walk in SparkLanceReaderBase), so a future nested-BLOB writer change cannot silently skip chunking and re-introduce the lance-core FFI abort. Document BLOB_READ_CHUNK_ROWS as pinned to lance-core 4.0.0's internal 512-row BLOB page size, revalidated on lance upgrades. --------- Co-authored-by: voon (cherry picked from commit 14fe7393a42fc1f6ac79533fa5cac18d0812330a) --- .../io/storage/HoodieSparkLanceReader.java | 10 +- .../hudi/io/storage/LanceRecordIterator.java | 172 +++++++++--- .../common/config/HoodieReaderConfig.java | 9 +- .../hudi/common/schema/HoodieSchema.java | 7 +- .../lance/SparkLanceReaderBase.scala | 32 ++- .../hudi/functional/TestLanceDataSource.scala | 265 +++++++++++++++++- 6 files changed, 444 insertions(+), 51 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java index bd651a9e9bef0..a930eaa4b921c 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java @@ -208,8 +208,16 @@ private ClosableIterator getUnsafeRowIterator(HoodieSchema requestedS // Pinned to CONTENT: compaction/merge/log-replay need actual bytes to rewrite. // The user-facing `hoodie.read.blob.inline.mode` is honored by SparkLanceReaderBase. FileReadOptions readOpts = FileReadOptions.builder().blobReadMode(BlobReadMode.CONTENT).build(); - ArrowReader arrowReader = lanceReader.readAll(columnNames, null, DEFAULT_BATCH_SIZE, readOpts); + // BLOB reads must be chunked to dodge a lance-core FFI abort (see LanceRecordIterator). + // containsBlobType() recurses through nested records/arrays/maps/unions, so a BLOB at any + // depth routes through the chunked path; a top-level-only check would silently skip + // chunking (and re-introduce the abort) if the writer ever gains nested-BLOB support. + if (requestedSchema.containsBlobType()) { + return LanceRecordIterator.chunkedBlobReader(allocator, lanceReader, columnNames, readOpts, + lanceReader.numRows(), requestedSparkSchema, path.toString(), null); + } + ArrowReader arrowReader = lanceReader.readAll(columnNames, null, DEFAULT_BATCH_SIZE, readOpts); return new LanceRecordIterator(allocator, lanceReader, arrowReader, requestedSparkSchema, path.toString()); } catch (Exception e) { allocator.close(); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/LanceRecordIterator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/LanceRecordIterator.java index 91a0421d01186..2cc6b5c314c72 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/LanceRecordIterator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/LanceRecordIterator.java @@ -32,39 +32,58 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; +import org.lance.file.FileReadOptions; import org.lance.file.LanceFileReader; import org.lance.spark.vectorized.LanceArrowColumnVector; +import org.lance.util.Range; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; /** - * Iterator for reading Lance files and converting Arrow batches to Spark {@link UnsafeRow}s. - * Used by both Hudi's internal Lance reader and Spark datasource integration. + * Iterator over a Lance file that converts Arrow batches to Spark {@link UnsafeRow}s, owning the + * {@link BufferAllocator}, {@link LanceFileReader}, {@link ArrowReader}(s) and current + * {@link ColumnarBatch}. Used by both Hudi's internal Lance reader and the Spark datasource. An + * optional {@link BlobDescriptorTransform} rewrites BLOB columns for DESCRIPTOR-mode reads. * - *

    The iterator manages the lifecycle of: - *

      - *
    • BufferAllocator - Arrow memory management
      • - *
    • LanceFileReader - Lance file handle
      • - *
    • ArrowReader - Arrow batch reader
      • - *
    • ColumnarBatch - Current batch being iterated
      • - *
    - * - *

    An optional {@link BlobDescriptorTransform} can be composed in to rewrite BLOB columns - * in DESCRIPTOR mode. + *

    BLOB chunked reading: lance-core 4.0.0 aborts the JVM in its Arrow C-stream export whenever a + * single {@code readAll} stream crosses Lance's internal BLOB page boundary (512 rows), and the + * requested {@code batchSize} does not change that. So BLOB reads issue one {@code readAll} per + * row-range chunk of {@link #BLOB_READ_CHUNK_ROWS} rows; non-BLOB reads use one streamed reader. */ public final class LanceRecordIterator implements ClosableIterator { + + /** + * Rows per {@code readAll} for BLOB reads. Must not exceed Lance's internal BLOB page size, + * which is 512 rows in lance-core 4.0.0 but is NOT exposed through any lance API; revalidate + * this constant against the batch-scale BLOB tests whenever {@code lance.version} is bumped. + */ + public static final int BLOB_READ_CHUNK_ROWS = 512; + + /** + * The sequence of {@link ArrowReader}s to drain, one after another. Single-reader + * mode yields one reader; BLOB chunked mode yields one fresh reader per row-range chunk. Each + * returned reader is owned (and closed) by {@link LanceRecordIterator}. + */ + private interface ArrowReaderSequence { + /** @return the next reader to drain, or {@code null} when the sequence is exhausted. */ + ArrowReader next() throws IOException; + } + private final BufferAllocator allocator; private final LanceFileReader lanceReader; - private final ArrowReader arrowReader; + private final ArrowReaderSequence readerSequence; private final StructType sparkSchema; private final UnsafeProjection projection; private final String path; private final BlobDescriptorTransform blobTransform; + /** The reader currently being drained; replaced as chunks are exhausted. */ + private ArrowReader currentReader; private ColumnarBatch currentBatch; private Iterator rowIterator; private ColumnVector[] columnVectors; @@ -84,7 +103,8 @@ public LanceRecordIterator(BufferAllocator allocator, } /** - * Creates a new Lance record iterator. + * Creates a new Lance record iterator that drains a single pre-built {@link ArrowReader}. + * Suitable for non-BLOB reads, where Lance's multi-batch FFI export is well-behaved. * * @param allocator Arrow buffer allocator for memory management * @param lanceReader Lance file reader @@ -100,15 +120,78 @@ public LanceRecordIterator(BufferAllocator allocator, StructType schema, String path, BlobDescriptorTransform blobTransform) { + this(allocator, lanceReader, singleReaderSequence(arrowReader), schema, path, blobTransform); + } + + private LanceRecordIterator(BufferAllocator allocator, + LanceFileReader lanceReader, + ArrowReaderSequence readerSequence, + StructType schema, + String path, + BlobDescriptorTransform blobTransform) { this.allocator = allocator; this.lanceReader = lanceReader; - this.arrowReader = arrowReader; + this.readerSequence = readerSequence; this.sparkSchema = schema; this.projection = UnsafeProjection.create(schema); this.path = path; this.blobTransform = blobTransform; } + /** + * Creates a Lance record iterator that reads a BLOB-containing file in fixed-size row-range + * chunks, issuing a fresh {@code readAll} per chunk to dodge the lance-core multi-page BLOB + * FFI-export panic (see class javadoc). + * + * @param allocator Arrow buffer allocator for memory management + * @param lanceReader open Lance file reader (the iterator takes ownership and closes it) + * @param columnNames columns to project, or {@code null} for all columns + * @param readOpts Lance read options (e.g. blob read mode) + * @param totalRows total rows in the file ({@code lanceReader.numRows()}) + * @param schema Spark schema for the records + * @param path File path (for error messages) + * @param blobTransform optional blob descriptor transform for DESCRIPTOR-mode reads + */ + public static LanceRecordIterator chunkedBlobReader(BufferAllocator allocator, + LanceFileReader lanceReader, + List columnNames, + FileReadOptions readOpts, + long totalRows, + StructType schema, + String path, + BlobDescriptorTransform blobTransform) { + ArrowReaderSequence sequence = new ArrowReaderSequence() { + private long nextStart = 0; + + @Override + public ArrowReader next() throws IOException { + if (nextStart >= totalRows) { + return null; + } + int start = Math.toIntExact(nextStart); + int end = Math.toIntExact(Math.min(nextStart + BLOB_READ_CHUNK_ROWS, totalRows)); + nextStart = end; + // A single range per readAll keeps the FFI stream within one BLOB page (<= 512 rows). + List ranges = Collections.singletonList(new Range(start, end)); + return lanceReader.readAll(columnNames, ranges, BLOB_READ_CHUNK_ROWS, readOpts); + } + }; + return new LanceRecordIterator(allocator, lanceReader, sequence, schema, path, blobTransform); + } + + private static ArrowReaderSequence singleReaderSequence(ArrowReader arrowReader) { + return new ArrowReaderSequence() { + private ArrowReader remaining = arrowReader; + + @Override + public ArrowReader next() { + ArrowReader r = remaining; + remaining = null; + return r; + } + }; + } + @Override public boolean hasNext() { if (rowIterator != null && rowIterator.hasNext()) { @@ -120,33 +203,50 @@ public boolean hasNext() { currentBatch = null; } - // Try to load next batch. Loop so zero-row batches (legitimately returned e.g. after - // filter pushdown) don't silently terminate iteration and drop subsequent non-empty batches. try { - while (arrowReader.loadNextBatch()) { - VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); - - // Build ColumnVector[] in Spark-schema order by looking each field up by name; - // lance-spark 0.4.0's VectorSchemaRoot may return the file's on-disk order, which - // would misalign the UnsafeProjection. Cached on the first batch and reused thereafter. - if (columnVectors == null) { - buildColumnVectors(root); + while (true) { + if (currentReader == null) { + currentReader = readerSequence.next(); + if (currentReader == null) { + return false; + } + // Each reader (range chunk) returns a distinct VectorSchemaRoot, so the cached + // column vectors must be rebuilt against the new reader's vectors. + columnVectors = null; } - currentBatch = new ColumnarBatch(columnVectors, root.getRowCount()); - rowIterator = currentBatch.rowIterator(); - rowIdInBatch = 0; - if (rowIterator.hasNext()) { - return true; + // Try to load next batch from the current reader. Loop so zero-row batches + // (legitimately returned e.g. after filter pushdown) don't silently terminate. + while (currentReader.loadNextBatch()) { + VectorSchemaRoot root = currentReader.getVectorSchemaRoot(); + + // Build ColumnVector[] in Spark-schema order by looking each field up by name; + // lance-spark 0.4.0's VectorSchemaRoot may return the file's on-disk order, which + // would misalign the UnsafeProjection. Cached per reader and reused thereafter. + if (columnVectors == null) { + buildColumnVectors(root); + } + + currentBatch = new ColumnarBatch(columnVectors, root.getRowCount()); + rowIterator = currentBatch.rowIterator(); + rowIdInBatch = 0; + if (rowIterator.hasNext()) { + return true; + } + currentBatch.close(); + currentBatch = null; } - currentBatch.close(); - currentBatch = null; + + // Current reader exhausted; close it and advance to the next chunk (if any). + currentReader.close(); + currentReader = null; + columnVectors = null; } } catch (IOException e) { throw new HoodieException("Failed to read next batch from Lance file: " + path, e); + } catch (Exception e) { + throw new HoodieException("Failed to advance Lance reader for file: " + path, e); } - - return false; } @Override @@ -197,6 +297,8 @@ public void close() { closed = true; ColumnarBatch batch = currentBatch; currentBatch = null; - LanceResourceCloser.closeAll(batch, arrowReader, lanceReader, allocator); + ArrowReader reader = currentReader; + currentReader = null; + LanceResourceCloser.closeAll(batch, reader, lanceReader, allocator); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java index 9cbab8f4468cd..a1ecf64dc419b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java @@ -112,8 +112,9 @@ public class HoodieReaderConfig extends HoodieConfig { .sinceVersion("1.2.0") .withValidValues(BLOB_INLINE_READ_MODE_CONTENT, BLOB_INLINE_READ_MODE_DESCRIPTOR) .withDocumentation("How Hudi interprets INLINE BLOB values on read. " - + "DESCRIPTOR (default) returns an OUT_OF_LINE-shaped reference pointing at the " - + "backing Lance file with the INLINE payload's position and size, so callers can " - + "skip the byte content read. " - + "CONTENT returns the raw inline bytes directly in the data field on every read."); + + "DESCRIPTOR (default) returns an OUT_OF_LINE-shaped reference (position and size) into " + + "the backing file, skipping the byte read. CONTENT returns the raw inline bytes in the " + + "data field. Materializing INLINE bytes via read_blob() requires CONTENT; under " + + "DESCRIPTOR it fails fast asking for CONTENT. Pass this as a read option, not a session " + + "config. OUT_OF_LINE blobs ignore this option."); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java index f38b9e2c93ed1..8c151af35a47a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java @@ -1390,7 +1390,12 @@ public HoodieSchema getNonNullType() { return HoodieSchema.createUnion(nonNullTypes); } - boolean containsBlobType() { + /** + * Recursively checks whether this schema is or contains a BLOB type at any nesting depth, + * descending through arrays, maps, unions and record fields. Unlike {@link #isBlobField()}, + * this also finds BLOBs nested inside records. + */ + public boolean containsBlobType() { if (getType() == HoodieSchemaType.BLOB) { return true; } else if (getType() == HoodieSchemaType.ARRAY) { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala index 65dec06cd3e00..ddbb798a57115 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala @@ -137,19 +137,29 @@ class SparkLanceReaderBase(enableVectorizedReader: Boolean) extends SparkColumna // the option regardless. val blobMode = resolveBlobReadMode(storageConf) val readOpts = FileReadOptions.builder().blobReadMode(blobMode).build() - val arrowReader = lanceReader.readAll(columnNames, null, DEFAULT_BATCH_SIZE, readOpts) // Compose the DESCRIPTOR-aware blob transform only when the user opted into that mode // AND the request actually has BLOB columns (otherwise the rewrite has nothing to do). - val blobFieldNames: java.util.Set[String] = - iteratorSchema.fields.collect { case f if isBlobField(f) => f.name }.toSet.asJava - val blobTransform = if (blobMode == BlobReadMode.DESCRIPTOR && !blobFieldNames.isEmpty) { - new BlobDescriptorTransform(blobFieldNames, filePath) + val blobFieldNames: Set[String] = + iteratorSchema.fields.collect { case f if isBlobField(f) => f.name }.toSet + val blobTransform = if (blobMode == BlobReadMode.DESCRIPTOR && blobFieldNames.nonEmpty) { + new BlobDescriptorTransform(blobFieldNames.asJava, filePath) } else { null } - lanceIterator = new LanceRecordIterator( - allocator, lanceReader, arrowReader, iteratorSchema, filePath, blobTransform) + // lance-core 4.0.0 aborts the JVM when a single readAll stream crosses Lance's internal + // BLOB page boundary (512 rows). For BLOB-containing reads, drain the file in <=512-row + // range chunks (one fresh readAll each); non-BLOB reads keep the single streamed reader. + // The detection recurses so a nested BLOB (unsupported by the writer today) still chunks. + lanceIterator = if (containsBlobField(iteratorSchema)) { + LanceRecordIterator.chunkedBlobReader( + allocator, lanceReader, columnNames, readOpts, lanceReader.numRows(), + iteratorSchema, filePath, blobTransform) + } else { + val arrowReader = lanceReader.readAll(columnNames, null, DEFAULT_BATCH_SIZE, readOpts) + new LanceRecordIterator( + allocator, lanceReader, arrowReader, iteratorSchema, filePath, blobTransform) + } // Register cleanup listener Option(TaskContext.get()).foreach { ctx => @@ -250,6 +260,14 @@ class SparkLanceReaderBase(enableVectorizedReader: Boolean) extends SparkColumna .getType == HoodieSchemaType.BLOB } + /** Recursively checks for a BLOB field (see [[isBlobField]]) at any nesting depth. */ + private def containsBlobField(dt: DataType): Boolean = dt match { + case s: StructType => s.fields.exists(f => isBlobField(f) || containsBlobField(f.dataType)) + case a: ArrayType => containsBlobField(a.elementType) + case m: MapType => containsBlobField(m.valueType) + case _ => false + } + private def forceFieldNullable(field: StructField): StructField = field.copy(nullable = true, dataType = forceTypeNullable(field.dataType)) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala index abce02568633b..419a7bcf64984 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala @@ -991,7 +991,7 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { // read_blob() on INLINE rows under DESCRIPTOR mode is unsupported by design: DESCRIPTOR // is metadata-only and the synthesized reference is an internal pointer into the .lance // file's storage layout, not user-facing metadata. BatchedBlobReader must throw with a - // message that names both INLINE and DESCRIPTOR so the failure is actionable. + // message that names INLINE, DESCRIPTOR and the CONTENT fix so the failure is actionable val viewName = s"${tableName}_view" spark.read.format("hudi") .option(modeKey, "DESCRIPTOR") @@ -1004,8 +1004,8 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { }) val msgChain = Iterator.iterate[Throwable](ex)(_.getCause).takeWhile(_ != null) .flatMap(t => Option(t.getMessage)).mkString(" | ") - assertTrue(msgChain.contains("INLINE") && msgChain.contains("DESCRIPTOR"), - s"error must mention INLINE and DESCRIPTOR; got: $msgChain") + assertTrue(Seq("INLINE", "DESCRIPTOR", "CONTENT").forall(msgChain.contains), + s"error must name INLINE, DESCRIPTOR and the CONTENT fix; got: $msgChain") } /** @@ -1981,6 +1981,253 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { writer.mode(saveMode).save(tablePath) } + + // The small-n blob tests above all read fewer than 512 rows, so they never cross Lance's + // internal BLOB page boundary (512 rows). BLOB reads regress specifically once a single base + // file holds more than 512 rows: a single Lance readAll stream then exports a second BLOB page + // through the Arrow C FFI, which panics in lance-core 4.0.0. These tests size the base file + // above 512 rows (single coalesced file) to exercise the chunked-read work-around. + private val BATCH_SCALE_ROWS = 1000 + private val BATCH_SCALE_PARALLELISM_OPTS = Map( + "hoodie.bulkinsert.shuffle.parallelism" -> "1", + "hoodie.insert.shuffle.parallelism" -> "1") + + /** + * Batch-scale INLINE BLOB read regression (HUDI-UNSTRUCTURED-001). Writes + * {@code BATCH_SCALE_ROWS} inline-blob rows into a single Lance base file and reads them back + * under CONTENT mode, materializing each via {@code read_blob()}. Reproduces the native Lance + * BLOB decoder failure that only surfaces once the read crosses the 512-row batch boundary. + */ + @ParameterizedTest + @EnumSource(value = classOf[HoodieTableType]) + def testBlobInlineContentBatchScale(tableType: HoodieTableType): Unit = { + val tableName = s"test_lance_blob_inline_batch_${tableType.name().toLowerCase}" + val tablePath = s"$basePath/$tableName" + + val payloadLen = 256 + val n = BATCH_SCALE_ROWS + val sparkSess = spark + import sparkSess.implicits._ + // Deterministic per-row payload: row i -> bytes (i + j) % 256, so a row/byte misalignment + // across the batch boundary surfaces as a byte mismatch rather than silently passing. + def payloadFor(i: Int): Array[Byte] = + (0 until payloadLen).map(j => ((i + j) % 256).toByte).toArray + val baseDf = (0 until n).map(i => (i, payloadFor(i))) + .toDF("id", "bytes") + .coalesce(1) + val rawDf = baseDf.select($"id", BlobTestHelpers.inlineBlobStructCol("payload", $"bytes")) + val canonicalSchema = StructType(Seq( + StructField("id", IntegerType, nullable = false), + StructField("payload", BlobType().asInstanceOf[StructType], nullable = true, + BlobTestHelpers.blobMetadata) + )) + val df = spark.createDataFrame(rawDf.rdd, canonicalSchema).coalesce(1) + + writeDataframe(tableType, tableName, tablePath, df, saveMode = SaveMode.Overwrite, + operation = Some("bulk_insert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id") ++ BATCH_SCALE_PARALLELISM_OPTS) + + assertLanceBlobEncoding(tablePath) + assertSingleLanceBaseFileSpansMultipleBatches(tablePath) + + val viewName = s"${tableName}_view" + spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .createOrReplaceTempView(viewName) + val materialized = spark.sql( + s"SELECT id, read_blob(payload) AS bytes FROM $viewName ORDER BY id").collect() + assertEquals(n, materialized.length, "row count mismatch after read_blob at batch scale") + materialized.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val bytes = row.getAs[Array[Byte]]("bytes") + assertArrayEquals(payloadFor(id), bytes, s"inline read_blob() bytes mismatch for id=$id") + } + } + + /** + * Comprehensive batch-scale VECTOR + OUT_OF_LINE BLOB regression (HUDI-UNSTRUCTURED-002 and 003). + * Parameterized over table type and {@code n} in {100, 1000}; n=1000 crosses the 512-row BLOB page + * boundary that trips the lance-core FFI panic without the chunked-read fix. Writes one Lance base + * file with a VECTOR(32) and an OUT_OF_LINE BLOB column, then validates across DataFrame and SQL + * reads: row count; exact vector values (incl. rows straddling the boundary); top-k IDs via + * {@code hudi_vector_search}; column projection and id-range filtering; {@code read_blob()} bytes + + * SHA-256 (full and filtered); and DESCRIPTOR-mode reference pass-through. Default Lance read + * allocator (256MB) suffices — no non-default settings required. + */ + @ParameterizedTest + @MethodSource(Array("vectorBlobBatchParams")) + def testVectorAndBlobBatchScale(tableType: HoodieTableType, n: Int): Unit = { + val tableName = s"test_lance_vec_blob_batch_${n}_${tableType.name().toLowerCase}" + val tablePath = s"$basePath/$tableName" + + val dim = 32 + val payloadLen = 512 + val externalDir = Files.createDirectories( + Paths.get(s"$basePath/_vec_blob_ext_${n}_${tableType.name().toLowerCase}")) + val extPath = BlobTestHelpers.createTestFile(externalDir, "vec_blob.bin", n * payloadLen) + + val sparkSess = spark + import sparkSess.implicits._ + // Deterministic, strictly monotonic-by-id vector: row i -> [(i*dim+j)/1000f]. Monotonicity + // makes nearest-neighbor ordering predictable; per-row distinctness makes a row/value + // misalignment across the batch boundary surface as a value (or top-k) mismatch. + def vectorFor(i: Int): Array[Float] = (0 until dim).map(j => (i * dim + j) / 1000.0f).toArray + // Deterministic blob bytes for row i: (i*payloadLen + k) % 256, matching assertBytesContent. + def expectedBlob(i: Int): Array[Byte] = + (0 until payloadLen).map(k => ((i * payloadLen + k) % 256).toByte).toArray + def sha256(bytes: Array[Byte]): Seq[Byte] = + java.security.MessageDigest.getInstance("SHA-256").digest(bytes).toSeq + + val baseDf = (0 until n) + .map(i => (i, vectorFor(i), extPath, (i.toLong * payloadLen), payloadLen.toLong)) + .toDF("id", "embedding", "path", "offset", "length") + .coalesce(1) + val rawDf = baseDf.select($"id", $"embedding", + BlobTestHelpers.blobStructCol("payload", $"path", $"offset", $"length")) + val vectorMeta = new MetadataBuilder() + .putString(HoodieSchema.TYPE_METADATA_FIELD, s"VECTOR($dim)").build() + val canonicalSchema = StructType(Seq( + StructField("id", IntegerType, nullable = false), + StructField("embedding", ArrayType(FloatType, containsNull = false), nullable = false, + vectorMeta), + StructField("payload", BlobType().asInstanceOf[StructType], nullable = true, + BlobTestHelpers.blobMetadata) + )) + val df = spark.createDataFrame(rawDf.rdd, canonicalSchema).coalesce(1) + + writeDataframe(tableType, tableName, tablePath, df, saveMode = SaveMode.Overwrite, + operation = Some("bulk_insert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id") ++ BATCH_SCALE_PARALLELISM_OPTS) + + if (n > 512) { + assertSingleLanceBaseFileSpansMultipleBatches(tablePath) + } + + // --- Vectors (DataFrame path): row count + exact values on a sample straddling the boundary. + val vecRows = spark.read.format("hudi").load(tablePath) + .select($"id", $"embedding").orderBy($"id").collect() + assertEquals(n, vecRows.length, "vector row count mismatch at batch scale") + val sampleIds = (Seq(0, 1, n / 2, n - 1) ++ (if (n > 512) Seq(511, 512, 513) else Seq.empty)) + .filter(i => i >= 0 && i < n).distinct + val vecById = vecRows.map(r => r.getInt(r.fieldIndex("id")) -> r).toMap + sampleIds.foreach { id => + val emb = vecById(id).getSeq[Float](vecById(id).fieldIndex("embedding")).toArray + assertEquals(dim, emb.length, s"vector dim mismatch for id=$id") + val expected = vectorFor(id) + (0 until dim).foreach { j => + assertEquals(expected(j), emb(j), 1e-6f, s"vector value mismatch id=$id j=$j") + } + } + + // --- Top-k vector search IDs: query near row q nudged toward higher ids by a small delta so + // the L2 ordering is strict (q, q+1, q-1). hudi_vector_search must return those exact IDs. + val tkView = s"${tableName}_tk" + spark.read.format("hudi").load(tablePath) + .select("id", "embedding").createOrReplaceTempView(tkView) + val q = n / 2 + val delta = 0.005f + val queryLiteral = vectorFor(q).map(v => (v + delta).toDouble).mkString(", ") + val topk = spark.sql( + s"""SELECT id, _hudi_distance + |FROM hudi_vector_search('$tkView', 'embedding', ARRAY($queryLiteral), 3, 'l2') + |ORDER BY _hudi_distance""".stripMargin).collect() + val topkIds = topk.map(_.getInt(0)).toSeq + assertEquals(Seq(q, q + 1, q - 1), topkIds, + s"top-3 vector-search IDs mismatch for query near id=$q") + + // --- Projection (id only) and predicate filter (id range) correctness. + val idOnly = spark.read.format("hudi").load(tablePath).select("id") + assertEquals(1, idOnly.schema.fields.length, "projection should yield a single column") + // collect() (not count()) so the id column is actually projected/read; a count() would push an + // empty required schema down the scan, a separate code path not under test here. + val idOnlyVals = idOnly.collect().map(_.getInt(0)).toSet + assertEquals((0 until n).toSet, idOnlyVals, "projected id set mismatch") + // For n=1000 choose a range that straddles the 512-row batch boundary (400..620). + val lo = if (n > 512) 400 else n / 4 + val hi = math.min(n, lo + (if (n > 512) 220 else 30)) + val filteredIds = spark.read.format("hudi").load(tablePath) + .where(s"id >= $lo AND id < $hi").select("id").orderBy("id") + .collect().map(_.getInt(0)).toSeq + assertEquals((lo until hi).toSeq, filteredIds, "filtered id range mismatch") + + // --- Blobs (SQL path, CONTENT): full-scan read_blob byte content + SHA-256 round-trip. + val viewName = s"${tableName}_view" + spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .createOrReplaceTempView(viewName) + val blobRows = spark.sql( + s"SELECT id, read_blob(payload) AS bytes FROM $viewName ORDER BY id").collect() + assertEquals(n, blobRows.length, "blob row count mismatch at batch scale") + blobRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val bytes = row.getAs[Array[Byte]]("bytes") + assertEquals(payloadLen, bytes.length, s"blob length mismatch for id=$id") + BlobTestHelpers.assertBytesContent(bytes, expectedOffset = id * payloadLen) + } + // SHA-256 round-trip on a sample (faithful to the AC's SHA256 requirement). + val blobById = blobRows.map(r => r.getInt(r.fieldIndex("id")) -> r.getAs[Array[Byte]]("bytes")).toMap + sampleIds.foreach { id => + assertEquals(sha256(expectedBlob(id)), sha256(blobById(id)), + s"blob SHA-256 mismatch for id=$id") + } + + // --- Blobs under a filter (SQL path, CONTENT): read_blob restricted to an id range. + val filteredBlobs = spark.sql( + s"SELECT id, read_blob(payload) AS bytes FROM $viewName WHERE id >= $lo AND id < $hi ORDER BY id") + .collect() + assertEquals(hi - lo, filteredBlobs.length, "filtered read_blob count mismatch") + filteredBlobs.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + assertEquals(sha256(expectedBlob(id)), sha256(row.getAs[Array[Byte]]("bytes")), + s"filtered blob SHA-256 mismatch for id=$id") + } + + // --- DESCRIPTOR-mode read at scale: exercises the chunked reader WITH the blob transform + // (re-initialized per chunk). OUT_OF_LINE references must pass through intact on both sides of + // the batch boundary. + val descRows = spark.read.format("hudi").option("hoodie.read.blob.inline.mode", "DESCRIPTOR") + .load(tablePath).select($"id", $"payload").orderBy($"id").collect() + val descById = descRows.map(r => r.getInt(r.fieldIndex("id")) -> r).toMap + sampleIds.foreach { id => + val payload = descById(id).getStruct(descById(id).fieldIndex("payload")) + assertEquals(HoodieSchema.Blob.OUT_OF_LINE, + payload.getString(payload.fieldIndex(HoodieSchema.Blob.TYPE)), s"type mismatch id=$id") + val ref = payload.getStruct(payload.fieldIndex(HoodieSchema.Blob.EXTERNAL_REFERENCE)) + assertTrue(ref.getString(ref.fieldIndex(HoodieSchema.Blob.EXTERNAL_REFERENCE_PATH)) + .endsWith(".bin"), s"external_path mismatch id=$id") + assertEquals(id.toLong * payloadLen, + ref.getLong(ref.fieldIndex(HoodieSchema.Blob.EXTERNAL_REFERENCE_OFFSET)), + s"reference offset mismatch id=$id") + } + } + + /** + * Guards the batch-scale tests' core assumption: exactly one Lance base file holds all rows and + * that file has more rows than a single Arrow read batch (512). If a future change splits the + * write across files or shrinks the row count below the batch size, the cross-batch drain path + * would no longer be exercised and the regression would silently stop reproducing. + */ + private def assertSingleLanceBaseFileSpansMultipleBatches(tablePath: String): Unit = { + val lanceFiles = Files.walk(Paths.get(tablePath)) + .filter(p => p.toString.endsWith(".lance")) + .collect(Collectors.toList[java.nio.file.Path]).asScala + assertEquals(1, lanceFiles.length, + s"expected exactly one Lance base file, found: ${lanceFiles.mkString(", ")}") + val allocator = new RootAllocator(64L * 1024 * 1024) + try { + val reader = LanceFileReader.open(lanceFiles.head.toString, allocator) + try { + assertTrue(reader.numRows() > 512, + s"base file must span >512 rows to cross a batch boundary, got ${reader.numRows()}") + } finally { + reader.close() + } + } finally { + allocator.close() + } + } } object TestLanceDataSource { @@ -1992,4 +2239,16 @@ object TestLanceDataSource { } yield Arguments.of(tableType, readMode: java.lang.String) java.util.stream.Stream.of(params: _*) } + + /** + * Cross-product of table types and row counts for the VECTOR+BLOB batch-scale suite. n=100 stays + * within one Arrow batch; n=1000 crosses the 512-row Lance BLOB page boundary. + */ + def vectorBlobBatchParams(): java.util.stream.Stream[Arguments] = { + val params = for { + tableType <- HoodieTableType.values() + n <- Array(100, 1000) + } yield Arguments.of(tableType, n: java.lang.Integer) + java.util.stream.Stream.of(params: _*) + } } From a8d886845ce5854eefd4b166978c9ff4446ee9cf Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Fri, 10 Jul 2026 18:05:25 +0700 Subject: [PATCH 150/255] fix(common): name the offending expression in BindVisitor's unsupported-predicate error (#19241) * fix(common): name the offending expression in BindVisitor's unsupported-predicate error BindVisitor#visitPredicate interpolated `this`, the visitor, rather than the predicate into its IllegalArgumentException, and omitted the space before "cannot". Since BindVisitor has no toString() override the message read "The expression org.apache.hudi.common.expression.BindVisitor@41a90fa8cannot be visited as predicate". The sibling PartialBindVisitor#visitPredicate already interpolates the predicate correctly. StringStartsWithAny is the only Predicate that can reach that guard, and the only class in Predicates.java without a toString() override, so the corrected message would still have printed an identity hash. Added a null-safe toString() to it, since HoodieBackedTableMetadata constructs it with a null left operand. Added TestBindVisitor covering the error message, and two TestPredicates cases covering the new toString(), including the null left operand shape. Closes #19240 * addressed review comments: fix toString rendering in Predicates Drop the prefix and suffix from Collectors.joining in StringStartsWithAny.toString() so the rendering matches the method-call shape of the sibling StringStartsWith.toString(), and scope its null-safety comment to the left operand, which is the only one HoodieBackedTableMetadata passes as null. Correct the startWith typo in StringStartsWith.toString() and pin the rendering with a test. --------- Co-authored-by: Vova Kolmakov (cherry picked from commit bc874774af1759c16d7ff42781c842485352cd03) --- .../apache/hudi/expression/BindVisitor.java | 2 +- .../apache/hudi/expression/Predicates.java | 10 +++- .../hudi/expression/TestBindVisitor.java | 53 +++++++++++++++++++ .../hudi/expression/TestPredicates.java | 21 ++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/expression/TestBindVisitor.java diff --git a/hudi-common/src/main/java/org/apache/hudi/expression/BindVisitor.java b/hudi-common/src/main/java/org/apache/hudi/expression/BindVisitor.java index 2b7e589af2138..5d4ea2fd52357 100644 --- a/hudi-common/src/main/java/org/apache/hudi/expression/BindVisitor.java +++ b/hudi-common/src/main/java/org/apache/hudi/expression/BindVisitor.java @@ -182,6 +182,6 @@ public Expression visitPredicate(Predicate predicate) { return Predicates.contains(left, right); } - throw new IllegalArgumentException("The expression " + this + "cannot be visited as predicate"); + throw new IllegalArgumentException("The expression " + predicate + " cannot be visited as predicate"); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java b/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java index e5dd07e2bfe6b..e03522a910a38 100644 --- a/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java +++ b/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java @@ -233,7 +233,7 @@ public static class StringStartsWith extends BinaryExpression implements Predica @Override public String toString() { - return getLeft().toString() + ".startWith(" + getRight().toString() + ")"; + return getLeft().toString() + ".startsWith(" + getRight().toString() + ")"; } @Override @@ -449,6 +449,14 @@ public Object eval(StructLike data) { return false; } + @Override + public String toString() { + // Only the left expression is absent: HoodieBackedTableMetadata passes null for the metadata table + // key filters, so it must not be dereferenced. The right operands are always non-null literals. + return left + ".startsWithAny(" + + right.stream().map(Expression::toString).collect(Collectors.joining(",")) + ")"; + } + public List getRightChildren() { return right; } diff --git a/hudi-common/src/test/java/org/apache/hudi/expression/TestBindVisitor.java b/hudi-common/src/test/java/org/apache/hudi/expression/TestBindVisitor.java new file mode 100644 index 0000000000000..6ef4644224926 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/expression/TestBindVisitor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.expression; + +import org.apache.hudi.internal.schema.Types; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestBindVisitor { + + private static Types.RecordType schema() { + ArrayList fields = new ArrayList<>(1); + fields.add(Types.Field.get(0, true, "a", Types.StringType.get())); + return Types.RecordType.get(fields, "schema"); + } + + @Test + void testUnsupportedPredicateErrorNamesTheExpression() { + BindVisitor bindVisitor = new BindVisitor(schema(), true); + Predicates.StringStartsWithAny startsWithAny = + Predicates.startsWithAny(new NameReference("a"), Collections.singletonList(Literal.from("Ja"))); + + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> startsWithAny.accept(bindVisitor)); + + assertTrue(e.getMessage().contains("NameReference(name=a).startsWithAny(Ja)"), e::getMessage); + assertFalse(e.getMessage().contains(BindVisitor.class.getName()), e::getMessage); + assertTrue(e.getMessage().contains(" cannot be visited as predicate"), e::getMessage); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/expression/TestPredicates.java b/hudi-common/src/test/java/org/apache/hudi/expression/TestPredicates.java index e2a83872cdd49..85481fdfd7140 100644 --- a/hudi-common/src/test/java/org/apache/hudi/expression/TestPredicates.java +++ b/hudi-common/src/test/java/org/apache/hudi/expression/TestPredicates.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -29,6 +30,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class TestPredicates { + @Test + void testStringStartsWithToString() { + Predicates.StringStartsWith predicate = Predicates.startsWith(Literal.from("key"), Literal.from("k1")); + assertEquals("key.startsWith(k1)", predicate.toString()); + } + @Test void testStringStartsWithAnyWhenMatched() { Expression left = Literal.from("key2_any"); @@ -52,4 +59,18 @@ void testStringStartsWithAnyWhenNotMatched() { assertEquals(Expression.Operator.STARTS_WITH, predicate.getOperator()); assertFalse((boolean) predicate.eval(null)); } + + @Test + void testStringStartsWithAnyToString() { + Predicates.StringStartsWithAny predicate = + Predicates.startsWithAny(Literal.from("key"), Arrays.asList(Literal.from("k1"), Literal.from("k2"))); + assertEquals("key.startsWithAny(k1,k2)", predicate.toString()); + } + + @Test + void testStringStartsWithAnyToStringIsNullSafeForAbsentLeft() { + Predicates.StringStartsWithAny predicate = + Predicates.startsWithAny(null, Collections.singletonList(Literal.from("key1"))); + assertEquals("null.startsWithAny(key1)", predicate.toString()); + } } From 51ebdf91257c1778c17e5d806fca967baa24445c Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Fri, 10 Jul 2026 06:32:00 -0700 Subject: [PATCH 151/255] refactor(spark): consolidate the vendored 3.x Avro serde forks into hudi-spark3-common (#19168) * refactor(spark): consolidate the vendored 3.x Avro serde forks into hudi-spark3-common The vendored spark-avro AvroSerializer and AvroDeserializer are duplicated across hudi-spark3.3.x, hudi-spark3.4.x and hudi-spark3.5.x. The 3.3 and 3.4 copies are byte-identical; the 3.5 copies differ only in how they import LegacyBehaviorPolicy. Move a single copy of each into hudi-spark3-common, which every 3.x version module already depends on, and delete the six duplicates. The one cross-version snag is the LegacyBehaviorPolicy enum: it is nested in SQLConf (org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy) on Spark 3.3/3.4 but a top-level object (org.apache.spark.sql.internal.LegacyBehaviorPolicy) on Spark 3.5, so no single explicit import resolves on all three (verified by compiling against the 3.3.4/3.4.3/3.5.5 catalyst jars). The shared source imports both containers via wildcards; exactly one of them contributes the enum on any given version, so there is no ambiguity and the bodies stay byte-identical to the originals. The per-version HoodieSpark3_xAvro{Serializer, Deserializer} wrappers are unchanged, since the shared classes keep the same constructors they called. Consolidating also gives each file a unique repo path, so it re-enters the coverage denominator: the identically-pathed copies were dropped by report-path resolution and were invisible to Codecov. * refactor(spark): consolidate the vendored 4.x Avro serde forks into hudi-spark4-common The vendored spark-avro AvroSerializer and AvroDeserializer are duplicated across hudi-spark4.0.x, hudi-spark4.1.x and hudi-spark4.2.x. AvroSerializer: the 4.0/4.1/4.2 bodies are identical except for the private convenience constructor that reads AVRO_REBASE_MODE_IN_WRITE from SQLConf. On Spark 4.0 that ConfigEntry is typed as String (so the read wraps it in LegacyBehaviorPolicy.withName), while on 4.1+ it is already a LegacyBehaviorPolicy.Value; a single shared source cannot express both. Since the only callers are the per-version HoodieSpark4_xAvroSerializer wrappers, move that read into each wrapper (verbatim per version) and drop the convenience constructor, leaving one shared AvroSerializer in hudi-spark4-common that all three versions use. AvroDeserializer: 4.1 and 4.2 are identical apart from one comment word, so they collapse to a single shared copy in hudi-spark4-common. Spark 4.0 is kept separate: it pulls in Avro 1.12.0 (fast reader off) and lacks the ~71-line read-side java.time normalization that 4.1+ need for Avro 1.12.1, and that block does not parameterize cleanly. Because hudi-spark4.0.x depends on hudi-spark4-common, a same-named copy would collide on the classpath, so the 4.0-only copy is renamed to Spark40AvroDeserializer (body unchanged). Consolidating also gives each file a unique repo path, so it re-enters the coverage denominator: the identically-pathed copies were dropped by report-path resolution and were invisible to Codecov. * Revert the 4.x Avro serde consolidation (unsound: vendored fork must shadow spark-sql per-module) Reverts commit 5a8169f05bbe. On Spark 4.x the avro connector is merged into the spark-sql artifact, so org.apache.spark.sql.avro.{AvroDeserializer, AvroSerializer,AvroUtils} ship inside spark-sql_2.13 itself (verified in the 4.0.2/4.1.1/4.2.0-preview4 jars). spark-sql is a provided dependency of every 4.x version module, so Spark's own AvroDeserializer/AvroSerializer are always on the 4.x compile classpath. The vendored fork shares Spark's FQN. While it lived as source in each version module, same-module source shadowed the spark-sql class during that module's compile. Moving it into the hudi-spark4-common jar turned it into a peer dependency-jar class that competes with spark-sql's copy, and spark-sql wins: the HoodieSpark4_xAvro{Deserializer,Serializer} wrappers then resolve the wrong class (compile error in 4.2.x: the 3-arg (Schema, DataType, LegacyBehaviorPolicy.Value) call cannot bind to Spark's constructors). Even where it compiled, the wrapper would silently link Spark's serde on a data path. The 3.x consolidation is kept: spark-sql/spark-catalyst 3.x do not contain the avro serde classes (they live only in the separate spark-avro artifact, which is not a dependency of the 3.x modules), so the vendored fork in hudi-spark3-common is the only such class on the 3.x classpath and resolves correctly. This revert restores the six 4.x serde files to be byte-identical to apache/master. (cherry picked from commit 7b843ae2c35fa00993a2a1b102c9ad7e1e116267) --- .../spark/sql/avro/AvroDeserializer.scala | 7 +- .../spark/sql/avro/AvroSerializer.scala | 9 +- .../spark/sql/avro/AvroDeserializer.scala | 531 ------------------ .../spark/sql/avro/AvroSerializer.scala | 490 ---------------- .../spark/sql/avro/AvroDeserializer.scala | 531 ------------------ .../spark/sql/avro/AvroSerializer.scala | 489 ---------------- 6 files changed, 13 insertions(+), 2044 deletions(-) rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala (97%) rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala (97%) delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala similarity index 97% rename from hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala index b5eba6be24cd3..2a9244508182c 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala @@ -33,7 +33,12 @@ import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArr import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime} import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy +// LegacyBehaviorPolicy is nested in SQLConf on Spark 3.3/3.4 (org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy) +// but a top-level object on Spark 3.5 (org.apache.spark.sql.internal.LegacyBehaviorPolicy). Importing both +// containers via wildcards lets this single shared source resolve the enum on every 3.x version: exactly one of +// the two wildcards contributes LegacyBehaviorPolicy on any given version, so there is no ambiguity. +import org.apache.spark.sql.internal._ +import org.apache.spark.sql.internal.SQLConf._ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala similarity index 97% rename from hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala index a1241b72e58bc..a432eec0ffedb 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala @@ -34,8 +34,13 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow} import org.apache.spark.sql.catalyst.util.{DateTimeUtils, RebaseDateTime} import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy +// LegacyBehaviorPolicy is nested in SQLConf on Spark 3.3/3.4 (org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy) +// but a top-level object on Spark 3.5 (org.apache.spark.sql.internal.LegacyBehaviorPolicy). Importing both +// containers via wildcards lets this single shared source resolve the enum on every 3.x version: exactly one of +// the two wildcards contributes LegacyBehaviorPolicy on any given version, so there is no ambiguity. The same two +// wildcards also keep the SQLConf object in scope for the SQLConf.get / SQLConf.AVRO_REBASE_MODE_IN_WRITE usages. +import org.apache.spark.sql.internal._ +import org.apache.spark.sql.internal.SQLConf._ import org.apache.spark.sql.types._ import java.nio.ByteBuffer diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala deleted file mode 100644 index b5eba6be24cd3..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala +++ /dev/null @@ -1,531 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.hudi.common.schema.HoodieSchema -import org.apache.hudi.common.schema.HoodieSchema.VectorLogicalType - -import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder} -import org.apache.avro.Conversions.DecimalConversion -import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis} -import org.apache.avro.Schema.Type._ -import org.apache.avro.generic._ -import org.apache.avro.util.Utf8 -import org.apache.spark.sql.avro.AvroDeserializer.{createDateRebaseFuncInRead, createTimestampRebaseFuncInRead, RebaseSpec} -import org.apache.spark.sql.avro.AvroUtils.{toFieldStr, AvroMatchedField} -import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters} -import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData} -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime} -import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY -import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -import java.math.BigDecimal -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.TimeZone - -import scala.collection.JavaConverters._ - -/** - * A deserializer to deserialize data in avro format to data in catalyst format. - * - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] class AvroDeserializer(rootAvroType: Schema, - rootCatalystType: DataType, - positionalFieldMatch: Boolean, - datetimeRebaseSpec: RebaseSpec, - filters: StructFilters) { - - def this(rootAvroType: Schema, - rootCatalystType: DataType, - datetimeRebaseMode: String) = { - this( - rootAvroType, - rootCatalystType, - positionalFieldMatch = false, - RebaseSpec(LegacyBehaviorPolicy.withName(datetimeRebaseMode)), - new NoopFilters) - } - - private lazy val decimalConversions = new DecimalConversion() - - private val dateRebaseFunc = createDateRebaseFuncInRead(datetimeRebaseSpec.mode, "Avro") - - private val timestampRebaseFunc = createTimestampRebaseFuncInRead(datetimeRebaseSpec, "Avro") - - private val converter: Any => Option[Any] = try { - rootCatalystType match { - // A shortcut for empty schema. - case st: StructType if st.isEmpty => - (_: Any) => Some(InternalRow.empty) - - case st: StructType => - val resultRow = new SpecificInternalRow(st.map(_.dataType)) - val fieldUpdater = new RowUpdater(resultRow) - val applyFilters = filters.skipRow(resultRow, _) - val writer = getRecordWriter(rootAvroType, st, Nil, Nil, applyFilters) - (data: Any) => { - val record = data.asInstanceOf[GenericRecord] - val skipRow = writer(fieldUpdater, record) - if (skipRow) None else Some(resultRow) - } - - case _ => - val tmpRow = new SpecificInternalRow(Seq(rootCatalystType)) - val fieldUpdater = new RowUpdater(tmpRow) - val writer = newWriter(rootAvroType, rootCatalystType, Nil, Nil) - (data: Any) => { - writer(fieldUpdater, 0, data) - Some(tmpRow.get(0, rootCatalystType)) - } - } - } catch { - case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException( - s"Cannot convert Avro type $rootAvroType to SQL type ${rootCatalystType.sql}.", ise) - } - - def deserialize(data: Any): Option[Any] = converter(data) - - /** - * Creates a writer to write avro values to Catalyst values at the given ordinal with the given - * updater. - */ - private def newWriter(avroType: Schema, - catalystType: DataType, - avroPath: Seq[String], - catalystPath: Seq[String]): (CatalystDataUpdater, Int, Any) => Unit = { - val errorPrefix = s"Cannot convert Avro ${toFieldStr(avroPath)} to " + - s"SQL ${toFieldStr(catalystPath)} because " - val incompatibleMsg = errorPrefix + - s"schema is incompatible (avroType = $avroType, sqlType = ${catalystType.sql})" - - (avroType.getType, catalystType) match { - case (NULL, NullType) => (updater, ordinal, _) => - updater.setNullAt(ordinal) - - // TODO: we can avoid boxing if future version of avro provide primitive accessors. - case (BOOLEAN, BooleanType) => (updater, ordinal, value) => - updater.setBoolean(ordinal, value.asInstanceOf[Boolean]) - - case (INT, IntegerType) => (updater, ordinal, value) => - updater.setInt(ordinal, value.asInstanceOf[Int]) - - case (INT, DateType) => (updater, ordinal, value) => - updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int])) - - case (LONG, LongType) => (updater, ordinal, value) => - updater.setLong(ordinal, value.asInstanceOf[Long]) - - case (LONG, TimestampType) => avroType.getLogicalType match { - // For backward compatibility, if the Avro type is Long and it is not logical type - // (the `null` case), the value is processed as timestamp type with millisecond precision. - case null | _: TimestampMillis => (updater, ordinal, value) => - val millis = value.asInstanceOf[Long] - val micros = DateTimeUtils.millisToMicros(millis) - updater.setLong(ordinal, timestampRebaseFunc(micros)) - case _: TimestampMicros => (updater, ordinal, value) => - val micros = value.asInstanceOf[Long] - updater.setLong(ordinal, timestampRebaseFunc(micros)) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"Avro logical type $other cannot be converted to SQL type ${TimestampType.sql}.") - } - - case (LONG, TimestampNTZType) => avroType.getLogicalType match { - // To keep consistent with TimestampType, if the Avro type is Long and it is not - // logical type (the `null` case), the value is processed as TimestampNTZ - // with millisecond precision. - case null | _: LocalTimestampMillis => (updater, ordinal, value) => - val millis = value.asInstanceOf[Long] - val micros = DateTimeUtils.millisToMicros(millis) - updater.setLong(ordinal, micros) - case _: LocalTimestampMicros => (updater, ordinal, value) => - val micros = value.asInstanceOf[Long] - updater.setLong(ordinal, micros) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"Avro logical type $other cannot be converted to SQL type ${TimestampNTZType.sql}.") - } - - // Handle VECTOR logical type (FLOAT, DOUBLE, INT8) - case (FIXED, ArrayType(elementType, false)) => avroType.getLogicalType match { - case vectorLogicalType: VectorLogicalType => - val dimension = vectorLogicalType.getDimension - val vecElementType = HoodieSchema.Vector.VectorElementType.fromString(vectorLogicalType.getElementType) - val elementSize = vecElementType.getElementSize - (updater, ordinal, value) => { - val bytes = value.asInstanceOf[GenericData.Fixed].bytes() - val expectedSize = Math.multiplyExact(dimension, elementSize) - if (bytes.length != expectedSize) { - throw new IncompatibleSchemaException( - s"VECTOR byte size mismatch: expected=$expectedSize, actual=${bytes.length}") - } - elementType match { - case FloatType => - val buffer = ByteBuffer.wrap(bytes).order(VectorLogicalType.VECTOR_BYTE_ORDER) - val floats = new Array[Float](dimension) - var i = 0; while (i < dimension) { floats(i) = buffer.getFloat(); i += 1 } - updater.set(ordinal, ArrayData.toArrayData(floats)) - case DoubleType => - val buffer = ByteBuffer.wrap(bytes).order(VectorLogicalType.VECTOR_BYTE_ORDER) - val doubles = new Array[Double](dimension) - var i = 0; while (i < dimension) { doubles(i) = buffer.getDouble(); i += 1 } - updater.set(ordinal, ArrayData.toArrayData(doubles)) - case ByteType => - updater.set(ordinal, ArrayData.toArrayData(bytes.clone())) - } - } - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - - // Before we upgrade Avro to 1.8 for logical type support, spark-avro converts Long to Date. - // For backward compatibility, we still keep this conversion. - case (LONG, DateType) => (updater, ordinal, value) => - updater.setInt(ordinal, (value.asInstanceOf[Long] / MILLIS_PER_DAY).toInt) - - case (FLOAT, FloatType) => (updater, ordinal, value) => - updater.setFloat(ordinal, value.asInstanceOf[Float]) - - case (DOUBLE, DoubleType) => (updater, ordinal, value) => - updater.setDouble(ordinal, value.asInstanceOf[Double]) - - case (STRING, StringType) => (updater, ordinal, value) => - val str = value match { - case s: String => UTF8String.fromString(s) - case s: Utf8 => - val bytes = new Array[Byte](s.getByteLength) - System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength) - UTF8String.fromBytes(bytes) - case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString) - } - updater.set(ordinal, str) - - case (ENUM, StringType) => (updater, ordinal, value) => - updater.set(ordinal, UTF8String.fromString(value.toString)) - - case (FIXED, BinaryType) => (updater, ordinal, value) => - updater.set(ordinal, value.asInstanceOf[GenericFixed].bytes().clone()) - - case (BYTES, BinaryType) => (updater, ordinal, value) => - val bytes = value match { - case b: ByteBuffer => - val bytes = new Array[Byte](b.remaining) - b.get(bytes) - // Do not forget to reset the position - b.rewind() - bytes - case b: Array[Byte] => b - case other => - throw new RuntimeException(errorPrefix + s"$other is not a valid avro binary.") - } - updater.set(ordinal, bytes) - - case (FIXED, _: DecimalType) => (updater, ordinal, value) => - val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal] - val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d) - val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale) - updater.setDecimal(ordinal, decimal) - - case (BYTES, _: DecimalType) => (updater, ordinal, value) => - val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal] - val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d) - val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale) - updater.setDecimal(ordinal, decimal) - - case (RECORD, st: StructType) => - // Avro datasource doesn't accept filters with nested attributes. See SPARK-32328. - // We can always return `false` from `applyFilters` for nested records. - val writeRecord = - getRecordWriter(avroType, st, avroPath, catalystPath, applyFilters = _ => false) - (updater, ordinal, value) => - val row = new SpecificInternalRow(st) - writeRecord(new RowUpdater(row), value.asInstanceOf[GenericRecord]) - updater.set(ordinal, row) - - case (ARRAY, ArrayType(elementType, containsNull)) => - val avroElementPath = avroPath :+ "element" - val elementWriter = newWriter(avroType.getElementType, elementType, - avroElementPath, catalystPath :+ "element") - (updater, ordinal, value) => - val collection = value.asInstanceOf[java.util.Collection[Any]] - val result = createArrayData(elementType, collection.size()) - val elementUpdater = new ArrayDataUpdater(result) - - var i = 0 - val iter = collection.iterator() - while (iter.hasNext) { - val element = iter.next() - if (element == null) { - if (!containsNull) { - throw new RuntimeException( - s"Array value at path ${toFieldStr(avroElementPath)} is not allowed to be null") - } else { - elementUpdater.setNullAt(i) - } - } else { - elementWriter(elementUpdater, i, element) - } - i += 1 - } - - updater.set(ordinal, result) - - case (MAP, MapType(keyType, valueType, valueContainsNull)) if keyType == StringType => - val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType, - avroPath :+ "key", catalystPath :+ "key") - val valueWriter = newWriter(avroType.getValueType, valueType, - avroPath :+ "value", catalystPath :+ "value") - (updater, ordinal, value) => - val map = value.asInstanceOf[java.util.Map[AnyRef, AnyRef]] - val keyArray = createArrayData(keyType, map.size()) - val keyUpdater = new ArrayDataUpdater(keyArray) - val valueArray = createArrayData(valueType, map.size()) - val valueUpdater = new ArrayDataUpdater(valueArray) - val iter = map.entrySet().iterator() - var i = 0 - while (iter.hasNext) { - val entry = iter.next() - assert(entry.getKey != null) - keyWriter(keyUpdater, i, entry.getKey) - if (entry.getValue == null) { - if (!valueContainsNull) { - throw new RuntimeException( - s"Map value at path ${toFieldStr(avroPath :+ "value")} is not allowed to be null") - } else { - valueUpdater.setNullAt(i) - } - } else { - valueWriter(valueUpdater, i, entry.getValue) - } - i += 1 - } - - // The Avro map will never have null or duplicated map keys, it's safe to create a - // ArrayBasedMapData directly here. - updater.set(ordinal, new ArrayBasedMapData(keyArray, valueArray)) - - case (UNION, _) => - val allTypes = avroType.getTypes.asScala - val nonNullTypes = allTypes.filter(_.getType != NULL) - val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava) - if (nonNullTypes.nonEmpty) { - if (nonNullTypes.length == 1) { - newWriter(nonNullTypes.head, catalystType, avroPath, catalystPath) - } else { - nonNullTypes.map(_.getType).toSeq match { - case Seq(a, b) if Set(a, b) == Set(INT, LONG) && catalystType == LongType => - (updater, ordinal, value) => value match { - case null => updater.setNullAt(ordinal) - case l: java.lang.Long => updater.setLong(ordinal, l) - case i: java.lang.Integer => updater.setLong(ordinal, i.longValue()) - } - - case Seq(a, b) if Set(a, b) == Set(FLOAT, DOUBLE) && catalystType == DoubleType => - (updater, ordinal, value) => value match { - case null => updater.setNullAt(ordinal) - case d: java.lang.Double => updater.setDouble(ordinal, d) - case f: java.lang.Float => updater.setDouble(ordinal, f.doubleValue()) - } - - case _ => - catalystType match { - case st: StructType if st.length == nonNullTypes.size => - val fieldWriters = nonNullTypes.zip(st.fields).map { - case (schema, field) => - newWriter(schema, field.dataType, avroPath, catalystPath :+ field.name) - }.toArray - (updater, ordinal, value) => { - val row = new SpecificInternalRow(st) - val fieldUpdater = new RowUpdater(row) - val i = GenericData.get().resolveUnion(nonNullAvroType, value) - fieldWriters(i)(fieldUpdater, i, value) - updater.set(ordinal, row) - } - - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - } - } - } else { - (updater, ordinal, _) => updater.setNullAt(ordinal) - } - - case (INT, _: YearMonthIntervalType) => (updater, ordinal, value) => - updater.setInt(ordinal, value.asInstanceOf[Int]) - - case (LONG, _: DayTimeIntervalType) => (updater, ordinal, value) => - updater.setLong(ordinal, value.asInstanceOf[Long]) - - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - } - - // TODO: move the following method in Decimal object on creating Decimal from BigDecimal? - private def createDecimal(decimal: BigDecimal, precision: Int, scale: Int): Decimal = { - if (precision <= Decimal.MAX_LONG_DIGITS) { - // Constructs a `Decimal` with an unscaled `Long` value if possible. - Decimal(decimal.unscaledValue().longValue(), precision, scale) - } else { - // Otherwise, resorts to an unscaled `BigInteger` instead. - Decimal(decimal, precision, scale) - } - } - - private def getRecordWriter( - avroType: Schema, - catalystType: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - applyFilters: Int => Boolean): (CatalystDataUpdater, GenericRecord) => Boolean = { - - val avroSchemaHelper = new AvroUtils.AvroSchemaHelper( - avroType, catalystType, avroPath, catalystPath, positionalFieldMatch) - - avroSchemaHelper.validateNoExtraCatalystFields(ignoreNullable = true) - // no need to validateNoExtraAvroFields since extra Avro fields are ignored - - val (validFieldIndexes, fieldWriters) = avroSchemaHelper.matchedFields.map { - case AvroMatchedField(catalystField, ordinal, avroField) => - val baseWriter = newWriter(avroField.schema(), catalystField.dataType, - avroPath :+ avroField.name, catalystPath :+ catalystField.name) - val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => { - if (value == null) { - fieldUpdater.setNullAt(ordinal) - } else { - baseWriter(fieldUpdater, ordinal, value) - } - } - (avroField.pos(), fieldWriter) - }.toArray.unzip - - (fieldUpdater, record) => { - var i = 0 - var skipRow = false - while (i < validFieldIndexes.length && !skipRow) { - fieldWriters(i)(fieldUpdater, record.get(validFieldIndexes(i))) - skipRow = applyFilters(i) - i += 1 - } - skipRow - } - } - - private def createArrayData(elementType: DataType, length: Int): ArrayData = elementType match { - case BooleanType => UnsafeArrayData.fromPrimitiveArray(new Array[Boolean](length)) - case ByteType => UnsafeArrayData.fromPrimitiveArray(new Array[Byte](length)) - case ShortType => UnsafeArrayData.fromPrimitiveArray(new Array[Short](length)) - case IntegerType => UnsafeArrayData.fromPrimitiveArray(new Array[Int](length)) - case LongType => UnsafeArrayData.fromPrimitiveArray(new Array[Long](length)) - case FloatType => UnsafeArrayData.fromPrimitiveArray(new Array[Float](length)) - case DoubleType => UnsafeArrayData.fromPrimitiveArray(new Array[Double](length)) - case _ => new GenericArrayData(new Array[Any](length)) - } - - /** - * A base interface for updating values inside catalyst data structure like `InternalRow` and - * `ArrayData`. - */ - sealed trait CatalystDataUpdater { - def set(ordinal: Int, value: Any): Unit - - def setNullAt(ordinal: Int): Unit = set(ordinal, null) - def setBoolean(ordinal: Int, value: Boolean): Unit = set(ordinal, value) - def setByte(ordinal: Int, value: Byte): Unit = set(ordinal, value) - def setShort(ordinal: Int, value: Short): Unit = set(ordinal, value) - def setInt(ordinal: Int, value: Int): Unit = set(ordinal, value) - def setLong(ordinal: Int, value: Long): Unit = set(ordinal, value) - def setDouble(ordinal: Int, value: Double): Unit = set(ordinal, value) - def setFloat(ordinal: Int, value: Float): Unit = set(ordinal, value) - def setDecimal(ordinal: Int, value: Decimal): Unit = set(ordinal, value) - } - - final class RowUpdater(row: InternalRow) extends CatalystDataUpdater { - override def set(ordinal: Int, value: Any): Unit = row.update(ordinal, value) - - override def setNullAt(ordinal: Int): Unit = row.setNullAt(ordinal) - override def setBoolean(ordinal: Int, value: Boolean): Unit = row.setBoolean(ordinal, value) - override def setByte(ordinal: Int, value: Byte): Unit = row.setByte(ordinal, value) - override def setShort(ordinal: Int, value: Short): Unit = row.setShort(ordinal, value) - override def setInt(ordinal: Int, value: Int): Unit = row.setInt(ordinal, value) - override def setLong(ordinal: Int, value: Long): Unit = row.setLong(ordinal, value) - override def setDouble(ordinal: Int, value: Double): Unit = row.setDouble(ordinal, value) - override def setFloat(ordinal: Int, value: Float): Unit = row.setFloat(ordinal, value) - override def setDecimal(ordinal: Int, value: Decimal): Unit = - row.setDecimal(ordinal, value, value.precision) - } - - final class ArrayDataUpdater(array: ArrayData) extends CatalystDataUpdater { - override def set(ordinal: Int, value: Any): Unit = array.update(ordinal, value) - - override def setNullAt(ordinal: Int): Unit = array.setNullAt(ordinal) - override def setBoolean(ordinal: Int, value: Boolean): Unit = array.setBoolean(ordinal, value) - override def setByte(ordinal: Int, value: Byte): Unit = array.setByte(ordinal, value) - override def setShort(ordinal: Int, value: Short): Unit = array.setShort(ordinal, value) - override def setInt(ordinal: Int, value: Int): Unit = array.setInt(ordinal, value) - override def setLong(ordinal: Int, value: Long): Unit = array.setLong(ordinal, value) - override def setDouble(ordinal: Int, value: Double): Unit = array.setDouble(ordinal, value) - override def setFloat(ordinal: Int, value: Float): Unit = array.setFloat(ordinal, value) - override def setDecimal(ordinal: Int, value: Decimal): Unit = array.update(ordinal, value) - } -} - -object AvroDeserializer { - - // NOTE: Following methods have been renamed in Spark 3.2.1 [1] making [[AvroDeserializer]] implementation - // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]]. - // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.2.x branch, - // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.2.0 as well as - // w/ Spark >= 3.2.1 - // - // [1] https://github.com/apache/spark/pull/34978 - - // Specification of rebase operation including `mode` and the time zone in which it is performed - case class RebaseSpec(mode: LegacyBehaviorPolicy.Value, originTimeZone: Option[String] = None) { - // Use the default JVM time zone for backward compatibility - def timeZone: String = originTimeZone.getOrElse(TimeZone.getDefault.getID) - } - - def createDateRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Int => Int = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => days: Int => - if (days < RebaseDateTime.lastSwitchJulianDay) { - throw DataSourceUtils.newRebaseExceptionInRead(format) - } - days - case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays - case LegacyBehaviorPolicy.CORRECTED => identity[Int] - } - - def createTimestampRebaseFuncInRead(rebaseSpec: RebaseSpec, - format: String): Long => Long = rebaseSpec.mode match { - case LegacyBehaviorPolicy.EXCEPTION => micros: Long => - if (micros < RebaseDateTime.lastSwitchJulianTs) { - throw DataSourceUtils.newRebaseExceptionInRead(format) - } - micros - case LegacyBehaviorPolicy.LEGACY => micros: Long => - RebaseDateTime.rebaseJulianToGregorianMicros(TimeZone.getTimeZone(rebaseSpec.timeZone), micros) - case LegacyBehaviorPolicy.CORRECTED => identity[Long] - } -} diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala deleted file mode 100644 index a1241b72e58bc..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.hudi.common.schema.HoodieSchema -import org.apache.hudi.common.schema.HoodieSchema.VectorLogicalType - -import org.apache.avro.{LogicalTypes, Schema} -import org.apache.avro.Conversions.DecimalConversion -import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis} -import org.apache.avro.Schema.Type -import org.apache.avro.Schema.Type._ -import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record} -import org.apache.avro.util.Utf8 -import org.apache.spark.internal.Logging -import org.apache.spark.sql.avro.AvroSerializer.{createDateRebaseFuncInWrite, createTimestampRebaseFuncInWrite} -import org.apache.spark.sql.avro.AvroUtils.{toFieldStr, AvroMatchedField} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow} -import org.apache.spark.sql.catalyst.util.{DateTimeUtils, RebaseDateTime} -import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.sql.types._ - -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.TimeZone - -import scala.collection.JavaConverters._ - -/** - * A serializer to serialize data in catalyst format to data in avro format. - * - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * NOTE: THIS IMPLEMENTATION HAS BEEN MODIFIED FROM ITS ORIGINAL VERSION WITH THE MODIFICATION - * BEING EXPLICITLY ANNOTATED INLINE. PLEASE MAKE SURE TO UNDERSTAND PROPERLY ALL THE - * MODIFICATIONS. - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] class AvroSerializer(rootCatalystType: DataType, - rootAvroType: Schema, - nullable: Boolean, - positionalFieldMatch: Boolean, - datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging { - - def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) = { - this(rootCatalystType, rootAvroType, nullable, positionalFieldMatch = false, - LegacyBehaviorPolicy.withName(SQLConf.get.getConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE, - LegacyBehaviorPolicy.CORRECTED.toString))) - } - - def serialize(catalystData: Any): Any = { - converter.apply(catalystData) - } - - private val dateRebaseFunc = createDateRebaseFuncInWrite( - datetimeRebaseMode, "Avro") - - private val timestampRebaseFunc = createTimestampRebaseFuncInWrite( - datetimeRebaseMode, "Avro") - - private val converter: Any => Any = { - val actualAvroType = resolveNullableType(rootAvroType, nullable) - val baseConverter = try { - rootCatalystType match { - case st: StructType => - newStructConverter(st, actualAvroType, Nil, Nil).asInstanceOf[Any => Any] - case _ => - val tmpRow = new SpecificInternalRow(Seq(rootCatalystType)) - val converter = newConverter(rootCatalystType, actualAvroType, Nil, Nil) - (data: Any) => - tmpRow.update(0, data) - converter.apply(tmpRow, 0) - } - } catch { - case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException( - s"Cannot convert SQL type ${rootCatalystType.sql} to Avro type $rootAvroType.", ise) - } - if (nullable) { - (data: Any) => - if (data == null) { - null - } else { - baseConverter.apply(data) - } - } else { - baseConverter - } - } - - private type Converter = (SpecializedGetters, Int) => Any - - private lazy val decimalConversions = new DecimalConversion() - - private def newConverter(catalystType: DataType, - avroType: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): Converter = { - val errorPrefix = s"Cannot convert SQL ${toFieldStr(catalystPath)} " + - s"to Avro ${toFieldStr(avroPath)} because " - (catalystType, avroType.getType) match { - case (NullType, NULL) => - (getter, ordinal) => null - case (BooleanType, BOOLEAN) => - (getter, ordinal) => getter.getBoolean(ordinal) - case (ByteType, INT) => - (getter, ordinal) => getter.getByte(ordinal).toInt - case (ShortType, INT) => - (getter, ordinal) => getter.getShort(ordinal).toInt - case (IntegerType, INT) => - (getter, ordinal) => getter.getInt(ordinal) - case (LongType, LONG) => - (getter, ordinal) => getter.getLong(ordinal) - case (FloatType, FLOAT) => - (getter, ordinal) => getter.getFloat(ordinal) - case (DoubleType, DOUBLE) => - (getter, ordinal) => getter.getDouble(ordinal) - case (d: DecimalType, FIXED) - if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) => - (getter, ordinal) => - val decimal = getter.getDecimal(ordinal, d.precision, d.scale) - decimalConversions.toFixed(decimal.toJavaBigDecimal, avroType, - LogicalTypes.decimal(d.precision, d.scale)) - - case (d: DecimalType, BYTES) - if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) => - (getter, ordinal) => - val decimal = getter.getDecimal(ordinal, d.precision, d.scale) - decimalConversions.toBytes(decimal.toJavaBigDecimal, avroType, - LogicalTypes.decimal(d.precision, d.scale)) - - // Handle VECTOR logical type (FLOAT, DOUBLE, INT8) - case (ArrayType(elementType, false), FIXED) => avroType.getLogicalType match { - case vectorLogicalType: VectorLogicalType => - val dimension = vectorLogicalType.getDimension - val vecElementType = HoodieSchema.Vector.VectorElementType.fromString(vectorLogicalType.getElementType) - val bufferSize = Math.multiplyExact(dimension, vecElementType.getElementSize) - (getter, ordinal) => { - val arrayData = getter.getArray(ordinal) - if (arrayData.numElements() != dimension) { - throw new IncompatibleSchemaException( - s"VECTOR dimension mismatch at ${toFieldStr(catalystPath)}: " + - s"expected=$dimension, actual=${arrayData.numElements()}") - } - elementType match { - case FloatType => - val buffer = ByteBuffer.allocate(bufferSize).order(VectorLogicalType.VECTOR_BYTE_ORDER) - var i = 0; while (i < dimension) { buffer.putFloat(arrayData.getFloat(i)); i += 1 } - new Fixed(avroType, buffer.array()) - case DoubleType => - val buffer = ByteBuffer.allocate(bufferSize).order(VectorLogicalType.VECTOR_BYTE_ORDER) - var i = 0; while (i < dimension) { buffer.putDouble(arrayData.getDouble(i)); i += 1 } - new Fixed(avroType, buffer.array()) - case ByteType => - val bytes = new Array[Byte](dimension) - var i = 0; while (i < dimension) { bytes(i) = arrayData.getByte(i); i += 1 } - new Fixed(avroType, bytes) - case _ => throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - } - case _ => throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - - case (StringType, ENUM) => - val enumSymbols: Set[String] = avroType.getEnumSymbols.asScala.toSet - (getter, ordinal) => - val data = getter.getUTF8String(ordinal).toString - if (!enumSymbols.contains(data)) { - throw new IncompatibleSchemaException(errorPrefix + - s""""$data" cannot be written since it's not defined in enum """ + - enumSymbols.mkString("\"", "\", \"", "\"")) - } - new EnumSymbol(avroType, data) - - case (StringType, STRING) => - (getter, ordinal) => new Utf8(getter.getUTF8String(ordinal).getBytes) - - case (BinaryType, FIXED) => - val size = avroType.getFixedSize - (getter, ordinal) => - val data: Array[Byte] = getter.getBinary(ordinal) - if (data.length != size) { - def len2str(len: Int): String = s"$len ${if (len > 1) "bytes" else "byte"}" - - throw new IncompatibleSchemaException(errorPrefix + len2str(data.length) + - " of binary data cannot be written into FIXED type with size of " + len2str(size)) - } - new Fixed(avroType, data) - - case (BinaryType, BYTES) => - (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal)) - - case (DateType, INT) => - (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal)) - - case (TimestampType, LONG) => avroType.getLogicalType match { - // For backward compatibility, if the Avro type is Long and it is not logical type - // (the `null` case), output the timestamp value as with millisecond precision. - case null | _: TimestampMillis => (getter, ordinal) => - DateTimeUtils.microsToMillis(timestampRebaseFunc(getter.getLong(ordinal))) - case _: TimestampMicros => (getter, ordinal) => - timestampRebaseFunc(getter.getLong(ordinal)) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"SQL type ${TimestampType.sql} cannot be converted to Avro logical type $other") - } - - case (TimestampNTZType, LONG) => avroType.getLogicalType match { - // To keep consistent with TimestampType, if the Avro type is Long and it is not - // logical type (the `null` case), output the TimestampNTZ as long value - // in millisecond precision. - case null | _: LocalTimestampMillis => (getter, ordinal) => - DateTimeUtils.microsToMillis(getter.getLong(ordinal)) - case _: LocalTimestampMicros => (getter, ordinal) => - getter.getLong(ordinal) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"SQL type ${TimestampNTZType.sql} cannot be converted to Avro logical type $other") - } - - case (ArrayType(et, containsNull), ARRAY) => - val elementConverter = newConverter( - et, resolveNullableType(avroType.getElementType, containsNull), - catalystPath :+ "element", avroPath :+ "element") - (getter, ordinal) => { - val arrayData = getter.getArray(ordinal) - val len = arrayData.numElements() - val result = new Array[Any](len) - var i = 0 - while (i < len) { - if (containsNull && arrayData.isNullAt(i)) { - result(i) = null - } else { - result(i) = elementConverter(arrayData, i) - } - i += 1 - } - // avro writer is expecting a Java Collection, so we convert it into - // `ArrayList` backed by the specified array without data copying. - java.util.Arrays.asList(result: _*) - } - - case (st: StructType, RECORD) => - val structConverter = newStructConverter(st, avroType, catalystPath, avroPath) - val numFields = st.length - (getter, ordinal) => structConverter(getter.getStruct(ordinal, numFields)) - - //////////////////////////////////////////////////////////////////////////////////////////// - // Following section is amended to the original (Spark's) implementation - // >>> BEGINS - //////////////////////////////////////////////////////////////////////////////////////////// - - case (st: StructType, UNION) => - val unionConverter = newUnionConverter(st, avroType, catalystPath, avroPath) - val numFields = st.length - (getter, ordinal) => unionConverter(getter.getStruct(ordinal, numFields)) - - //////////////////////////////////////////////////////////////////////////////////////////// - // <<< ENDS - //////////////////////////////////////////////////////////////////////////////////////////// - - case (MapType(kt, vt, valueContainsNull), MAP) if kt == StringType => - val valueConverter = newConverter( - vt, resolveNullableType(avroType.getValueType, valueContainsNull), - catalystPath :+ "value", avroPath :+ "value") - (getter, ordinal) => - val mapData = getter.getMap(ordinal) - val len = mapData.numElements() - val result = new java.util.HashMap[String, Any](len) - val keyArray = mapData.keyArray() - val valueArray = mapData.valueArray() - var i = 0 - while (i < len) { - val key = keyArray.getUTF8String(i).toString - if (valueContainsNull && valueArray.isNullAt(i)) { - result.put(key, null) - } else { - result.put(key, valueConverter(valueArray, i)) - } - i += 1 - } - result - - case (_: YearMonthIntervalType, INT) => - (getter, ordinal) => getter.getInt(ordinal) - - case (_: DayTimeIntervalType, LONG) => - (getter, ordinal) => getter.getLong(ordinal) - - case _ => - throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - } - - private def newStructConverter(catalystStruct: StructType, - avroStruct: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): InternalRow => Record = { - - val avroSchemaHelper = new AvroUtils.AvroSchemaHelper( - avroStruct, catalystStruct, avroPath, catalystPath, positionalFieldMatch) - - avroSchemaHelper.validateNoExtraCatalystFields(ignoreNullable = false) - avroSchemaHelper.validateNoExtraRequiredAvroFields() - - val (avroIndices, fieldConverters) = avroSchemaHelper.matchedFields.map { - case AvroMatchedField(catalystField, _, avroField) => - val converter = newConverter(catalystField.dataType, - resolveNullableType(avroField.schema(), catalystField.nullable), - catalystPath :+ catalystField.name, avroPath :+ avroField.name) - (avroField.pos(), converter) - }.toArray.unzip - - val numFields = catalystStruct.length - row: InternalRow => - val result = new Record(avroStruct) - var i = 0 - while (i < numFields) { - if (row.isNullAt(i)) { - result.put(avroIndices(i), null) - } else { - result.put(avroIndices(i), fieldConverters(i).apply(row, i)) - } - i += 1 - } - result - } - - //////////////////////////////////////////////////////////////////////////////////////////// - // Following section is amended to the original (Spark's) implementation - // >>> BEGINS - //////////////////////////////////////////////////////////////////////////////////////////// - - private def newUnionConverter(catalystStruct: StructType, - avroUnion: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): InternalRow => Any = { - if (avroUnion.getType != UNION || !canMapUnion(catalystStruct, avroUnion)) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " + - s"Avro type $avroUnion.") - } - val nullable = avroUnion.getTypes.size() > 0 && avroUnion.getTypes.get(0).getType == Type.NULL - val avroInnerTypes = if (nullable) { - avroUnion.getTypes.asScala.tail - } else { - avroUnion.getTypes.asScala - } - val fieldConverters = catalystStruct.zip(avroInnerTypes).map { - case (f1, f2) => newConverter(f1.dataType, f2, catalystPath, avroPath) - } - val numFields = catalystStruct.length - (row: InternalRow) => - var i = 0 - var result: Any = null - while (i < numFields) { - if (!row.isNullAt(i)) { - if (result != null) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " + - s"Avro union $avroUnion. Record has more than one optional values set") - } - result = fieldConverters(i).apply(row, i) - } - i += 1 - } - if (!nullable && result == null) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " + - s"Avro union $avroUnion. Record has no values set, while should have exactly one") - } - result - } - - private def canMapUnion(catalystStruct: StructType, avroStruct: Schema): Boolean = { - (avroStruct.getTypes.size() > 0 && - avroStruct.getTypes.get(0).getType == Type.NULL && - avroStruct.getTypes.size() - 1 == catalystStruct.length) || avroStruct.getTypes.size() == catalystStruct.length - } - - //////////////////////////////////////////////////////////////////////////////////////////// - // <<< ENDS - //////////////////////////////////////////////////////////////////////////////////////////// - - - /** - * Resolve a possibly nullable Avro Type. - * - * An Avro type is nullable when it is a [[UNION]] of two types: one null type and another - * non-null type. This method will check the nullability of the input Avro type and return the - * non-null type within when it is nullable. Otherwise it will return the input Avro type - * unchanged. It will throw an [[UnsupportedAvroTypeException]] when the input Avro type is an - * unsupported nullable type. - * - * It will also log a warning message if the nullability for Avro and catalyst types are - * different. - */ - private def resolveNullableType(avroType: Schema, nullable: Boolean): Schema = { - val (avroNullable, resolvedAvroType) = resolveAvroType(avroType) - warnNullabilityDifference(avroNullable, nullable) - resolvedAvroType - } - - /** - * Check the nullability of the input Avro type and resolve it when it is nullable. The first - * return value is a [[Boolean]] indicating if the input Avro type is nullable. The second - * return value is the possibly resolved type. - */ - private def resolveAvroType(avroType: Schema): (Boolean, Schema) = { - if (avroType.getType == Type.UNION) { - val fields = avroType.getTypes.asScala - val actualType = fields.filter(_.getType != Type.NULL) - if (fields.length == 2 && actualType.length == 1) { - (true, actualType.head) - } else { - // This is just a normal union, not used to designate nullability - (false, avroType) - } - } else { - (false, avroType) - } - } - - /** - * log a warning message if the nullability for Avro and catalyst types are different. - */ - private def warnNullabilityDifference(avroNullable: Boolean, catalystNullable: Boolean): Unit = { - if (avroNullable && !catalystNullable) { - logWarning("Writing Avro files with nullable Avro schema and non-nullable catalyst schema.") - } - if (!avroNullable && catalystNullable) { - logWarning("Writing Avro files with non-nullable Avro schema and nullable catalyst " + - "schema will throw runtime exception if there is a record with null value.") - } - } -} - -object AvroSerializer { - - // NOTE: Following methods have been renamed in Spark 3.2.1 [1] making [[AvroSerializer]] implementation - // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]]. - // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.2.x branch, - // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.2.0 as well as - // w/ Spark >= 3.2.1 - // - // [1] https://github.com/apache/spark/pull/34978 - - def createDateRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Int => Int = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => days: Int => - if (days < RebaseDateTime.lastSwitchGregorianDay) { - throw DataSourceUtils.newRebaseExceptionInWrite(format) - } - days - case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays - case LegacyBehaviorPolicy.CORRECTED => identity[Int] - } - - def createTimestampRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Long => Long = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => micros: Long => - if (micros < RebaseDateTime.lastSwitchGregorianTs) { - throw DataSourceUtils.newRebaseExceptionInWrite(format) - } - micros - case LegacyBehaviorPolicy.LEGACY => - val timeZone = SQLConf.get.sessionLocalTimeZone - RebaseDateTime.rebaseGregorianToJulianMicros(TimeZone.getTimeZone(timeZone), _) - case LegacyBehaviorPolicy.CORRECTED => identity[Long] - } - -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala deleted file mode 100644 index d10ff1edfe879..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala +++ /dev/null @@ -1,531 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.hudi.common.schema.HoodieSchema -import org.apache.hudi.common.schema.HoodieSchema.VectorLogicalType - -import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder} -import org.apache.avro.Conversions.DecimalConversion -import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis} -import org.apache.avro.Schema.Type._ -import org.apache.avro.generic._ -import org.apache.avro.util.Utf8 -import org.apache.spark.sql.avro.AvroDeserializer.{createDateRebaseFuncInRead, createTimestampRebaseFuncInRead, RebaseSpec} -import org.apache.spark.sql.avro.AvroUtils.{toFieldStr, AvroMatchedField} -import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters} -import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData} -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime} -import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY -import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.LegacyBehaviorPolicy -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -import java.math.BigDecimal -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.TimeZone - -import scala.collection.JavaConverters._ - -/** - * A deserializer to deserialize data in avro format to data in catalyst format. - * - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] class AvroDeserializer(rootAvroType: Schema, - rootCatalystType: DataType, - positionalFieldMatch: Boolean, - datetimeRebaseSpec: RebaseSpec, - filters: StructFilters) { - - def this(rootAvroType: Schema, - rootCatalystType: DataType, - datetimeRebaseMode: String) = { - this( - rootAvroType, - rootCatalystType, - positionalFieldMatch = false, - RebaseSpec(LegacyBehaviorPolicy.withName(datetimeRebaseMode)), - new NoopFilters) - } - - private lazy val decimalConversions = new DecimalConversion() - - private val dateRebaseFunc = createDateRebaseFuncInRead(datetimeRebaseSpec.mode, "Avro") - - private val timestampRebaseFunc = createTimestampRebaseFuncInRead(datetimeRebaseSpec, "Avro") - - private val converter: Any => Option[Any] = try { - rootCatalystType match { - // A shortcut for empty schema. - case st: StructType if st.isEmpty => - (_: Any) => Some(InternalRow.empty) - - case st: StructType => - val resultRow = new SpecificInternalRow(st.map(_.dataType)) - val fieldUpdater = new RowUpdater(resultRow) - val applyFilters = filters.skipRow(resultRow, _) - val writer = getRecordWriter(rootAvroType, st, Nil, Nil, applyFilters) - (data: Any) => { - val record = data.asInstanceOf[GenericRecord] - val skipRow = writer(fieldUpdater, record) - if (skipRow) None else Some(resultRow) - } - - case _ => - val tmpRow = new SpecificInternalRow(Seq(rootCatalystType)) - val fieldUpdater = new RowUpdater(tmpRow) - val writer = newWriter(rootAvroType, rootCatalystType, Nil, Nil) - (data: Any) => { - writer(fieldUpdater, 0, data) - Some(tmpRow.get(0, rootCatalystType)) - } - } - } catch { - case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException( - s"Cannot convert Avro type $rootAvroType to SQL type ${rootCatalystType.sql}.", ise) - } - - def deserialize(data: Any): Option[Any] = converter(data) - - /** - * Creates a writer to write avro values to Catalyst values at the given ordinal with the given - * updater. - */ - private def newWriter(avroType: Schema, - catalystType: DataType, - avroPath: Seq[String], - catalystPath: Seq[String]): (CatalystDataUpdater, Int, Any) => Unit = { - val errorPrefix = s"Cannot convert Avro ${toFieldStr(avroPath)} to " + - s"SQL ${toFieldStr(catalystPath)} because " - val incompatibleMsg = errorPrefix + - s"schema is incompatible (avroType = $avroType, sqlType = ${catalystType.sql})" - - (avroType.getType, catalystType) match { - case (NULL, NullType) => (updater, ordinal, _) => - updater.setNullAt(ordinal) - - // TODO: we can avoid boxing if future version of avro provide primitive accessors. - case (BOOLEAN, BooleanType) => (updater, ordinal, value) => - updater.setBoolean(ordinal, value.asInstanceOf[Boolean]) - - case (INT, IntegerType) => (updater, ordinal, value) => - updater.setInt(ordinal, value.asInstanceOf[Int]) - - case (INT, DateType) => (updater, ordinal, value) => - updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int])) - - case (LONG, LongType) => (updater, ordinal, value) => - updater.setLong(ordinal, value.asInstanceOf[Long]) - - case (LONG, TimestampType) => avroType.getLogicalType match { - // For backward compatibility, if the Avro type is Long and it is not logical type - // (the `null` case), the value is processed as timestamp type with millisecond precision. - case null | _: TimestampMillis => (updater, ordinal, value) => - val millis = value.asInstanceOf[Long] - val micros = DateTimeUtils.millisToMicros(millis) - updater.setLong(ordinal, timestampRebaseFunc(micros)) - case _: TimestampMicros => (updater, ordinal, value) => - val micros = value.asInstanceOf[Long] - updater.setLong(ordinal, timestampRebaseFunc(micros)) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"Avro logical type $other cannot be converted to SQL type ${TimestampType.sql}.") - } - - case (LONG, TimestampNTZType) => avroType.getLogicalType match { - // To keep consistent with TimestampType, if the Avro type is Long and it is not - // logical type (the `null` case), the value is processed as TimestampNTZ - // with millisecond precision. - case null | _: LocalTimestampMillis => (updater, ordinal, value) => - val millis = value.asInstanceOf[Long] - val micros = DateTimeUtils.millisToMicros(millis) - updater.setLong(ordinal, micros) - case _: LocalTimestampMicros => (updater, ordinal, value) => - val micros = value.asInstanceOf[Long] - updater.setLong(ordinal, micros) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"Avro logical type $other cannot be converted to SQL type ${TimestampNTZType.sql}.") - } - - // Handle VECTOR logical type (FLOAT, DOUBLE, INT8) - case (FIXED, ArrayType(elementType, false)) => avroType.getLogicalType match { - case vectorLogicalType: VectorLogicalType => - val dimension = vectorLogicalType.getDimension - val vecElementType = HoodieSchema.Vector.VectorElementType.fromString(vectorLogicalType.getElementType) - val elementSize = vecElementType.getElementSize - (updater, ordinal, value) => { - val bytes = value.asInstanceOf[GenericData.Fixed].bytes() - val expectedSize = Math.multiplyExact(dimension, elementSize) - if (bytes.length != expectedSize) { - throw new IncompatibleSchemaException( - s"VECTOR byte size mismatch: expected=$expectedSize, actual=${bytes.length}") - } - elementType match { - case FloatType => - val buffer = ByteBuffer.wrap(bytes).order(VectorLogicalType.VECTOR_BYTE_ORDER) - val floats = new Array[Float](dimension) - var i = 0; while (i < dimension) { floats(i) = buffer.getFloat(); i += 1 } - updater.set(ordinal, ArrayData.toArrayData(floats)) - case DoubleType => - val buffer = ByteBuffer.wrap(bytes).order(VectorLogicalType.VECTOR_BYTE_ORDER) - val doubles = new Array[Double](dimension) - var i = 0; while (i < dimension) { doubles(i) = buffer.getDouble(); i += 1 } - updater.set(ordinal, ArrayData.toArrayData(doubles)) - case ByteType => - updater.set(ordinal, ArrayData.toArrayData(bytes.clone())) - } - } - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - - // Before we upgrade Avro to 1.8 for logical type support, spark-avro converts Long to Date. - // For backward compatibility, we still keep this conversion. - case (LONG, DateType) => (updater, ordinal, value) => - updater.setInt(ordinal, (value.asInstanceOf[Long] / MILLIS_PER_DAY).toInt) - - case (FLOAT, FloatType) => (updater, ordinal, value) => - updater.setFloat(ordinal, value.asInstanceOf[Float]) - - case (DOUBLE, DoubleType) => (updater, ordinal, value) => - updater.setDouble(ordinal, value.asInstanceOf[Double]) - - case (STRING, StringType) => (updater, ordinal, value) => - val str = value match { - case s: String => UTF8String.fromString(s) - case s: Utf8 => - val bytes = new Array[Byte](s.getByteLength) - System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength) - UTF8String.fromBytes(bytes) - case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString) - } - updater.set(ordinal, str) - - case (ENUM, StringType) => (updater, ordinal, value) => - updater.set(ordinal, UTF8String.fromString(value.toString)) - - case (FIXED, BinaryType) => (updater, ordinal, value) => - updater.set(ordinal, value.asInstanceOf[GenericFixed].bytes().clone()) - - case (BYTES, BinaryType) => (updater, ordinal, value) => - val bytes = value match { - case b: ByteBuffer => - val bytes = new Array[Byte](b.remaining) - b.get(bytes) - // Do not forget to reset the position - b.rewind() - bytes - case b: Array[Byte] => b - case other => - throw new RuntimeException(errorPrefix + s"$other is not a valid avro binary.") - } - updater.set(ordinal, bytes) - - case (FIXED, _: DecimalType) => (updater, ordinal, value) => - val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal] - val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d) - val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale) - updater.setDecimal(ordinal, decimal) - - case (BYTES, _: DecimalType) => (updater, ordinal, value) => - val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal] - val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d) - val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale) - updater.setDecimal(ordinal, decimal) - - case (RECORD, st: StructType) => - // Avro datasource doesn't accept filters with nested attributes. See SPARK-32328. - // We can always return `false` from `applyFilters` for nested records. - val writeRecord = - getRecordWriter(avroType, st, avroPath, catalystPath, applyFilters = _ => false) - (updater, ordinal, value) => - val row = new SpecificInternalRow(st) - writeRecord(new RowUpdater(row), value.asInstanceOf[GenericRecord]) - updater.set(ordinal, row) - - case (ARRAY, ArrayType(elementType, containsNull)) => - val avroElementPath = avroPath :+ "element" - val elementWriter = newWriter(avroType.getElementType, elementType, - avroElementPath, catalystPath :+ "element") - (updater, ordinal, value) => - val collection = value.asInstanceOf[java.util.Collection[Any]] - val result = createArrayData(elementType, collection.size()) - val elementUpdater = new ArrayDataUpdater(result) - - var i = 0 - val iter = collection.iterator() - while (iter.hasNext) { - val element = iter.next() - if (element == null) { - if (!containsNull) { - throw new RuntimeException( - s"Array value at path ${toFieldStr(avroElementPath)} is not allowed to be null") - } else { - elementUpdater.setNullAt(i) - } - } else { - elementWriter(elementUpdater, i, element) - } - i += 1 - } - - updater.set(ordinal, result) - - case (MAP, MapType(keyType, valueType, valueContainsNull)) if keyType == StringType => - val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType, - avroPath :+ "key", catalystPath :+ "key") - val valueWriter = newWriter(avroType.getValueType, valueType, - avroPath :+ "value", catalystPath :+ "value") - (updater, ordinal, value) => - val map = value.asInstanceOf[java.util.Map[AnyRef, AnyRef]] - val keyArray = createArrayData(keyType, map.size()) - val keyUpdater = new ArrayDataUpdater(keyArray) - val valueArray = createArrayData(valueType, map.size()) - val valueUpdater = new ArrayDataUpdater(valueArray) - val iter = map.entrySet().iterator() - var i = 0 - while (iter.hasNext) { - val entry = iter.next() - assert(entry.getKey != null) - keyWriter(keyUpdater, i, entry.getKey) - if (entry.getValue == null) { - if (!valueContainsNull) { - throw new RuntimeException( - s"Map value at path ${toFieldStr(avroPath :+ "value")} is not allowed to be null") - } else { - valueUpdater.setNullAt(i) - } - } else { - valueWriter(valueUpdater, i, entry.getValue) - } - i += 1 - } - - // The Avro map will never have null or duplicated map keys, it's safe to create a - // ArrayBasedMapData directly here. - updater.set(ordinal, new ArrayBasedMapData(keyArray, valueArray)) - - case (UNION, _) => - val allTypes = avroType.getTypes.asScala - val nonNullTypes = allTypes.filter(_.getType != NULL) - val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava) - if (nonNullTypes.nonEmpty) { - if (nonNullTypes.length == 1) { - newWriter(nonNullTypes.head, catalystType, avroPath, catalystPath) - } else { - nonNullTypes.map(_.getType).toSeq match { - case Seq(a, b) if Set(a, b) == Set(INT, LONG) && catalystType == LongType => - (updater, ordinal, value) => value match { - case null => updater.setNullAt(ordinal) - case l: java.lang.Long => updater.setLong(ordinal, l) - case i: java.lang.Integer => updater.setLong(ordinal, i.longValue()) - } - - case Seq(a, b) if Set(a, b) == Set(FLOAT, DOUBLE) && catalystType == DoubleType => - (updater, ordinal, value) => value match { - case null => updater.setNullAt(ordinal) - case d: java.lang.Double => updater.setDouble(ordinal, d) - case f: java.lang.Float => updater.setDouble(ordinal, f.doubleValue()) - } - - case _ => - catalystType match { - case st: StructType if st.length == nonNullTypes.size => - val fieldWriters = nonNullTypes.zip(st.fields).map { - case (schema, field) => - newWriter(schema, field.dataType, avroPath, catalystPath :+ field.name) - }.toArray - (updater, ordinal, value) => { - val row = new SpecificInternalRow(st) - val fieldUpdater = new RowUpdater(row) - val i = GenericData.get().resolveUnion(nonNullAvroType, value) - fieldWriters(i)(fieldUpdater, i, value) - updater.set(ordinal, row) - } - - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - } - } - } else { - (updater, ordinal, _) => updater.setNullAt(ordinal) - } - - case (INT, _: YearMonthIntervalType) => (updater, ordinal, value) => - updater.setInt(ordinal, value.asInstanceOf[Int]) - - case (LONG, _: DayTimeIntervalType) => (updater, ordinal, value) => - updater.setLong(ordinal, value.asInstanceOf[Long]) - - case _ => throw new IncompatibleSchemaException(incompatibleMsg) - } - } - - // TODO: move the following method in Decimal object on creating Decimal from BigDecimal? - private def createDecimal(decimal: BigDecimal, precision: Int, scale: Int): Decimal = { - if (precision <= Decimal.MAX_LONG_DIGITS) { - // Constructs a `Decimal` with an unscaled `Long` value if possible. - Decimal(decimal.unscaledValue().longValue(), precision, scale) - } else { - // Otherwise, resorts to an unscaled `BigInteger` instead. - Decimal(decimal, precision, scale) - } - } - - private def getRecordWriter( - avroType: Schema, - catalystType: StructType, - avroPath: Seq[String], - catalystPath: Seq[String], - applyFilters: Int => Boolean): (CatalystDataUpdater, GenericRecord) => Boolean = { - - val avroSchemaHelper = new AvroUtils.AvroSchemaHelper( - avroType, catalystType, avroPath, catalystPath, positionalFieldMatch) - - avroSchemaHelper.validateNoExtraCatalystFields(ignoreNullable = true) - // no need to validateNoExtraAvroFields since extra Avro fields are ignored - - val (validFieldIndexes, fieldWriters) = avroSchemaHelper.matchedFields.map { - case AvroMatchedField(catalystField, ordinal, avroField) => - val baseWriter = newWriter(avroField.schema(), catalystField.dataType, - avroPath :+ avroField.name, catalystPath :+ catalystField.name) - val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => { - if (value == null) { - fieldUpdater.setNullAt(ordinal) - } else { - baseWriter(fieldUpdater, ordinal, value) - } - } - (avroField.pos(), fieldWriter) - }.toArray.unzip - - (fieldUpdater, record) => { - var i = 0 - var skipRow = false - while (i < validFieldIndexes.length && !skipRow) { - fieldWriters(i)(fieldUpdater, record.get(validFieldIndexes(i))) - skipRow = applyFilters(i) - i += 1 - } - skipRow - } - } - - private def createArrayData(elementType: DataType, length: Int): ArrayData = elementType match { - case BooleanType => UnsafeArrayData.fromPrimitiveArray(new Array[Boolean](length)) - case ByteType => UnsafeArrayData.fromPrimitiveArray(new Array[Byte](length)) - case ShortType => UnsafeArrayData.fromPrimitiveArray(new Array[Short](length)) - case IntegerType => UnsafeArrayData.fromPrimitiveArray(new Array[Int](length)) - case LongType => UnsafeArrayData.fromPrimitiveArray(new Array[Long](length)) - case FloatType => UnsafeArrayData.fromPrimitiveArray(new Array[Float](length)) - case DoubleType => UnsafeArrayData.fromPrimitiveArray(new Array[Double](length)) - case _ => new GenericArrayData(new Array[Any](length)) - } - - /** - * A base interface for updating values inside catalyst data structure like `InternalRow` and - * `ArrayData`. - */ - sealed trait CatalystDataUpdater { - def set(ordinal: Int, value: Any): Unit - - def setNullAt(ordinal: Int): Unit = set(ordinal, null) - def setBoolean(ordinal: Int, value: Boolean): Unit = set(ordinal, value) - def setByte(ordinal: Int, value: Byte): Unit = set(ordinal, value) - def setShort(ordinal: Int, value: Short): Unit = set(ordinal, value) - def setInt(ordinal: Int, value: Int): Unit = set(ordinal, value) - def setLong(ordinal: Int, value: Long): Unit = set(ordinal, value) - def setDouble(ordinal: Int, value: Double): Unit = set(ordinal, value) - def setFloat(ordinal: Int, value: Float): Unit = set(ordinal, value) - def setDecimal(ordinal: Int, value: Decimal): Unit = set(ordinal, value) - } - - final class RowUpdater(row: InternalRow) extends CatalystDataUpdater { - override def set(ordinal: Int, value: Any): Unit = row.update(ordinal, value) - - override def setNullAt(ordinal: Int): Unit = row.setNullAt(ordinal) - override def setBoolean(ordinal: Int, value: Boolean): Unit = row.setBoolean(ordinal, value) - override def setByte(ordinal: Int, value: Byte): Unit = row.setByte(ordinal, value) - override def setShort(ordinal: Int, value: Short): Unit = row.setShort(ordinal, value) - override def setInt(ordinal: Int, value: Int): Unit = row.setInt(ordinal, value) - override def setLong(ordinal: Int, value: Long): Unit = row.setLong(ordinal, value) - override def setDouble(ordinal: Int, value: Double): Unit = row.setDouble(ordinal, value) - override def setFloat(ordinal: Int, value: Float): Unit = row.setFloat(ordinal, value) - override def setDecimal(ordinal: Int, value: Decimal): Unit = - row.setDecimal(ordinal, value, value.precision) - } - - final class ArrayDataUpdater(array: ArrayData) extends CatalystDataUpdater { - override def set(ordinal: Int, value: Any): Unit = array.update(ordinal, value) - - override def setNullAt(ordinal: Int): Unit = array.setNullAt(ordinal) - override def setBoolean(ordinal: Int, value: Boolean): Unit = array.setBoolean(ordinal, value) - override def setByte(ordinal: Int, value: Byte): Unit = array.setByte(ordinal, value) - override def setShort(ordinal: Int, value: Short): Unit = array.setShort(ordinal, value) - override def setInt(ordinal: Int, value: Int): Unit = array.setInt(ordinal, value) - override def setLong(ordinal: Int, value: Long): Unit = array.setLong(ordinal, value) - override def setDouble(ordinal: Int, value: Double): Unit = array.setDouble(ordinal, value) - override def setFloat(ordinal: Int, value: Float): Unit = array.setFloat(ordinal, value) - override def setDecimal(ordinal: Int, value: Decimal): Unit = array.update(ordinal, value) - } -} - -object AvroDeserializer { - - // NOTE: Following methods have been renamed in Spark 3.2.1 [1] making [[AvroDeserializer]] implementation - // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]]. - // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.2.x branch, - // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.2.0 as well as - // w/ Spark >= 3.2.1 - // - // [1] https://github.com/apache/spark/pull/34978 - - // Specification of rebase operation including `mode` and the time zone in which it is performed - case class RebaseSpec(mode: LegacyBehaviorPolicy.Value, originTimeZone: Option[String] = None) { - // Use the default JVM time zone for backward compatibility - def timeZone: String = originTimeZone.getOrElse(TimeZone.getDefault.getID) - } - - def createDateRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Int => Int = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => days: Int => - if (days < RebaseDateTime.lastSwitchJulianDay) { - throw DataSourceUtils.newRebaseExceptionInRead(format) - } - days - case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays - case LegacyBehaviorPolicy.CORRECTED => identity[Int] - } - - def createTimestampRebaseFuncInRead(rebaseSpec: RebaseSpec, - format: String): Long => Long = rebaseSpec.mode match { - case LegacyBehaviorPolicy.EXCEPTION => micros: Long => - if (micros < RebaseDateTime.lastSwitchJulianTs) { - throw DataSourceUtils.newRebaseExceptionInRead(format) - } - micros - case LegacyBehaviorPolicy.LEGACY => micros: Long => - RebaseDateTime.rebaseJulianToGregorianMicros(TimeZone.getTimeZone(rebaseSpec.timeZone), micros) - case LegacyBehaviorPolicy.CORRECTED => identity[Long] - } -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala deleted file mode 100644 index 756ef82a55d21..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.avro - -import org.apache.hudi.common.schema.HoodieSchema -import org.apache.hudi.common.schema.HoodieSchema.VectorLogicalType - -import org.apache.avro.{LogicalTypes, Schema} -import org.apache.avro.Conversions.DecimalConversion -import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis} -import org.apache.avro.Schema.Type -import org.apache.avro.Schema.Type._ -import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record} -import org.apache.avro.util.Utf8 -import org.apache.spark.internal.Logging -import org.apache.spark.sql.avro.AvroSerializer.{createDateRebaseFuncInWrite, createTimestampRebaseFuncInWrite} -import org.apache.spark.sql.avro.AvroUtils.{toFieldStr, AvroMatchedField} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow} -import org.apache.spark.sql.catalyst.util.{DateTimeUtils, RebaseDateTime} -import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} -import org.apache.spark.sql.types._ - -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.TimeZone - -import scala.collection.JavaConverters._ - -/** - * A serializer to serialize data in catalyst format to data in avro format. - * - * NOTE: This code is borrowed from Spark 3.3.0 - * This code is borrowed, so that we can better control compatibility w/in Spark minor - * branches (3.2.x, 3.1.x, etc) - * - * NOTE: THIS IMPLEMENTATION HAS BEEN MODIFIED FROM ITS ORIGINAL VERSION WITH THE MODIFICATION - * BEING EXPLICITLY ANNOTATED INLINE. PLEASE MAKE SURE TO UNDERSTAND PROPERLY ALL THE - * MODIFICATIONS. - * - * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY - */ -private[sql] class AvroSerializer(rootCatalystType: DataType, - rootAvroType: Schema, - nullable: Boolean, - positionalFieldMatch: Boolean, - datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging { - - def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) = { - this(rootCatalystType, rootAvroType, nullable, positionalFieldMatch = false, - LegacyBehaviorPolicy.withName(SQLConf.get.getConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE, - LegacyBehaviorPolicy.CORRECTED.toString))) - } - - def serialize(catalystData: Any): Any = { - converter.apply(catalystData) - } - - private val dateRebaseFunc = createDateRebaseFuncInWrite( - datetimeRebaseMode, "Avro") - - private val timestampRebaseFunc = createTimestampRebaseFuncInWrite( - datetimeRebaseMode, "Avro") - - private val converter: Any => Any = { - val actualAvroType = resolveNullableType(rootAvroType, nullable) - val baseConverter = try { - rootCatalystType match { - case st: StructType => - newStructConverter(st, actualAvroType, Nil, Nil).asInstanceOf[Any => Any] - case _ => - val tmpRow = new SpecificInternalRow(Seq(rootCatalystType)) - val converter = newConverter(rootCatalystType, actualAvroType, Nil, Nil) - (data: Any) => - tmpRow.update(0, data) - converter.apply(tmpRow, 0) - } - } catch { - case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException( - s"Cannot convert SQL type ${rootCatalystType.sql} to Avro type $rootAvroType.", ise) - } - if (nullable) { - (data: Any) => - if (data == null) { - null - } else { - baseConverter.apply(data) - } - } else { - baseConverter - } - } - - private type Converter = (SpecializedGetters, Int) => Any - - private lazy val decimalConversions = new DecimalConversion() - - private def newConverter(catalystType: DataType, - avroType: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): Converter = { - val errorPrefix = s"Cannot convert SQL ${toFieldStr(catalystPath)} " + - s"to Avro ${toFieldStr(avroPath)} because " - (catalystType, avroType.getType) match { - case (NullType, NULL) => - (getter, ordinal) => null - case (BooleanType, BOOLEAN) => - (getter, ordinal) => getter.getBoolean(ordinal) - case (ByteType, INT) => - (getter, ordinal) => getter.getByte(ordinal).toInt - case (ShortType, INT) => - (getter, ordinal) => getter.getShort(ordinal).toInt - case (IntegerType, INT) => - (getter, ordinal) => getter.getInt(ordinal) - case (LongType, LONG) => - (getter, ordinal) => getter.getLong(ordinal) - case (FloatType, FLOAT) => - (getter, ordinal) => getter.getFloat(ordinal) - case (DoubleType, DOUBLE) => - (getter, ordinal) => getter.getDouble(ordinal) - case (d: DecimalType, FIXED) - if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) => - (getter, ordinal) => - val decimal = getter.getDecimal(ordinal, d.precision, d.scale) - decimalConversions.toFixed(decimal.toJavaBigDecimal, avroType, - LogicalTypes.decimal(d.precision, d.scale)) - - case (d: DecimalType, BYTES) - if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) => - (getter, ordinal) => - val decimal = getter.getDecimal(ordinal, d.precision, d.scale) - decimalConversions.toBytes(decimal.toJavaBigDecimal, avroType, - LogicalTypes.decimal(d.precision, d.scale)) - - // Handle VECTOR logical type (FLOAT, DOUBLE, INT8) - case (ArrayType(elementType, false), FIXED) => avroType.getLogicalType match { - case vectorLogicalType: VectorLogicalType => - val dimension = vectorLogicalType.getDimension - val vecElementType = HoodieSchema.Vector.VectorElementType.fromString(vectorLogicalType.getElementType) - val bufferSize = Math.multiplyExact(dimension, vecElementType.getElementSize) - (getter, ordinal) => { - val arrayData = getter.getArray(ordinal) - if (arrayData.numElements() != dimension) { - throw new IncompatibleSchemaException( - s"VECTOR dimension mismatch at ${toFieldStr(catalystPath)}: " + - s"expected=$dimension, actual=${arrayData.numElements()}") - } - elementType match { - case FloatType => - val buffer = ByteBuffer.allocate(bufferSize).order(VectorLogicalType.VECTOR_BYTE_ORDER) - var i = 0; while (i < dimension) { buffer.putFloat(arrayData.getFloat(i)); i += 1 } - new Fixed(avroType, buffer.array()) - case DoubleType => - val buffer = ByteBuffer.allocate(bufferSize).order(VectorLogicalType.VECTOR_BYTE_ORDER) - var i = 0; while (i < dimension) { buffer.putDouble(arrayData.getDouble(i)); i += 1 } - new Fixed(avroType, buffer.array()) - case ByteType => - val bytes = new Array[Byte](dimension) - var i = 0; while (i < dimension) { bytes(i) = arrayData.getByte(i); i += 1 } - new Fixed(avroType, bytes) - case _ => throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - } - case _ => throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - - case (StringType, ENUM) => - val enumSymbols: Set[String] = avroType.getEnumSymbols.asScala.toSet - (getter, ordinal) => - val data = getter.getUTF8String(ordinal).toString - if (!enumSymbols.contains(data)) { - throw new IncompatibleSchemaException(errorPrefix + - s""""$data" cannot be written since it's not defined in enum """ + - enumSymbols.mkString("\"", "\", \"", "\"")) - } - new EnumSymbol(avroType, data) - - case (StringType, STRING) => - (getter, ordinal) => new Utf8(getter.getUTF8String(ordinal).getBytes) - - case (BinaryType, FIXED) => - val size = avroType.getFixedSize - (getter, ordinal) => - val data: Array[Byte] = getter.getBinary(ordinal) - if (data.length != size) { - def len2str(len: Int): String = s"$len ${if (len > 1) "bytes" else "byte"}" - - throw new IncompatibleSchemaException(errorPrefix + len2str(data.length) + - " of binary data cannot be written into FIXED type with size of " + len2str(size)) - } - new Fixed(avroType, data) - - case (BinaryType, BYTES) => - (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal)) - - case (DateType, INT) => - (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal)) - - case (TimestampType, LONG) => avroType.getLogicalType match { - // For backward compatibility, if the Avro type is Long and it is not logical type - // (the `null` case), output the timestamp value as with millisecond precision. - case null | _: TimestampMillis => (getter, ordinal) => - DateTimeUtils.microsToMillis(timestampRebaseFunc(getter.getLong(ordinal))) - case _: TimestampMicros => (getter, ordinal) => - timestampRebaseFunc(getter.getLong(ordinal)) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"SQL type ${TimestampType.sql} cannot be converted to Avro logical type $other") - } - - case (TimestampNTZType, LONG) => avroType.getLogicalType match { - // To keep consistent with TimestampType, if the Avro type is Long and it is not - // logical type (the `null` case), output the TimestampNTZ as long value - // in millisecond precision. - case null | _: LocalTimestampMillis => (getter, ordinal) => - DateTimeUtils.microsToMillis(getter.getLong(ordinal)) - case _: LocalTimestampMicros => (getter, ordinal) => - getter.getLong(ordinal) - case other => throw new IncompatibleSchemaException(errorPrefix + - s"SQL type ${TimestampNTZType.sql} cannot be converted to Avro logical type $other") - } - - case (ArrayType(et, containsNull), ARRAY) => - val elementConverter = newConverter( - et, resolveNullableType(avroType.getElementType, containsNull), - catalystPath :+ "element", avroPath :+ "element") - (getter, ordinal) => { - val arrayData = getter.getArray(ordinal) - val len = arrayData.numElements() - val result = new Array[Any](len) - var i = 0 - while (i < len) { - if (containsNull && arrayData.isNullAt(i)) { - result(i) = null - } else { - result(i) = elementConverter(arrayData, i) - } - i += 1 - } - // avro writer is expecting a Java Collection, so we convert it into - // `ArrayList` backed by the specified array without data copying. - java.util.Arrays.asList(result: _*) - } - - case (st: StructType, RECORD) => - val structConverter = newStructConverter(st, avroType, catalystPath, avroPath) - val numFields = st.length - (getter, ordinal) => structConverter(getter.getStruct(ordinal, numFields)) - - //////////////////////////////////////////////////////////////////////////////////////////// - // Following section is amended to the original (Spark's) implementation - // >>> BEGINS - //////////////////////////////////////////////////////////////////////////////////////////// - - case (st: StructType, UNION) => - val unionConverter = newUnionConverter(st, avroType, catalystPath, avroPath) - val numFields = st.length - (getter, ordinal) => unionConverter(getter.getStruct(ordinal, numFields)) - - //////////////////////////////////////////////////////////////////////////////////////////// - // <<< ENDS - //////////////////////////////////////////////////////////////////////////////////////////// - - case (MapType(kt, vt, valueContainsNull), MAP) if kt == StringType => - val valueConverter = newConverter( - vt, resolveNullableType(avroType.getValueType, valueContainsNull), - catalystPath :+ "value", avroPath :+ "value") - (getter, ordinal) => - val mapData = getter.getMap(ordinal) - val len = mapData.numElements() - val result = new java.util.HashMap[String, Any](len) - val keyArray = mapData.keyArray() - val valueArray = mapData.valueArray() - var i = 0 - while (i < len) { - val key = keyArray.getUTF8String(i).toString - if (valueContainsNull && valueArray.isNullAt(i)) { - result.put(key, null) - } else { - result.put(key, valueConverter(valueArray, i)) - } - i += 1 - } - result - - case (_: YearMonthIntervalType, INT) => - (getter, ordinal) => getter.getInt(ordinal) - - case (_: DayTimeIntervalType, LONG) => - (getter, ordinal) => getter.getLong(ordinal) - - case _ => - throw new IncompatibleSchemaException(errorPrefix + - s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)") - } - } - - private def newStructConverter(catalystStruct: StructType, - avroStruct: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): InternalRow => Record = { - - val avroSchemaHelper = new AvroUtils.AvroSchemaHelper( - avroStruct, catalystStruct, avroPath, catalystPath, positionalFieldMatch) - - avroSchemaHelper.validateNoExtraCatalystFields(ignoreNullable = false) - avroSchemaHelper.validateNoExtraRequiredAvroFields() - - val (avroIndices, fieldConverters) = avroSchemaHelper.matchedFields.map { - case AvroMatchedField(catalystField, _, avroField) => - val converter = newConverter(catalystField.dataType, - resolveNullableType(avroField.schema(), catalystField.nullable), - catalystPath :+ catalystField.name, avroPath :+ avroField.name) - (avroField.pos(), converter) - }.toArray.unzip - - val numFields = catalystStruct.length - row: InternalRow => - val result = new Record(avroStruct) - var i = 0 - while (i < numFields) { - if (row.isNullAt(i)) { - result.put(avroIndices(i), null) - } else { - result.put(avroIndices(i), fieldConverters(i).apply(row, i)) - } - i += 1 - } - result - } - - //////////////////////////////////////////////////////////////////////////////////////////// - // Following section is amended to the original (Spark's) implementation - // >>> BEGINS - //////////////////////////////////////////////////////////////////////////////////////////// - - private def newUnionConverter(catalystStruct: StructType, - avroUnion: Schema, - catalystPath: Seq[String], - avroPath: Seq[String]): InternalRow => Any = { - if (avroUnion.getType != UNION || !canMapUnion(catalystStruct, avroUnion)) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " + - s"Avro type $avroUnion.") - } - val nullable = avroUnion.getTypes.size() > 0 && avroUnion.getTypes.get(0).getType == Type.NULL - val avroInnerTypes = if (nullable) { - avroUnion.getTypes.asScala.tail - } else { - avroUnion.getTypes.asScala - } - val fieldConverters = catalystStruct.zip(avroInnerTypes).map { - case (f1, f2) => newConverter(f1.dataType, f2, catalystPath, avroPath) - } - val numFields = catalystStruct.length - (row: InternalRow) => - var i = 0 - var result: Any = null - while (i < numFields) { - if (!row.isNullAt(i)) { - if (result != null) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " + - s"Avro union $avroUnion. Record has more than one optional values set") - } - result = fieldConverters(i).apply(row, i) - } - i += 1 - } - if (!nullable && result == null) { - throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " + - s"Avro union $avroUnion. Record has no values set, while should have exactly one") - } - result - } - - private def canMapUnion(catalystStruct: StructType, avroStruct: Schema): Boolean = { - (avroStruct.getTypes.size() > 0 && - avroStruct.getTypes.get(0).getType == Type.NULL && - avroStruct.getTypes.size() - 1 == catalystStruct.length) || avroStruct.getTypes.size() == catalystStruct.length - } - - //////////////////////////////////////////////////////////////////////////////////////////// - // <<< ENDS - //////////////////////////////////////////////////////////////////////////////////////////// - - - /** - * Resolve a possibly nullable Avro Type. - * - * An Avro type is nullable when it is a [[UNION]] of two types: one null type and another - * non-null type. This method will check the nullability of the input Avro type and return the - * non-null type within when it is nullable. Otherwise it will return the input Avro type - * unchanged. It will throw an [[UnsupportedAvroTypeException]] when the input Avro type is an - * unsupported nullable type. - * - * It will also log a warning message if the nullability for Avro and catalyst types are - * different. - */ - private def resolveNullableType(avroType: Schema, nullable: Boolean): Schema = { - val (avroNullable, resolvedAvroType) = resolveAvroType(avroType) - warnNullabilityDifference(avroNullable, nullable) - resolvedAvroType - } - - /** - * Check the nullability of the input Avro type and resolve it when it is nullable. The first - * return value is a [[Boolean]] indicating if the input Avro type is nullable. The second - * return value is the possibly resolved type. - */ - private def resolveAvroType(avroType: Schema): (Boolean, Schema) = { - if (avroType.getType == Type.UNION) { - val fields = avroType.getTypes.asScala - val actualType = fields.filter(_.getType != Type.NULL) - if (fields.length == 2 && actualType.length == 1) { - (true, actualType.head) - } else { - // This is just a normal union, not used to designate nullability - (false, avroType) - } - } else { - (false, avroType) - } - } - - /** - * log a warning message if the nullability for Avro and catalyst types are different. - */ - private def warnNullabilityDifference(avroNullable: Boolean, catalystNullable: Boolean): Unit = { - if (avroNullable && !catalystNullable) { - logWarning("Writing Avro files with nullable Avro schema and non-nullable catalyst schema.") - } - if (!avroNullable && catalystNullable) { - logWarning("Writing Avro files with non-nullable Avro schema and nullable catalyst " + - "schema will throw runtime exception if there is a record with null value.") - } - } -} - -object AvroSerializer { - - // NOTE: Following methods have been renamed in Spark 3.2.1 [1] making [[AvroSerializer]] implementation - // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]]. - // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.2.x branch, - // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.2.0 as well as - // w/ Spark >= 3.2.1 - // - // [1] https://github.com/apache/spark/pull/34978 - - def createDateRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Int => Int = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => days: Int => - if (days < RebaseDateTime.lastSwitchGregorianDay) { - throw DataSourceUtils.newRebaseExceptionInWrite(format) - } - days - case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays - case LegacyBehaviorPolicy.CORRECTED => identity[Int] - } - - def createTimestampRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value, - format: String): Long => Long = rebaseMode match { - case LegacyBehaviorPolicy.EXCEPTION => micros: Long => - if (micros < RebaseDateTime.lastSwitchGregorianTs) { - throw DataSourceUtils.newRebaseExceptionInWrite(format) - } - micros - case LegacyBehaviorPolicy.LEGACY => - val timeZone = SQLConf.get.sessionLocalTimeZone - RebaseDateTime.rebaseGregorianToJulianMicros(TimeZone.getTimeZone(timeZone), _) - case LegacyBehaviorPolicy.CORRECTED => identity[Long] - } - -} From 133eee39cf3cbffa29b8de7cf70e9d992555aa0f Mon Sep 17 00:00:00 2001 From: Joy <33287603+Joy-2000@users.noreply.github.com> Date: Sat, 11 Jul 2026 07:27:59 +0800 Subject: [PATCH 152/255] fix(flink): prevent data loss on global failover for streaming writes (#19237) * fix(flink): prevent data loss on global failover for streaming writes --------- Co-authored-by: jiangyu84 Co-authored-by: danny0405 (cherry picked from commit be8a54c4ebdaf287993d24fefba60a58952a9c9e) --- .../hudi/client/BaseHoodieWriteClient.java | 23 +++++ .../HoodieBackedTableMetadataWriter.java | 9 ++ .../TestHoodieBackedTableMetadataWriter.java | 27 +++++ .../FlinkStreamingMetadataWriteHandler.java | 30 +++++- .../hudi/client/HoodieFlinkWriteClient.java | 25 +++++ .../hudi/client/TestFlinkWriteClient.java | 98 ++++++++++++++++++- .../sink/StreamWriteOperatorCoordinator.java | 28 +++--- .../apache/hudi/sink/utils/EventBuffers.java | 10 +- .../TestStreamWriteOperatorCoordinator.java | 94 ++++++++++++------ .../hudi/sink/TestWriteMergeOnRead.java | 35 +++++++ 10 files changed, 327 insertions(+), 52 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index f5d050d880986..73f5de7b94ef4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -679,6 +679,29 @@ protected void postCommit(HoodieTable table, HoodieCommitMetadata metadata, Stri } } + /** + * Performs post-commit cleanup when the instant is already completed and commit metadata is not + * available to invoke the regular post-commit hook. This can happen while recovering a streaming + * metadata-table write after failover. The table is recreated from the write configuration so its + * marker directory can still be removed, and the heartbeat is always stopped even if marker cleanup + * fails. + * + * @param instantTime the completed instant to clean up + */ + public void postCommit(String instantTime) { + try { + HoodieTable table = createTable(config); + context.setJobStatus(this.getClass().getSimpleName(), "Cleaning up marker directories for commit " + instantTime + " in table " + + config.getTableName()); + // Delete the marker directory for the instant. + WriteMarkersFactory.get(config.getMarkersType(), table, instantTime) + .quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism()); + metrics.updateTableServiceInstantMetrics(table.getActiveTimeline()); + } finally { + this.heartbeatClient.stop(instantTime); + } + } + /** * Triggers cleaning and archival for the table of interest. This method is called outside of locks. So, internal callers should ensure they acquire lock whereever applicable. * @param table instance of {@link HoodieTable} of interest. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index df61ed996b1f0..2185418fe8eba 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1469,6 +1469,12 @@ public O secondaryWriteToMetadataTablePartitions(I preppedRecords, String instan @Override public void completeStreamingCommit(String instantTime, HoodieEngineContext context, List partialWriteStats, HoodieCommitMetadata metadata) { + if (metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(instantTime)) { + LOG.info("Skipping streaming metadata commit completion for already completed instant {}", instantTime); + getWriteClient().postCommit(instantTime); + return; + } + List allWriteStats = new ArrayList<>(partialWriteStats); // update metadata for left over partitions which does not have streaming writes support. allWriteStats.addAll(prepareAndWriteToNonStreamingPartitions(metadata, instantTime).map(WriteStatus::getStat).collectAsList()); @@ -1907,6 +1913,9 @@ && compareTimestamps(commitToRollbackInstant.getCompletionTime(), LESSER_THAN_OR public void close() throws Exception { if (metadata != null) { metadata.close(); + // Keep the closed reader reference: guarded update paths use its presence to proceed and + // mayBeReinitMetadataReader() detects the closed file-system view and reopens the reader. + // Nullifying it here would silently skip subsequent metadata updates, such as rollbacks. } if (writeClient != null) { writeClient.close(); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java index 7fcc2ce9a5d85..449578f3968d3 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.config.HoodieTableServiceManagerConfig; import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; @@ -47,6 +48,7 @@ import org.mockito.MockedStatic; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,6 +65,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; @@ -71,6 +74,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; class TestHoodieBackedTableMetadataWriter { @@ -89,6 +93,29 @@ void setUp() { when(metadataConfig.getMaxReaderBufferSize()).thenReturn(1024); } + @Test + void completeStreamingCommitSkipsAlreadyCompletedMetadataInstant() { + String instantTime = "20260709120000000"; + HoodieBackedTableMetadataWriter, List> metadataWriter = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + HoodieEngineContext engineContext = mock(HoodieEngineContext.class); + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); + HoodieTimeline completedTimeline = mock(HoodieTimeline.class); + BaseHoodieWriteClient writeClient = mock(BaseHoodieWriteClient.class); + + metadataWriter.metadataMetaClient = metadataMetaClient; + when(metadataMetaClient.getActiveTimeline()).thenReturn(activeTimeline); + when(activeTimeline.filterCompletedInstants()).thenReturn(completedTimeline); + when(completedTimeline.containsInstant(instantTime)).thenReturn(true); + when(metadataWriter.initializeWriteClient()).thenReturn(writeClient); + + metadataWriter.completeStreamingCommit(instantTime, engineContext, Collections.emptyList(), mock(HoodieCommitMetadata.class)); + + verify(writeClient).postCommit(instantTime); + verifyNoMoreInteractions(writeClient); + } + @ParameterizedTest @CsvSource(value = { "true,true,false,true", diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/FlinkStreamingMetadataWriteHandler.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/FlinkStreamingMetadataWriteHandler.java index f500f99e0635b..33c34b8c00a14 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/FlinkStreamingMetadataWriteHandler.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/FlinkStreamingMetadataWriteHandler.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.metadata.FlinkHoodieBackedTableMetadataWriter; import org.apache.hudi.metadata.HoodieTableMetadataWriter; import org.apache.hudi.table.HoodieTable; @@ -83,6 +84,27 @@ public void startCommit(String instantTime, HoodieTable table) { metadataWriterOpt.get().startCommit(instantTime); } + /** + * Start only the metadata table heartbeat for an existing streaming write instant. + * + *

    This is used by coordinator recommit where the metadata table instant and + * the streaming index files may already exist and must not be rolled back. + * + * @param instantTime The instant time + * @param table The hoodie table + */ + public void startHeartbeat(String instantTime, HoodieTable table) { + Option metadataWriterOpt = getMetadataWriter(instantTime, table); + ValidationUtils.checkState(metadataWriterOpt.isPresent(), + "Should not be reachable. Metadata Writer should have been instantiated by now"); + ValidationUtils.checkState(metadataWriterOpt.get() instanceof FlinkHoodieBackedTableMetadataWriter, + "Flink streaming metadata writes expect a Flink metadata writer"); + FlinkHoodieBackedTableMetadataWriter metadataWriter = (FlinkHoodieBackedTableMetadataWriter) metadataWriterOpt.get(); + if (metadataWriter.getWriteClient().getConfig().getFailedWritesCleanPolicy().isLazy()) { + metadataWriter.getWriteClient().getHeartbeatClient().start(instantTime); + } + } + /** * Clean resources after streaming write to the metadata table in index write function or stop * heartbeat for instant in the coordinator. This method removes the metadata writer associated @@ -93,11 +115,13 @@ public void startCommit(String instantTime, HoodieTable table) { public void cleanResources(String instantTime) { Option metadataWriterOpt = this.metadataWriterMap.remove(instantTime); if (metadataWriterOpt == null || metadataWriterOpt.isEmpty()) { - log.warn("Metadata writer for {} has not been initialized, no need to stop heartbeat.", instantTime); + log.debug("Metadata writer for {} has already been closed, skip closing.", instantTime); return; } - try { - metadataWriterOpt.get().close(); + try (HoodieTableMetadataWriter metadataWriter = metadataWriterOpt.get()) { + if (metadataWriter instanceof FlinkHoodieBackedTableMetadataWriter) { + ((FlinkHoodieBackedTableMetadataWriter) metadataWriter).getWriteClient().getHeartbeatClient().stop(instantTime); + } } catch (Exception e) { throw new HoodieException("Failed to close the metadata writer for instant: " + instantTime, e); } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java index 228209cf1df6f..51ef11b390131 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java @@ -138,6 +138,20 @@ public void cleanResources(String instantTime) { } } + /** + * Restart the heartbeat for a recommitted instant. + * + * @param instantTime The instant time + */ + public void restartHeartbeat(String instantTime) { + if (getConfig().getFailedWritesCleanPolicy().isLazy()) { + getHeartbeatClient().start(instantTime); + } + if (isStreamingWriteMetadataTable) { + this.streamingMetadataWriteHandler.startHeartbeat(instantTime, getHoodieTable()); + } + } + /** * Performs streaming write operations to metadata partitions. * This method retrieves the metadata writer for the given instant time and table, @@ -629,4 +643,15 @@ public void close() { ((MiniBatchHandle) writeHandle).closeGracefully(); } } + + /** + * Flink keeps the heartbeat active when a commit attempt fails because the coordinator may need to + * recommit the instant after failover. Successful commits stop the heartbeat from {@code postCommit}, + * while {@link #cleanResources(String)} cleans up data-table and metadata-table resources for + * instants discarded or resolved by the coordinator. Therefore this generic hook is intentionally + * a no-op. + */ + @Override + public void releaseResources(String instantTime) { + } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java index 33ae4238962a1..d0ae6a30a54a4 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java @@ -19,27 +19,47 @@ package org.apache.hudi.client; +import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieCleanConfig; +import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestFlinkWriteClient extends HoodieFlinkClientTestHarness { @BeforeEach - private void setup() throws IOException { + void setup() throws IOException { initPath(); initFileSystem(); initMetaClient(); } + @AfterEach + void teardown() throws IOException { + cleanupResources(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testWriteClientAndTableServiceClientWithTimelineServer( @@ -61,4 +81,80 @@ public void testWriteClientAndTableServiceClientWithTimelineServer( writeClient.close(); } + + @Test + public void testReleaseAndPostCommitResourcesForLazyFailedWrites() throws IOException { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withEngineType(EngineType.FLINK) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .build()) + .build(); + + AtomicBoolean failTableCreation = new AtomicBoolean(false); + writeClient = new HoodieFlinkWriteClient(context, writeConfig) { + @Override + protected HoodieTable createTable(HoodieWriteConfig config) { + if (failTableCreation.get()) { + throw new HoodieException("Expected table creation failure"); + } + return super.createTable(config); + } + }; + String instantTime = "20260709120000000"; + writeClient.restartHeartbeat(instantTime); + + assertTrue(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metaClient.getBasePath().toString(), instantTime)); + + writeClient.releaseResources(instantTime); + assertTrue(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metaClient.getBasePath().toString(), instantTime)); + + writeClient.postCommit(instantTime); + assertFalse(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metaClient.getBasePath().toString(), instantTime)); + + String failedPostCommitInstantTime = "20260709120000001"; + writeClient.restartHeartbeat(failedPostCommitInstantTime); + failTableCreation.set(true); + assertThrows(HoodieException.class, () -> writeClient.postCommit(failedPostCommitInstantTime)); + assertFalse(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metaClient.getBasePath().toString(), failedPostCommitInstantTime)); + } + + @Test + public void testCleanResourcesCleansMetadataTableHeartbeatForStreamingMetadataWrites() throws IOException { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withEngineType(EngineType.FLINK) + .withIndexConfig(HoodieIndexConfig.newBuilder() + .withIndexType(HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX) + .build()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withStreamingWriteEnabled(true) + .withEnableGlobalRecordLevelIndex(true) + .build()) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .build()) + .build(); + + writeClient = new HoodieFlinkWriteClient(context, writeConfig, true); + String instantTime = "20260709120000000"; + writeClient.restartHeartbeat(instantTime); + + String metadataTableBasePath = metaClient.getBasePath() + + "/" + HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH; + assertTrue(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metadataTableBasePath, instantTime)); + + writeClient.cleanResources(instantTime); + assertFalse(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metaClient.getBasePath().toString(), instantTime)); + assertFalse(HoodieHeartbeatClient.heartbeatExists( + metaClient.getStorage(), metadataTableBasePath, instantTime)); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java index 7e9d8cfa4cf55..686aaaa9509ca 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java @@ -252,7 +252,7 @@ public void start() throws Exception { if (OptionsResolver.isMultiWriter(conf)) { initClientIds(conf); } - restoreEvents(); + restoreEvents(Long.MAX_VALUE); } catch (Throwable throwable) { log.error("Failed to start operator coordinator.", throwable); context.failJob(throwable); @@ -325,14 +325,16 @@ public void notifyCheckpointComplete(long checkpointId) { @Override public void resetToCheckpoint(long checkpointID, byte[] checkpointData) { if (checkpointData != null) { - initEventBufferIfNecessary(); - this.eventBuffers.addEventsToBuffer(SerializationUtils.deserialize(checkpointData)); // resetToCheckpoint() is called in two cases: // 1. The job is restarted from state, start() will be called later. // 2. The job is recovered from global failover. The coordinator is already started, and start() will not be called again. - if (executor != null && tableState.isRecordLevelIndex) { - // use sync execution here to make sure the recommitting finishes before RLI bootstrapping - this.executor.executeSync(this::restoreEvents, "Recommit pending instants on resetting to checkpoint: %s.", checkpointID); + if (this.eventBuffers == null) { + // case1: the events restore is moved to start() since it requires the write/meta client instantiation. + initEventBuffer(); + this.eventBuffers.addEventsToBuffer(SerializationUtils.deserialize(checkpointData)); + } else { + // case2: restore the events directly. + restoreEvents(checkpointID); } } } @@ -433,10 +435,11 @@ private CompletableFuture handleInFlightInstantsRequest(Co // Utilities // ------------------------------------------------------------------------- - private void restoreEvents() { + private void restoreEvents(long checkpointId) { if (this.eventBuffers.nonEmpty()) { - final HoodieTimeline completedTimeline = this.metaClient.getActiveTimeline().filterCompletedInstants(); + final HoodieTimeline completedTimeline = this.metaClient.reloadActiveTimeline().filterCompletedInstants(); this.eventBuffers.getEventBufferStream() + .filter(entry -> entry.getKey() < checkpointId) .forEach(entry -> recommitInstant(completedTimeline, entry.getKey(), entry.getValue().getLeft(), entry.getValue().getRight())); this.metaClient.reloadActiveTimeline(); } @@ -503,6 +506,10 @@ private void initEventBufferIfNecessary() { if (this.eventBuffers != null) { return; } + initEventBuffer(); + } + + private void initEventBuffer() { // initialize event buffer this.eventBuffers = EventBuffers.getInstance(conf, this.parallelism); } @@ -544,13 +551,12 @@ private boolean recommitInstant(HoodieTimeline completedTimeline, long checkpoin log.info("Recommit instant {}", instant); // Recommit should start heartbeat for lazy failed writes clean policy to avoid aborting for heartbeat expired; // The following up checkpoints would recommit the instant. - if (writeClient.getConfig().getFailedWritesCleanPolicy().isLazy()) { - writeClient.getHeartbeatClient().start(instant); - } + writeClient.restartHeartbeat(instant); return commitInstant(checkpointId, instant, bootstrapBuffer); } else { // clean the corresponding event buffer if the instant is already committed. eventBuffers.reset(checkpointId); + writeClient.cleanResources(instant); return false; } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/EventBuffers.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/EventBuffers.java index 87e1010e2227a..10f89637e342c 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/EventBuffers.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/EventBuffers.java @@ -171,7 +171,7 @@ public String getPendingInstants() { */ public Map> getAllCompletedEvents() { return this.eventBuffers.entrySet().stream() - .filter(entry -> entry.getValue().getRight().allEventsCompleted()) + .filter(entry -> entry.getValue().getRight().allEventsReceived()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } @@ -241,14 +241,6 @@ public void resetBuffer(WriteMetadataEvent event) { } } - /** - * Return true if there is no event sent by eager flushing from writers. - */ - public boolean allEventsCompleted() { - return Stream.concat(Arrays.stream(dataWriteEventBuffer), Arrays.stream(indexWriteEventBuffer)) - .allMatch(event -> event == null || event.isLastBatch()); - } - /** * Return true if all the events in the data write buffer are null. */ diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java index d5903ef946dcd..bb57dc45314c6 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java @@ -20,6 +20,7 @@ import org.apache.hudi.client.WriteStatus; import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieWriteStat; @@ -106,7 +107,7 @@ public class TestStreamWriteOperatorCoordinator { @BeforeEach public void before() throws Exception { - coordinator = createCoordinator(TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()), 2); + coordinator = startCoordinator(TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()), 2); } @AfterEach @@ -142,6 +143,12 @@ public void testTableInitialized() throws IOException { } } + /** + * Verifies both coordinator restore paths. In case 1, a newly constructed coordinator receives + * checkpoint data before {@link StreamWriteOperatorCoordinator#start()}, so it must deserialize + * the event buffers for restoration during start. In case 2, global failover resets an already + * started coordinator, so it recommits its live event buffers directly. + */ @ParameterizedTest @ValueSource(booleans = {true, false}) public void testCheckpointAndRestore(boolean isStreamingIndexWriteEnabled) throws Exception { @@ -151,7 +158,7 @@ public void testCheckpointAndRestore(boolean isStreamingIndexWriteEnabled) throw conf.set(FlinkOptions.INDEX_TYPE, GLOBAL_RECORD_LEVEL_INDEX.name()); conf.set(FlinkOptions.INDEX_WRITE_TASKS, 2); } - coordinator = createCoordinator(conf, 2); + coordinator = startCoordinator(conf, 2); requestInstantTime(-1); String instant = coordinator.getInstant(); @@ -171,16 +178,32 @@ public void testCheckpointAndRestore(boolean isStreamingIndexWriteEnabled) throw CompletableFuture future = new CompletableFuture<>(); coordinator.checkpointCoordinator(1, future); - coordinator.notifyCheckpointComplete(1); + + // Case 1: job restart restores checkpoint data before the coordinator starts. + try (StreamWriteOperatorCoordinator restoredCoordinator = createCoordinator(conf, 2)) { + restoredCoordinator.resetToCheckpoint(1, future.get()); + + EventBuffers.EventBuffer eventBuffer = restoredCoordinator.getEventBuffer(-1); + assertEquals(2, eventBuffer.getDataWriteEventBuffer().length); + assertEquals(isStreamingIndexWriteEnabled ? 2 : 0, eventBuffer.getIndexWriteEventBuffer().length); + } + + // Case 2: global failover recommits the live buffers of the already started coordinator. coordinator.resetToCheckpoint(1, future.get()); - EventBuffers.EventBuffer eventBuffer = coordinator.getEventBuffer(); - assertEquals(2, eventBuffer.getDataWriteEventBuffer().length); - assertEquals(isStreamingIndexWriteEnabled ? 2 : 0, eventBuffer.getIndexWriteEventBuffer().length); + assertNull(coordinator.getEventBuffer()); + assertTrue(StreamerUtil.createMetaClient(conf).reloadActiveTimeline() + .filterCompletedInstants().containsInstant(instant)); } + /** + * Verifies legacy checkpoint compatibility for both restore paths. Case 1 deserializes the legacy + * checkpoint into a newly constructed coordinator. Case 2 intentionally does not deserialize the + * checkpoint because a coordinator surviving global failover recommits its live buffers directly. + */ @Test public void testRestoreFromLegacyState() throws Exception { + Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); requestInstantTime(-1); String instant = coordinator.getInstant(); assertNotEquals("", instant); @@ -192,7 +215,6 @@ public void testRestoreFromLegacyState() throws Exception { CompletableFuture future = new CompletableFuture<>(); coordinator.checkpointCoordinator(1, future); - coordinator.notifyCheckpointComplete(1); Map> eventBuffers = SerializationUtils.deserialize(future.get()); // convert to legacy event buffers @@ -200,11 +222,22 @@ public void testRestoreFromLegacyState() throws Exception { eventBuffers.forEach((ckpId, eventBuffer) -> { legacyEventBuffers.put(ckpId, Pair.of(eventBuffer.getLeft(), eventBuffer.getRight().getDataWriteEventBuffer())); }); - // simulate recovering from legacy state + + // Case 1: job restart restores legacy checkpoint data before the coordinator starts. + try (StreamWriteOperatorCoordinator restoredCoordinator = createCoordinator(conf, 2)) { + restoredCoordinator.resetToCheckpoint(1, SerializationUtils.serialize(legacyEventBuffers)); + + EventBuffers.EventBuffer eventBuffer = restoredCoordinator.getEventBuffer(-1); + assertEquals(2, eventBuffer.getDataWriteEventBuffer().length); + assertEquals(0, eventBuffer.getIndexWriteEventBuffer().length); + } + + // Case 2: global failover ignores checkpoint bytes and recommits the live buffers. coordinator.resetToCheckpoint(1, SerializationUtils.serialize(legacyEventBuffers)); - EventBuffers.EventBuffer eventBuffer = coordinator.getEventBuffer(); - assertEquals(2, eventBuffer.getDataWriteEventBuffer().length); - assertEquals(0, eventBuffer.getIndexWriteEventBuffer().length); + + assertNull(coordinator.getEventBuffer()); + assertTrue(StreamerUtil.createMetaClient(conf).reloadActiveTimeline() + .filterCompletedInstants().containsInstant(instant)); } @Test @@ -226,7 +259,7 @@ public void testReceiveInvalidEvent() { public void testEventReset() throws Exception { Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); - coordinator = createCoordinator(conf, 2); + coordinator = startCoordinator(conf, 2); CompletableFuture future = new CompletableFuture<>(); coordinator.checkpointCoordinator(1, future); String instant = requestInstantTime(0); @@ -269,7 +302,7 @@ public void testCheckpointCompleteWithPartialEvents() throws Exception { conf.setString(HoodieWriteConfig.ALLOW_EMPTY_COMMIT.key(), "false"); OperatorCoordinator.Context context = new MockOperatorCoordinatorContext(new OperatorID(), 2); - coordinator = createCoordinator(conf, 2); + coordinator = startCoordinator(conf, 2); final CompletableFuture future = new CompletableFuture<>(); coordinator.checkpointCoordinator(1, future); @@ -315,7 +348,7 @@ public void testRecordLevelIndexFlag(String indexType) throws Exception { conf.set(FlinkOptions.INDEX_TYPE, indexType); conf.set(FlinkOptions.INDEX_WRITE_TASKS, 1); - try (StreamWriteOperatorCoordinator coordinator = createCoordinator(conf, 1)) { + try (StreamWriteOperatorCoordinator coordinator = startCoordinator(conf, 1)) { assertTrue(getRecordLevelIndexFlag(coordinator)); } } @@ -327,7 +360,7 @@ public void testStopHeartbeatForUncommittedEventWithLazyCleanPolicy() throws Exc // override the default configuration Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.setString(HoodieCleanConfig.FAILED_WRITES_CLEANER_POLICY.key(), HoodieFailedWritesCleaningPolicy.LAZY.name()); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); assertTrue(coordinator.getWriteClient().getConfig().getFailedWritesCleanPolicy().isLazy()); @@ -373,7 +406,7 @@ public void testHiveSyncInvoked() throws Exception { // override the default configuration Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.set(FlinkOptions.HIVE_SYNC_ENABLED, true); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); String instant = mockWriteWithMetadata(0); assertNotEquals("", instant); @@ -391,7 +424,7 @@ void testSyncMetadataTable() throws Exception { int metadataCompactionDeltaCommits = 5; conf.set(FlinkOptions.METADATA_ENABLED, true); conf.set(FlinkOptions.METADATA_COMPACTION_DELTA_COMMITS, metadataCompactionDeltaCommits); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); String instant = coordinator.getInstant(); assertEquals("", instant); @@ -468,7 +501,7 @@ void testSyncMetadataTableWithLogCompaction() throws Exception { conf.set(FlinkOptions.METADATA_ENABLED, true); conf.set(FlinkOptions.METADATA_COMPACTION_DELTA_COMMITS, 20); conf.setString("hoodie.metadata.log.compaction.enable", "true"); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); String instant = coordinator.getInstant(); assertEquals("", instant); @@ -512,7 +545,7 @@ void testSyncMetadataTableWithRollback() throws Exception { // override the default configuration Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.set(FlinkOptions.METADATA_ENABLED, true); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); String instant = coordinator.getInstant(); assertEquals("", instant); @@ -536,7 +569,7 @@ void testSyncMetadataTableWithRollback() throws Exception { metadataTableMetaClient.getActiveTimeline().transitionRequestedToInflight(HoodieActiveTimeline.DELTA_COMMIT_ACTION, instant); metadataTableMetaClient.reloadActiveTimeline(); // reset the coordinator to mimic the job failover. - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); // write another commit with new instant on the metadata timeline instant = mockWriteWithMetadata(ckp); @@ -555,7 +588,7 @@ public void testEndInputIsTheLastEvent() throws Exception { Logger logger = Mockito.mock(Logger.class); // avoid too many logs by executor NonThrownExecutor executor = NonThrownExecutor.builder(logger).waitForTasksFinish(true).build(); - try (StreamWriteOperatorCoordinator coordinator = createCoordinator(conf, 1)) { + try (StreamWriteOperatorCoordinator coordinator = startCoordinator(conf, 1)) { coordinator.start(); coordinator.setExecutor(executor); TimeUnit.SECONDS.sleep(5); // wait for handled bootstrap event @@ -593,7 +626,7 @@ void testLockForMetadataTable() throws Exception { conf.setString(HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key(), WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL.name()); conf.setString("hoodie.write.lock.client.num_retries", "1"); - coordinator = createCoordinator(conf, 1); + coordinator = startCoordinator(conf, 1); String instant = coordinator.getInstant(); assertEquals("", instant); @@ -618,7 +651,7 @@ void testLockForMetadataTable() throws Exception { public void testCommitOnEmptyBatch() throws Exception { Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.setString(HoodieWriteConfig.ALLOW_EMPTY_COMMIT.key(), "true"); - try (StreamWriteOperatorCoordinator coordinator = createCoordinator(conf, 2)) { + try (StreamWriteOperatorCoordinator coordinator = startCoordinator(conf, 2)) { // Coordinator start the instant String instant = requestInstantTime(coordinator, -1); @@ -649,7 +682,7 @@ public void testCommitOnEmptyBatch() throws Exception { void testHandleInFlightInstantsRequest() throws Exception { Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); conf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name()); - coordinator = createCoordinator(conf, 2); + coordinator = startCoordinator(conf, 2); // Request an instant time to create an initial instant String instant1 = requestInstantTime(1); @@ -701,15 +734,20 @@ private String requestInstantTime(StreamWriteOperatorCoordinator coordinator, lo } } - private static StreamWriteOperatorCoordinator createCoordinator(Configuration conf, int subTasks) throws Exception { - MockOperatorCoordinatorContext coordinatorContext = new MockOperatorCoordinatorContext(new OperatorID(), subTasks); - StreamWriteOperatorCoordinator coordinator = new StreamWriteOperatorCoordinator(conf, coordinatorContext); + private static StreamWriteOperatorCoordinator startCoordinator(Configuration conf, int subTasks) throws Exception { + StreamWriteOperatorCoordinator coordinator = createCoordinator(conf, subTasks); coordinator.start(); + MockOperatorCoordinatorContext coordinatorContext = (MockOperatorCoordinatorContext) coordinator.getContext(); coordinator.setExecutor(new MockCoordinatorExecutor(coordinatorContext)); coordinator.setInstantRequestExecutor(new MockCoordinatorExecutor(coordinatorContext)); return coordinator; } + private static StreamWriteOperatorCoordinator createCoordinator(Configuration conf, int subTasks) { + MockOperatorCoordinatorContext coordinatorContext = new MockOperatorCoordinatorContext(new OperatorID(), subTasks); + return new StreamWriteOperatorCoordinator(conf, coordinatorContext); + } + private String mockWriteWithMetadata(long checkpointId) { String instant = requestInstantTime(checkpointId); OperatorEvent event = createOperatorEvent(0, checkpointId, instant, "par1", false, true, 0.1); @@ -759,7 +797,7 @@ private static WriteMetadataEvent createOperatorEvent( HoodieWriteStat writeStat = new HoodieWriteStat(); writeStat.setPartitionPath(partitionPath); writeStat.setFileId("fileId123"); - writeStat.setPath("path123"); + writeStat.setPath(partitionPath + "/" + FSUtils.makeBaseFileName(instant, "1-0-1", "fileId123", ".parquet")); writeStat.setFileSizeInBytes(123); writeStat.setTotalWriteBytes(123); writeStat.setNumWrites(1); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java index 5e4deee54c966..a0ddf33eb703d 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java @@ -317,6 +317,41 @@ public void testRecommitOnJobRestartTriggeringGlobalFailover() throws Exception .end(); } + @Test + public void testRecommitOnGlobalFailoverWithStreamingIndex() throws Exception { + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.GLOBAL_RECORD_LEVEL_INDEX.name()); + conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key(), "true"); + conf.setString(HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key(), "true"); + conf.set(FlinkOptions.WRITE_COMMIT_ACK_TIMEOUT, 10_000L); + conf.set(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true); + + Map expected = new HashMap<>(); + expected.put("par1", "[id1,par1,id1,Danny,23,1,par1]"); + expected.put("par2", "[id3,par2,id3,Julian,53,3,par2]"); + + preparePipeline(conf) + .consume(TestData.DATA_SET_PART1) + .emptyEventBuffer() + .checkpoint(1) + .assertNextEvent(1, "par1") + .consume(TestData.DATA_SET_PART3) + .checkpoint(2) + // both ckp-1 and ckp-2 are not committing + .assertNextEvent(1, "par2") + .checkCompletedInstantCount(0) + // Simulate the global failover path. The coordinator resets to ckp-2, recommits ckp-1 + // from the coordinator state, and recommits ckp-2 from the restored writer state. + .jobFailover() + .checkCompletedInstantCount(2) + // This is already the global failover path, so recommit does not need to trigger another failover. + .assertGlobalFailure(false) + .checkIndexLoaded( + new HoodieKey("id1", "par1"), + new HoodieKey("id3", "par2")) + .checkWrittenData(expected, 2) + .end(); + } + @Test public void testInsertDuplicateRecordsWithCDCMode() throws Exception { conf.set(FlinkOptions.WRITE_COMMIT_ACK_TIMEOUT, 10_000L); From 4c1b2a4956e56db76516471405892cdd7f9dc0e1 Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 01:17:31 +0800 Subject: [PATCH 153/255] fix(build): import OperationConverter from its release-branch package TestOperationConverter, added by 86871e04030c (#19224), was picked in at master's path org/apache/hudi/util/ where OperationConverter also lives, so it resolved the class same-package with no import. On release-1.2.1 the class is still at org/apache/hudi/client/utils/ (package org.apache.hudi.client.utils), so the unqualified reference did not resolve and hudi-client-common failed to test-compile with "cannot find symbol: class OperationConverter". Add the explicit import. Master is unaffected: the package move there makes the import unnecessary, not wrong. --- .../test/java/org/apache/hudi/util/TestOperationConverter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java index e8cee32e7f4de..811bea9152197 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/util/TestOperationConverter.java @@ -18,6 +18,7 @@ package org.apache.hudi.util; +import org.apache.hudi.client.utils.OperationConverter; import org.apache.hudi.common.model.WriteOperationType; import org.junit.jupiter.api.Test; From 68b134cfb35caf0b2892ea4f5eac35d6e9e47126 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 11 Jul 2026 13:21:09 +0800 Subject: [PATCH 154/255] =?UTF-8?q?fix(spark):=20read=20INLINE=20blobs=20a?= =?UTF-8?q?s=20CONTENT=20on=20internal=20write-side=20Lance=E2=80=A6=20(#1?= =?UTF-8?q?9236)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(spark): read INLINE blobs as CONTENT on internal write-side Lance reads MOR compaction, clustering, and upsert merge read Lance base files through SparkReaderContextFactory -> SparkFileFormatInternalRowReaderContext -> SparkLanceReaderBase, which resolves hoodie.read.blob.inline.mode from the broadcast Hadoop conf. The factory never set it, so the DESCRIPTOR default applied and rewrites persisted INLINE blobs with null data, silently losing the bytes of every carried-over row (all rows, in the clustering case). - Pin hoodie.read.blob.inline.mode=CONTENT in the conf that SparkReaderContextFactory broadcasts; the factory only serves internal write-side and index reads, never user-facing queries, which build their own conf from session options. - Add a per-row guard in HoodieSparkLanceWriter that rejects the descriptor-leak shape {type=INLINE, data=null, reference!=null} so any future leak fails loudly instead of dropping bytes; {INLINE, null, null} stays writable. - Fix the stale comment in SparkLanceReaderBase claiming CONTENT is the config default (it is DESCRIPTOR). - Un-mask testBlobInlineCompactionRoundTrip by forcing all rows into one file group so untouched rows actually go through the compaction rewrite, and correct its doc about which reader compaction uses. - Add testBlobInlineClusteringRoundTrip and writer-guard tests. Fixes #19232 * test(spark): assert INLINE blob bytes via plain projection before read_blob Assert payload.data under CONTENT mode before calling read_blob() in the compaction and clustering round-trip tests. On a regression the failure now reads as explicit data loss (data is null after the rewrite) instead of read_blob()'s misleading DESCRIPTOR-mode IllegalStateException, which made HoodieSparkLanceReader's CONTENT pin for compaction correctness. * test(spark): assert single file group in blob compaction round-trip test The un-masking of #19232 relies on coalesce(1) plus shuffle parallelism 1 forcing all rows into one file group, but nothing asserted it. Pin the invariant before the anyHadLogs scan so drifting back to multiple file groups (log-free groups compaction never rewrites) fails loudly instead of silently re-masking the regression. * docs(spark): fix DESCRIPTOR-mode comment; transform keeps type=INLINE BlobDescriptorTransform never emits OUT_OF_LINE; it synthesizes the reference sub-struct while preserving type=INLINE. The old wording described the exact wrong mental model that hid #19232. * test(spark): guard CONTENT pin in TestSparkReaderContextFactory Assert hoodie.read.blob.inline.mode=CONTENT on the captured broadcast Configuration so dropping the pin in SparkReaderContextFactory fails a fast unit test instead of only the Lance functional suite. * refactor(spark): hoist blob struct layout into shared BlobStructLayout BlobDescriptorTransform and the HoodieSparkLanceWriter guard each re-derived the BLOB struct ordinals, arities and INLINE token. Move them into a package-private BlobStructLayout holder (arities sourced from HoodieSchema.Blob) so the two decoders cannot drift apart. * docs(spark): correct stale CONTENT-pin attributions for Lance readers The deltaCommits assertion message still credited the pin to HoodieSparkLanceReader; it lives in SparkReaderContextFactory. The test docstring understated HoodieSparkLanceReader's callers (bloom-index lookups and legacy HoodieWriteMergeHandle merges also use it), and the pin comment inside that reader still claimed compaction/merge/log-replay which now go through SparkLanceReaderBase. * fix(spark): widen blob guard error to cover user query round-trips After the SparkReaderContextFactory pin, internal rewrites cannot produce the descriptor shape; the realistic trigger is a user reading a blob table at the DESCRIPTOR default and writing the rows back (INSERT INTO ... SELECT). Stop attributing the leak solely to internal rewrites so that user knows to set hoodie.read.blob.inline.mode=CONTENT on their read. * test(spark): assert clustering completed and rewrote base files in blob inline clustering test Address review on testBlobInlineClusteringRoundTrip. getLastClusteringInstant.isPresent is satisfied by a REQUESTED/INFLIGHT replacecommit (getTimelineOfActions filters by action only), so assert the instant isCompleted. Also snapshot the first commit's base files and assert the post-clustering set is disjoint, proving the rewrite ran rather than the byte assertions reading the untouched originals (the masking mode of #19232). * fix(test): define missing engineContext in latestBaseFileNames helper The latestBaseFileNames helper in testBlobInlineClusteringRoundTrip referenced engineContext without binding it, causing a Scala compile error. Define it locally via HoodieLocalEngineContext(mc.getStorageConf), matching the three other FileSystemViewManager call sites in this file. * docs(test): reword CONTENT-projection comments to the shape they actually backstop The old comments described catching a persisted DESCRIPTOR leak ({INLINE, null data, populated reference}), but validateBlobRow in HoodieSparkLanceWriter now rejects that shape inside the rewrite, so the test would fail at the compaction/clustering write before the projection runs. Reword both sites (compaction + clustering) to the shape the guard deliberately allows, {INLINE, null, null}, and point at TestSparkReaderContextFactory as what pins the CONTENT config. * test(spark): cover CoW upsert merge path for INLINE blob preservation Compaction and clustering resolve their reader via getReaderContextFactory, but a CoW upsert rewrites the base file through FileGroupReaderBasedMergeHandle, which resolves it via getReaderContextFactoryForWrite -- a separate path that branches on the record merger type. No blob test exercised it. Add a CoW round-trip that bulk-inserts INLINE blobs into a single file group, upserts a subset, proves the merge rewrote the base file (disjoint base file names, no deltacommits), and verifies touched rows carry new bytes while untouched rows retain the originals. Hoist latestBaseFileNames out of the clustering test for reuse. (cherry picked from commit 120d73fcc4dba7ddfae0b9693f487190a8410d34) --- .../common/SparkReaderContextFactory.java | 5 + .../io/storage/BlobDescriptorTransform.java | 24 +- .../hudi/io/storage/BlobStructLayout.java | 46 +++ .../io/storage/HoodieSparkLanceReader.java | 6 +- .../io/storage/HoodieSparkLanceWriter.java | 58 ++- .../common/TestSparkReaderContextFactory.java | 6 + .../lance/SparkLanceReaderBase.scala | 8 +- .../storage/TestHoodieSparkLanceWriter.java | 92 +++++ .../hudi/functional/TestLanceDataSource.scala | 379 +++++++++++++++++- 9 files changed, 589 insertions(+), 35 deletions(-) create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobStructLayout.java diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/SparkReaderContextFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/SparkReaderContextFactory.java index 7dd00a9e430c9..f9c99d1db0224 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/SparkReaderContextFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/SparkReaderContextFactory.java @@ -23,6 +23,7 @@ import org.apache.hudi.SparkAdapterSupport$; import org.apache.hudi.SparkFileFormatInternalRowReaderContext; import org.apache.hudi.client.utils.SparkInternalSchemaConverter; +import org.apache.hudi.common.config.HoodieReaderConfig; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.engine.ReaderContextFactory; import org.apache.hudi.common.model.HoodieFileFormat; @@ -89,6 +90,10 @@ public SparkReaderContextFactory(HoodieSparkEngineContext hoodieSparkEngineConte // Broadcast: Configuration. Configuration configs = getHadoopConfiguration(jsc.hadoopConfiguration()); schemaEvolutionConfigs.forEach(configs::set); + // Internal write-side reads (compaction, clustering, merge) must materialize INLINE blob bytes + // so they can be rewritten into the new base file. DESCRIPTOR is a query-only optimization; if + // it leaked onto this path the rewrite would persist null blob data and silently drop the bytes. + configs.set(HoodieReaderConfig.BLOB_INLINE_READ_MODE.key(), HoodieReaderConfig.BLOB_INLINE_READ_MODE_CONTENT); configs.set(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE().key(), sqlConf.getConfString(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE().key())); configs.set(SQLConf.PARQUET_WRITE_LEGACY_FORMAT().key(), sqlConf.getConfString(SQLConf.PARQUET_WRITE_LEGACY_FORMAT().key())); configurationBroadcast = jsc.broadcast(new SerializableConfiguration(configs)); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobDescriptorTransform.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobDescriptorTransform.java index b893811cf28e2..1032c5d368064 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobDescriptorTransform.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobDescriptorTransform.java @@ -18,7 +18,6 @@ package org.apache.hudi.io.storage; -import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.exception.HoodieException; import org.apache.spark.sql.catalyst.InternalRow; @@ -35,6 +34,14 @@ import java.util.HashSet; import java.util.Set; +import static org.apache.hudi.io.storage.BlobStructLayout.DATA_IDX; +import static org.apache.hudi.io.storage.BlobStructLayout.FIELD_COUNT; +import static org.apache.hudi.io.storage.BlobStructLayout.INLINE_UTF8; +import static org.apache.hudi.io.storage.BlobStructLayout.OUT_OF_LINE_UTF8; +import static org.apache.hudi.io.storage.BlobStructLayout.REF_FIELD_COUNT; +import static org.apache.hudi.io.storage.BlobStructLayout.REF_IDX; +import static org.apache.hudi.io.storage.BlobStructLayout.TYPE_IDX; + /** * Per-row transform that rewrites BLOB columns from Lance's DESCRIPTOR shape into the Hudi BLOB * shape. Composed into {@link LanceRecordIterator} for DESCRIPTOR-mode reads. @@ -52,16 +59,6 @@ */ public final class BlobDescriptorTransform { - private static final UTF8String OUT_OF_LINE_UTF8 = - UTF8String.fromString(HoodieSchema.Blob.OUT_OF_LINE); - private static final UTF8String INLINE_UTF8 = - UTF8String.fromString(HoodieSchema.Blob.INLINE); - - // Child field indices within the Hudi BLOB struct: {type(0), data(1), reference(2)}. - private static final int TYPE_IDX = 0; - private static final int DATA_IDX = 1; - private static final int REF_IDX = 2; - private final Set blobFieldNames; private final UTF8String lanceFilePathUtf8; private final String lanceFilePath; @@ -105,7 +102,7 @@ UnsafeRow transformRow(InternalRow row, int rowId, ColumnVector[] columnVectors, for (int i = 0; i < outputFields.length; i++) { if (blobColumnIndices.contains(i)) { rowBuffer[i] = row.isNullAt(i) ? null - : buildBlobOutputRow(row.getStruct(i, 3), columnVectors[i], rowId); + : buildBlobOutputRow(row.getStruct(i, FIELD_COUNT), columnVectors[i], rowId); } else { rowBuffer[i] = row.isNullAt(i) ? null : row.get(i, outputFields[i].dataType()); } @@ -131,7 +128,8 @@ private InternalRow buildBlobOutputRow(InternalRow blobStruct, ColumnVector blob UTF8String type = blobStruct.getUTF8String(TYPE_IDX); if (type.equals(OUT_OF_LINE_UTF8)) { - InternalRow refRow = blobStruct.isNullAt(REF_IDX) ? null : blobStruct.getStruct(REF_IDX, 4); + InternalRow refRow = blobStruct.isNullAt(REF_IDX) ? null + : blobStruct.getStruct(REF_IDX, REF_FIELD_COUNT); return new GenericInternalRow(new Object[] { OUT_OF_LINE_UTF8, null, refRow }); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobStructLayout.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobStructLayout.java new file mode 100644 index 0000000000000..e49e68e4b9c90 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobStructLayout.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage; + +import org.apache.hudi.common.schema.HoodieSchema; + +import org.apache.spark.unsafe.types.UTF8String; + +/** + * Layout of the Hudi BLOB struct {@code {type, data, reference}} as decoded by the Spark-side + * Lance code. Shared by {@link BlobDescriptorTransform} (read) and {@link HoodieSparkLanceWriter} + * (write) so the two cannot drift apart. Ordinals mirror the field order defined by + * {@link HoodieSchema.Blob}. + */ +final class BlobStructLayout { + + // Child field indices within the Hudi BLOB struct: {type(0), data(1), reference(2)}. + static final int TYPE_IDX = 0; + static final int DATA_IDX = 1; + static final int REF_IDX = 2; + static final int FIELD_COUNT = HoodieSchema.Blob.getFieldCount(); + static final int REF_FIELD_COUNT = HoodieSchema.Blob.getReferenceFieldCount(); + + // Precomputed type tokens, compared against each row's type field without per-row allocation. + static final UTF8String INLINE_UTF8 = UTF8String.fromString(HoodieSchema.Blob.INLINE); + static final UTF8String OUT_OF_LINE_UTF8 = UTF8String.fromString(HoodieSchema.Blob.OUT_OF_LINE); + + private BlobStructLayout() { + } +} diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java index a930eaa4b921c..040c63bd01584 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceReader.java @@ -205,8 +205,10 @@ private ClosableIterator getUnsafeRowIterator(HoodieSchema requestedS columnNames.add(field.name()); } - // Pinned to CONTENT: compaction/merge/log-replay need actual bytes to rewrite. - // The user-facing `hoodie.read.blob.inline.mode` is honored by SparkLanceReaderBase. + // Pinned to CONTENT: callers (LanceUtils stats/key reads, bloom-index lookups via + // HoodieReadHandle, old-base-file reads in the legacy HoodieWriteMergeHandle merge path) + // need actual bytes. Default-path compaction/merge reads go through SparkLanceReaderBase + // instead, which honors the user-facing `hoodie.read.blob.inline.mode`. FileReadOptions readOpts = FileReadOptions.builder().blobReadMode(BlobReadMode.CONTENT).build(); // BLOB reads must be chunked to dodge a lance-core FFI abort (see LanceRecordIterator). diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceWriter.java index 3bdf12d9059b9..eb36a1f9a9559 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkLanceWriter.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.io.lance.HoodieBaseLanceWriter; import org.apache.hudi.io.storage.row.HoodieBloomFilterRowWriteSupport; @@ -67,6 +68,11 @@ import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.PARTITION_PATH_METADATA_FIELD; import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.RECORD_KEY_METADATA_FIELD; import static org.apache.hudi.common.util.ValidationUtils.checkArgument; +import static org.apache.hudi.io.storage.BlobStructLayout.DATA_IDX; +import static org.apache.hudi.io.storage.BlobStructLayout.FIELD_COUNT; +import static org.apache.hudi.io.storage.BlobStructLayout.INLINE_UTF8; +import static org.apache.hudi.io.storage.BlobStructLayout.REF_IDX; +import static org.apache.hudi.io.storage.BlobStructLayout.TYPE_IDX; /** * Spark Lance file writer implementing {@link HoodieSparkFileWriter} and {@link HoodieInternalRowFileWriter}. @@ -96,6 +102,10 @@ public class HoodieSparkLanceWriter extends HoodieBaseLanceWriter seqIdGenerator; private final long maxFileSize; + // Top-level BLOB column ordinals and their names (parallel arrays), used by the per-row + // descriptor-leak guard. Empty when the schema has no BLOB columns, making the guard a no-op. + private final int[] blobFieldOrdinals; + private final String[] blobFieldNames; private long recordCountForNextSizeCheck = MIN_RECORDS_FOR_SIZE_CHECK; /** @@ -159,6 +169,20 @@ private HoodieSparkLanceWriter(StoragePath file, super(file, DEFAULT_BATCH_SIZE, allocatorSize, flushByteWatermark, bloomFilterOpt.map(HoodieBloomFilterRowWriteSupport::new)); this.sparkSchema = enrichSparkSchemaForLance(sparkSchema); + StructField[] topFields = this.sparkSchema.fields(); + List blobOrds = new ArrayList<>(); + for (int i = 0; i < topFields.length; i++) { + if (isBlobField(topFields[i])) { + blobOrds.add(i); + } + } + this.blobFieldOrdinals = new int[blobOrds.size()]; + this.blobFieldNames = new String[blobOrds.size()]; + for (int i = 0; i < blobOrds.size(); i++) { + int ord = blobOrds.get(i); + this.blobFieldOrdinals[i] = ord; + this.blobFieldNames[i] = topFields[ord].name(); + } Schema baseArrow = LanceArrowUtils.toArrowSchema(this.sparkSchema, DEFAULT_TIMEZONE, true); // Force LargeBinary + `lance-encoding:blob=true` on each BLOB's nested `data` Arrow leaf. // Can't be expressed Spark-side: toArrowSchema drops nested-field metadata, and tagging @@ -367,7 +391,7 @@ public void writeRow(InternalRow row) throws IOException { @Override protected ArrowWriter createArrowWriter(VectorSchemaRoot root) { - return SparkArrowWriter.of(LanceArrowWriter.create(root, sparkSchema)); + return SparkArrowWriter.of(LanceArrowWriter.create(root, sparkSchema), blobFieldOrdinals, blobFieldNames); } /** @@ -446,12 +470,44 @@ protected void updateRecordMetadata(InternalRow row, @AllArgsConstructor(staticName = "of") private static class SparkArrowWriter implements ArrowWriter { private final LanceArrowWriter lanceArrowWriter; + // Parallel arrays of top-level BLOB column ordinals and names for the per-row guard. + private final int[] blobFieldOrdinals; + private final String[] blobFieldNames; @Override public void write(InternalRow row) { + validateBlobRow(row); lanceArrowWriter.write(row); } + /** + * Reject descriptor-shaped BLOB rows before they are persisted. A row is descriptor-shaped when + * a top-level BLOB struct is non-null, its type is INLINE, its data is null, and its reference + * is non-null; this only arises when a DESCRIPTOR-mode read leaks onto a write path, and writing + * it would silently drop the inline bytes. The legitimate empty-inline shape {INLINE, null, null} + * stays writable. + */ + private void validateBlobRow(InternalRow row) { + for (int i = 0; i < blobFieldOrdinals.length; i++) { + int ordinal = blobFieldOrdinals[i]; + if (row.isNullAt(ordinal)) { + continue; + } + InternalRow blob = row.getStruct(ordinal, FIELD_COUNT); + if (blob.isNullAt(TYPE_IDX) || !INLINE_UTF8.equals(blob.getUTF8String(TYPE_IDX))) { + continue; + } + if (blob.isNullAt(DATA_IDX) && !blob.isNullAt(REF_IDX)) { + throw new HoodieException( + "BLOB column '" + blobFieldNames[i] + "' has an INLINE row with null data but a " + + "populated reference: a DESCRIPTOR-mode read leaked into the write path, and " + + "persisting it would silently drop the blob bytes. Internal rewrites " + + "(compaction, clustering, merge) and any query whose rows are written back " + + "must read with hoodie.read.blob.inline.mode=CONTENT."); + } + } + } + @Override public void reset() { lanceArrowWriter.reset(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestSparkReaderContextFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestSparkReaderContextFactory.java index 77b280cdd91f6..f48df51c47f21 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestSparkReaderContextFactory.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestSparkReaderContextFactory.java @@ -20,6 +20,7 @@ import org.apache.hudi.HoodieSparkUtils; import org.apache.hudi.client.utils.SparkInternalSchemaConverter; +import org.apache.hudi.common.config.HoodieReaderConfig; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.model.ActionType; import org.apache.hudi.common.table.HoodieTableConfig; @@ -113,6 +114,11 @@ void testGetSchemaEvolutionConfigurations() { String inlineClassName = createdConfig.get("fs." + InLineFileSystem.SCHEME + ".impl"); assertEquals(InLineFileSystem.class.getName(), inlineClassName); + // Internal write-side reads must pin CONTENT; a DESCRIPTOR leak here drops blob bytes (#19232). + assertEquals( + HoodieReaderConfig.BLOB_INLINE_READ_MODE_CONTENT, + createdConfig.get(HoodieReaderConfig.BLOB_INLINE_READ_MODE.key())); + assertEquals( "0001_0005.deltacommit,0002_0006.deltacommit,0003_0007.commit", createdConfig.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)); diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala index ddbb798a57115..c653d3f0f9e53 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala @@ -131,10 +131,10 @@ class SparkLanceReaderBase(enableVectorizedReader: Boolean) extends SparkColumna null } - // Honor `hoodie.read.blob.inline.mode`. CONTENT (default) materializes INLINE bytes in - // the `data` column; DESCRIPTOR surfaces per-row {position, size} which the descriptor - // iterator rewrites into Hudi OUT_OF_LINE references. Non-blob Lance columns ignore - // the option regardless. + // Honor `hoodie.read.blob.inline.mode`. DESCRIPTOR (default) surfaces per-row + // {position, size} which the descriptor iterator turns into a synthesized `reference` + // while leaving `type = INLINE`; CONTENT is the opt-in mode that materializes INLINE + // bytes in the `data` column. Non-blob Lance columns ignore the option regardless. val blobMode = resolveBlobReadMode(storageConf) val readOpts = FileReadOptions.builder().blobReadMode(blobMode).build() diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java index 722833a7756c0..7f245f65ef6df 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.io.memory.HoodieArrowAllocator; import org.apache.hudi.storage.HoodieStorage; @@ -639,4 +640,95 @@ public void testValidateNoVariantColumns_variantInNullableUnion_throws() { () -> HoodieSparkLanceWriter.validateNoVariantColumns(record)); assertTrue(ex.getMessage().contains("payload"), "Error should name the field: " + ex.getMessage()); } + + // ----- INLINE blob descriptor-leak guard tests ----- + + /** + * Builds a two-column schema (id INT + a canonical BLOB column) so the writer recognizes the + * second column as a blob via the {@code hudi_type=BLOB} field metadata. The struct layout is + * the canonical one produced by {@link org.apache.spark.sql.types.BlobType}: type, data, + * reference. + */ + private StructType createBlobSchema() { + StructType blobStruct = (StructType) org.apache.spark.sql.types.BlobType.dataType(); + Metadata blobMetadata = new MetadataBuilder() + .putString(HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name()) + .build(); + return new StructType() + .add(new StructField("id", DataTypes.IntegerType, false, Metadata.empty())) + .add(new StructField("payload", blobStruct, true, blobMetadata)); + } + + /** + * Builds the reference sub-struct {external_path, offset, length, managed}. + */ + private InternalRow blobReference(String externalPath, Long offset, Long length, boolean managed) { + return new GenericInternalRow(new Object[] { + externalPath == null ? null : UTF8String.fromString(externalPath), + offset, + length, + managed + }); + } + + /** + * Writing an INLINE blob with null {@code data} but a non-null {@code reference} is the + * descriptor-leak shape: it means an internal write-side read handed the writer a DESCRIPTOR row + * (reference populated, bytes dropped) instead of CONTENT. The writer must reject it with a + * {@link HoodieException} that points at {@code hoodie.read.blob.inline.mode}, rather than + * silently persisting a blob with no bytes (#19232). + */ + @Test + public void testWriteInlineBlobWithNullDataAndReference_throws() { + StructType schema = createBlobSchema(); + StoragePath path = new StoragePath(tempDir.getAbsolutePath() + "/test_blob_descriptor_leak.lance"); + + InternalRow reference = blobReference("s3://bucket/blob.bin", 0L, 1024L, false); + InternalRow blob = new GenericInternalRow(new Object[] { + UTF8String.fromString(HoodieSchema.Blob.INLINE), // type = INLINE + null, // data = null (leaked) + reference // reference = non-null (leaked) + }); + InternalRow row = new GenericInternalRow(new Object[] {1, blob}); + + HoodieException ex = assertThrows(HoodieException.class, () -> { + try (HoodieSparkLanceWriter writer = HoodieSparkLanceWriter.builder() + .file(path).sparkSchema(schema).instantTime(instantTime).taskContextSupplier(taskContextSupplier) + .storage(storage).bloomFilterOpt(Option.of(simpleBloomFilter)).build()) { + writer.writeRow("key1", row); + } + }); + assertTrue(ex.getMessage() != null && ex.getMessage().contains("hoodie.read.blob.inline.mode"), + "Descriptor-leak guard must reference hoodie.read.blob.inline.mode: " + ex.getMessage()); + } + + /** + * An INLINE blob with null {@code data} AND null {@code reference} is a legitimate null inline + * payload, not a descriptor leak. The writer must accept it and close cleanly. + */ + @Test + public void testWriteInlineBlobWithNullDataAndNullReference_succeeds() throws Exception { + StructType schema = createBlobSchema(); + StoragePath path = new StoragePath(tempDir.getAbsolutePath() + "/test_blob_null_inline.lance"); + + InternalRow blob = new GenericInternalRow(new Object[] { + UTF8String.fromString(HoodieSchema.Blob.INLINE), // type = INLINE + null, // data = null (legit null payload) + null // reference = null + }); + InternalRow row = new GenericInternalRow(new Object[] {1, blob}); + + try (HoodieSparkLanceWriter writer = HoodieSparkLanceWriter.builder() + .file(path).sparkSchema(schema).instantTime(instantTime).taskContextSupplier(taskContextSupplier) + .storage(storage).bloomFilterOpt(Option.of(simpleBloomFilter)).build()) { + writer.writeRow("key1", row); + } + + assertTrue(storage.exists(path), "Lance file with a null INLINE payload should be written"); + try (BufferAllocator allocator = HoodieArrowAllocator.newChildAllocator( + "testWriteInlineBlobWithNullDataAndNullReference", TEST_LANCE_DATA_ALLOCATOR_SIZE); + LanceFileReader reader = LanceFileReader.open(path.toString(), allocator)) { + assertEquals(1, reader.numRows(), "The single null-payload row should be written"); + } + } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala index 419a7bcf64984..b281b83bf6c54 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLanceDataSource.scala @@ -1303,12 +1303,22 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { /** * Compaction must preserve INLINE blob bytes under the DESCRIPTOR default. MOR compaction reads - * the base file via {@link HoodieSparkLanceReader}, which hard-pins CONTENT regardless of the - * user-facing {@code hoodie.read.blob.inline.mode}. If that pin were to honor the default - * (DESCRIPTOR), compaction would read null {@code data} and rewrite a base file without bytes, - * silently corrupting untouched rows. This test inserts INLINE blobs, upserts a subset to force - * compaction, and asserts that touched rows carry the new bytes while untouched rows retain the - * originals. + * the base file through the internal write-side reader stack + * SparkReaderContextFactory -> SparkFileFormatInternalRowReaderContext -> SparkLanceReaderBase, + * not through HoodieSparkLanceReader (that reader serves LanceUtils stats/key reads, + * bloom-index key lookups, and legacy HoodieWriteMergeHandle merges, with its own CONTENT pin). + * SparkLanceReaderBase honors {@code hoodie.read.blob.inline.mode}, whose DESCRIPTOR default + * would read null {@code data} and rewrite a base file without bytes, silently corrupting + * untouched rows. Correctness now relies on SparkReaderContextFactory pinning + * {@code hoodie.read.blob.inline.mode=CONTENT} in the broadcast conf used by all internal reads. + * User-facing queries are unaffected because they build their own conf. + * + * The test forces all rows into a single file group (coalesce(1) plus bulk-insert/insert + * shuffle parallelism 1) so the untouched rows genuinely go through the compaction rewrite. + * Without that, untouched rows land in log-free file groups that compaction never rewrites, + * which is how the original bug (#19232) stayed masked. This test inserts INLINE blobs, + * upserts a subset to force compaction, and asserts that touched rows carry the new bytes while + * untouched rows retain the originals. */ @Test def testBlobInlineCompactionRoundTrip(): Unit = { @@ -1332,7 +1342,7 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { def asInlineDf(idToBytes: Seq[(Int, Array[Byte])]): DataFrame = { val rawDf = idToBytes.toDF("id", "bytes") .select($"id", BlobTestHelpers.inlineBlobStructCol("payload", $"bytes")) - spark.createDataFrame(rawDf.rdd, canonicalSchema) + spark.createDataFrame(rawDf.rdd, canonicalSchema).coalesce(1) } // First commit: bulk_insert ids 0..5 with the initial pattern. Lands in a base file. @@ -1340,7 +1350,9 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { asInlineDf(initialPayloads.zipWithIndex.map { case (b, i) => (i, b) }), saveMode = SaveMode.Overwrite, operation = Some("bulk_insert"), - extraOptions = Map(PRECOMBINE_FIELD.key() -> "id")) + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id", + "hoodie.bulkinsert.shuffle.parallelism" -> "1", + "hoodie.insert.shuffle.parallelism" -> "1")) assertLanceBlobEncoding(tablePath) @@ -1366,9 +1378,9 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { .getInstants.asScala val deltaCommits = completedInstants.filter(_.getAction == "deltacommit") assertTrue(deltaCommits.nonEmpty, - "Upsert must have written a deltacommit on MOR — without log files the compaction " + + "Upsert must have written a deltacommit on MOR -- without log files the compaction " + "round-trip below would be a no-op and the test would silently pass even if the " + - "CONTENT-pin in HoodieSparkLanceReader were broken.") + "CONTENT-pin in SparkReaderContextFactory were broken.") val compactionCommits = completedInstants.filter(_.getAction == "commit") assertTrue(compactionCommits.nonEmpty, "Compaction commit should be present after upsert") @@ -1386,6 +1398,13 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { val fsView = viewManager.getFileSystemView(metaClient) try { fsView.loadAllPartitions() + // Pin the single-file-group invariant the whole test rests on. If rows ever spread across + // multiple file groups, the untouched ids could sit in log-free groups that compaction never + // rewrites, and the round-trip below would pass without exercising the regression (#19232). + assertEquals(1L, fsView.getAllFileGroups("").count(), + s"All rows must land in exactly one file group (coalesce(1) + shuffle parallelism 1) " + + s"at $tablePath; otherwise untouched ids may sit in log-free file groups that " + + s"compaction never rewrites and the regression is not exercised") val anyHadLogs = fsView.getAllFileGroups("").iterator().asScala.exists { fg => fg.getAllFileSlices.iterator().asScala.exists(_.hasLogFiles) } @@ -1422,11 +1441,34 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { s"DESCRIPTOR default should populate reference on plain read (id=$id)") } - // read_blob() under CONTENT mode is what we use to verify the post-compaction bytes - // because read_blob() on INLINE rows throws under the DESCRIPTOR default. The bytes can - // only come back if HoodieSparkLanceReader's CONTENT pin held during the compactor's - // base-file read — otherwise untouched ids 3..5 would have been rewritten with null - // `data` and CONTENT-mode read would surface that. + // Byte check via a plain projection under CONTENT. A broken rewrite could produce two shapes: + // - {INLINE, null data, populated reference}: a DESCRIPTOR-mode read leaked into the write + // path. HoodieSparkLanceWriter.validateBlobRow rejects this shape and fails the compaction + // itself, so it can never reach the base file. + // - {INLINE, null data, null reference}: legitimate for an empty inline blob, so the writer + // guard lets it through. If a rewrite dropped the bytes this way, only the null-data + // assertion below would catch it. + // The CONTENT pin on internal reads is unit-tested in TestSparkReaderContextFactory. + val contentRows = spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .select($"id", $"payload") + .orderBy($"id") + .collect() + assertEquals(numRows, contentRows.length) + contentRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val payload = row.getStruct(row.fieldIndex("payload")) + assertFalse(payload.isNullAt(payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"null data under CONTENT: the compaction rewrite dropped the bytes (id=$id)") + assertArrayEquals(expected(id), + payload.getAs[Array[Byte]](payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"INLINE data bytes must survive the compaction rewrite (id=$id)") + } + + // read_blob() under CONTENT verifies the same bytes through the SQL expression path + // (read_blob() on INLINE rows throws under the DESCRIPTOR default, so CONTENT is + // required here). val viewName = s"${tableName}_view" spark.read.format("hudi") .option("hoodie.read.blob.inline.mode", "CONTENT") @@ -1443,6 +1485,313 @@ class TestLanceDataSource extends HoodieSparkClientTestBase { } } + /** + * Clustering must preserve INLINE blob bytes under the DESCRIPTOR default. Clustering rewrites + * ALL rows through MultipleSparkJobExecutionStrategy.readRecordsForGroupAsRow, which reads base + * files through the same internal write-side reader context as compaction + * (SparkReaderContextFactory -> SparkFileFormatInternalRowReaderContext -> SparkLanceReaderBase). + * Before SparkReaderContextFactory pinned {@code hoodie.read.blob.inline.mode=CONTENT} for + * internal reads, the first clustering of a Lance table with INLINE blobs read null {@code data} + * (the DESCRIPTOR default) and rewrote every row's blob with null bytes, silently losing all + * blob content (#19232). This test bulk-inserts INLINE blobs, triggers inline clustering, + * asserts a replacecommit actually completed, and verifies every row's bytes survived. + */ + @Test + def testBlobInlineClusteringRoundTrip(): Unit = { + val tableType = HoodieTableType.COPY_ON_WRITE + val tableName = "test_lance_blob_inline_cluster_cow" + val tablePath = s"$basePath/$tableName" + + val payloadLen = 1024 + val numRows = 5 + val expectedPayloads: Seq[Array[Byte]] = (0 until numRows).map { i => + (0 until payloadLen).map(j => ((i + j) % 256).toByte).toArray + } + val sparkSess = spark + import sparkSess.implicits._ + + val canonicalSchema = StructType(Seq( + StructField("id", IntegerType, nullable = false), + StructField("payload", BlobType().asInstanceOf[StructType], nullable = true, + BlobTestHelpers.blobMetadata) + )) + def asInlineDf(idToBytes: Seq[(Int, Array[Byte])]): DataFrame = { + val rawDf = idToBytes.toDF("id", "bytes") + .select($"id", BlobTestHelpers.inlineBlobStructCol("payload", $"bytes")) + spark.createDataFrame(rawDf.rdd, canonicalSchema) + } + + // First commit: bulk_insert ids 0..4 with the initial pattern into a base file. + writeDataframe(tableType, tableName, tablePath, + asInlineDf(expectedPayloads.zipWithIndex.map { case (b, i) => (i, b) }), + saveMode = SaveMode.Overwrite, + operation = Some("bulk_insert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id")) + + assertLanceBlobEncoding(tablePath) + + // Snapshot the first commit's base file(s); the disjointness check below uses them to prove + // clustering rewrote (not skipped) them, else the byte checks pass on stale bytes (#19232). + val metaClientAfterFirst = HoodieTableMetaClient.builder() + .setConf(HoodieTestUtils.getDefaultStorageConf) + .setBasePath(tablePath) + .build() + val preClusterBaseFiles = latestBaseFileNames(metaClientAfterFirst, tablePath) + assertFalse(preClusterBaseFiles.isEmpty, "First commit should have written at least one base file") + + // Second commit: a small bulk_insert that trips inline clustering (max.commits=1). Clustering + // rewrites the existing base file's rows through readRecordsForGroupAsRow, which must read the + // INLINE bytes as CONTENT, not the DESCRIPTOR default, or every rewritten row loses its bytes. + val extraPayloads = (numRows until numRows + 2).map { i => + (i, (0 until payloadLen).map(j => ((i + j) % 256).toByte).toArray) + } + writeDataframe(tableType, tableName, tablePath, + asInlineDf(extraPayloads), + operation = Some("bulk_insert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id", + "hoodie.clustering.inline" -> "true", + "hoodie.clustering.inline.max.commits" -> "1")) + + // Require a COMPLETED replacecommit. getLastClusteringInstant filters by action only, so a + // REQUESTED/INFLIGHT instant satisfies isPresent; isCompleted confirms the rewrite finished. + val metaClient = HoodieTableMetaClient.builder() + .setConf(HoodieTestUtils.getDefaultStorageConf) + .setBasePath(tablePath) + .build() + val lastClustering = metaClient.getActiveTimeline.getLastClusteringInstant + assertTrue(lastClustering.isPresent && lastClustering.get.isCompleted, + "A COMPLETED clustering (replacecommit) instant must exist after inline clustering; without a " + + "completed rewrite the blob-loss regression below could not be exercised") + + // ...and that it rewrote the base file(s) into new ones. Disjoint sets prove the rewrite ran + // instead of the byte checks reading untouched originals (#19232). + val postClusterBaseFiles = latestBaseFileNames(metaClient, tablePath) + assertTrue(preClusterBaseFiles.intersect(postClusterBaseFiles).isEmpty, + s"Post-clustering base files must be disjoint from the pre-clustering base file(s), proving the " + + s"rewrite ran (pre=$preClusterBaseFiles, post=$postClusterBaseFiles)") + + // Read back in CONTENT mode and assert every row's bytes survived the clustering rewrite. + val allExpected: Map[Int, Array[Byte]] = + (expectedPayloads.zipWithIndex.map { case (b, i) => i -> b } ++ extraPayloads).toMap + + // Byte check via a plain projection under CONTENT. As in the compaction test: a DESCRIPTOR + // leak already fails the rewrite in HoodieSparkLanceWriter.validateBlobRow, so what this + // catches is the guard-allowed empty-inline shape {INLINE, null data, null reference}, + // where dropped bytes would persist silently. + val contentRows = spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .select($"id", $"payload") + .orderBy($"id") + .collect() + assertEquals(allExpected.size, contentRows.length) + contentRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val payload = row.getStruct(row.fieldIndex("payload")) + assertFalse(payload.isNullAt(payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"null data under CONTENT: the clustering rewrite dropped the bytes (id=$id)") + assertArrayEquals(allExpected(id), + payload.getAs[Array[Byte]](payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"INLINE data bytes must survive the clustering rewrite (id=$id)") + } + + val viewName = s"${tableName}_view" + spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .createOrReplaceTempView(viewName) + val materialized = spark.sql( + s"SELECT id, read_blob(payload) AS bytes FROM $viewName ORDER BY id").collect() + assertEquals(allExpected.size, materialized.length) + materialized.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val bytes = row.getAs[Array[Byte]]("bytes") + assertArrayEquals(allExpected(id), bytes, + s"read_blob() must return correct bytes post-clustering (id=$id)") + } + } + + /** + * A CoW upsert merge must preserve INLINE blob bytes under the DESCRIPTOR default. + * + * Compaction and clustering (covered above) obtain their reader through + * {@code HoodieEngineContext.getReaderContextFactory}. A CoW upsert takes a different path: it + * rewrites the base file through FileGroupReaderBasedMergeHandle, which resolves its reader + * through {@code getReaderContextFactoryForWrite}. That method branches on the record merger + * type: AvroReaderContextFactory for AVRO, SparkReaderContextFactory for SPARK (the datasource + * default, used here). A DESCRIPTOR leak on this branch would rewrite untouched rows with null + * {@code data}, the same silent loss as #19232. + * + * The test bulk-inserts INLINE blobs into a single file group, upserts a subset, proves the + * merge rewrote the base file, and verifies touched rows carry the new bytes while untouched + * rows keep the originals. + */ + @Test + def testBlobInlineCowUpsertMergeRoundTrip(): Unit = { + val tableType = HoodieTableType.COPY_ON_WRITE + val tableName = "test_lance_blob_inline_upsert_merge_cow" + val tablePath = s"$basePath/$tableName" + + val payloadLen = 1024 + val numRows = 6 + val initialPayloads: Seq[Array[Byte]] = (0 until numRows).map { i => + (0 until payloadLen).map(j => ((i + j) % 256).toByte).toArray + } + val sparkSess = spark + import sparkSess.implicits._ + + val canonicalSchema = StructType(Seq( + StructField("id", IntegerType, nullable = false), + StructField("payload", BlobType().asInstanceOf[StructType], nullable = true, + BlobTestHelpers.blobMetadata) + )) + def asInlineDf(idToBytes: Seq[(Int, Array[Byte])]): DataFrame = { + val rawDf = idToBytes.toDF("id", "bytes") + .select($"id", BlobTestHelpers.inlineBlobStructCol("payload", $"bytes")) + spark.createDataFrame(rawDf.rdd, canonicalSchema).coalesce(1) + } + + // First commit: bulk_insert ids 0..5 into a single base file. A single file group is required + // so the untouched ids 3..5 genuinely pass through the merge rewrite; in their own group the + // upsert would never touch them and the byte checks below would pass vacuously. + writeDataframe(tableType, tableName, tablePath, + asInlineDf(initialPayloads.zipWithIndex.map { case (b, i) => (i, b) }), + saveMode = SaveMode.Overwrite, + operation = Some("bulk_insert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id", + "hoodie.bulkinsert.shuffle.parallelism" -> "1", + "hoodie.insert.shuffle.parallelism" -> "1")) + + assertLanceBlobEncoding(tablePath) + + val metaClientAfterFirst = HoodieTableMetaClient.builder() + .setConf(HoodieTestUtils.getDefaultStorageConf) + .setBasePath(tablePath) + .build() + val preUpsertBaseFiles = latestBaseFileNames(metaClientAfterFirst, tablePath) + assertEquals(1, preUpsertBaseFiles.size, + s"All rows must land in exactly one base file (coalesce(1) + shuffle parallelism 1) at " + + s"$tablePath, got $preUpsertBaseFiles; otherwise untouched ids sit in file groups the " + + s"upsert never rewrites and the merge path is not exercised") + + // Second commit: upsert ids 0..2 with all-0xEE payloads. On CoW this routes every existing + // file-group record through the merge handle's CONTENT-pinned base-file read and rewrite. + val updatedPayloadByte: Byte = 0xEE.toByte + val updatedIds = 0 until 3 + val updatedPayloads = updatedIds.map(i => (i, Array.fill[Byte](payloadLen)(updatedPayloadByte))) + writeDataframe(tableType, tableName, tablePath, + asInlineDf(updatedPayloads), + operation = Some("upsert"), + extraOptions = Map(PRECOMBINE_FIELD.key() -> "id")) + + // The upsert must have stayed on the CoW commit path: two completed commits, no deltacommits + // (a deltacommit would mean an append path that never rewrites the base file). + val metaClient = HoodieTableMetaClient.builder() + .setConf(HoodieTestUtils.getDefaultStorageConf) + .setBasePath(tablePath) + .build() + val completedInstants = metaClient.reloadActiveTimeline().filterCompletedInstants() + .getInstants.asScala + assertEquals(2, completedInstants.count(_.getAction == "commit"), + "Expected exactly two completed commits (bulk_insert + upsert) on CoW") + assertTrue(completedInstants.forall(_.getAction != "deltacommit"), + "CoW upsert must not write deltacommits; the merge rewrite would not be exercised") + + // The merge must also have replaced the base file: a single new name, disjoint from the + // pre-upsert one. If the old name were still the latest, the untouched ids were never + // merged and the byte checks below would read stale bytes. + val postUpsertBaseFiles = latestBaseFileNames(metaClient, tablePath) + assertEquals(1, postUpsertBaseFiles.size, + s"Upsert must keep all rows in one file group, got $postUpsertBaseFiles") + assertTrue(preUpsertBaseFiles.intersect(postUpsertBaseFiles).isEmpty, + s"Post-upsert base file must differ from the pre-upsert one, proving the merge rewrote it " + + s"(pre=$preUpsertBaseFiles, post=$postUpsertBaseFiles)") + + val expected: Map[Int, Array[Byte]] = ( + updatedIds.map(i => i -> Array.fill[Byte](payloadLen)(updatedPayloadByte)) ++ + (updatedIds.length until numRows).map(i => i -> initialPayloads(i)) + ).toMap + + // Plain read yields the DESCRIPTOR shape, confirming the user-facing default end-to-end. + val readRows = spark.read.format("hudi") + .load(tablePath) + .select($"id", $"payload") + .orderBy($"id") + .collect() + assertEquals(numRows, readRows.length) + readRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val payload = row.getStruct(row.fieldIndex("payload")) + assertEquals(HoodieSchema.Blob.INLINE, + payload.getString(payload.fieldIndex(HoodieSchema.Blob.TYPE)), + s"Type must remain INLINE post-merge (id=$id)") + assertTrue(payload.isNullAt(payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"DESCRIPTOR default should null `data` on plain read (id=$id)") + assertNotNull(payload.getStruct(payload.fieldIndex(HoodieSchema.Blob.EXTERNAL_REFERENCE)), + s"DESCRIPTOR default should populate reference on plain read (id=$id)") + } + + // Byte check via a plain projection under CONTENT. As in the compaction test: a DESCRIPTOR + // leak already fails the merge in HoodieSparkLanceWriter.validateBlobRow, so what this + // catches is the guard-allowed empty-inline shape {INLINE, null data, null reference}, + // where dropped bytes would persist silently. + val contentRows = spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .select($"id", $"payload") + .orderBy($"id") + .collect() + assertEquals(numRows, contentRows.length) + contentRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val payload = row.getStruct(row.fieldIndex("payload")) + assertFalse(payload.isNullAt(payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"null data under CONTENT: the merge rewrite dropped the bytes (id=$id)") + assertArrayEquals(expected(id), + payload.getAs[Array[Byte]](payload.fieldIndex(HoodieSchema.Blob.INLINE_DATA_FIELD)), + s"INLINE data bytes must survive the merge rewrite (id=$id)") + } + + // read_blob() under CONTENT verifies the same bytes through the SQL expression path. + val viewName = s"${tableName}_view" + spark.read.format("hudi") + .option("hoodie.read.blob.inline.mode", "CONTENT") + .load(tablePath) + .createOrReplaceTempView(viewName) + val materializedRows = spark.sql( + s"SELECT id, read_blob(payload) AS bytes FROM $viewName ORDER BY id").collect() + assertEquals(numRows, materializedRows.length) + materializedRows.foreach { row => + val id = row.getInt(row.fieldIndex("id")) + val bytes = row.getAs[Array[Byte]]("bytes") + assertArrayEquals(expected(id), bytes, + s"read_blob() must return correct bytes post-merge (id=$id)") + } + } + + /** + * Latest base file name per file group under {@code tablePath}. Snapshots taken before and + * after a rewriting table service (clustering, CoW upsert merge) are compared for disjointness + * to prove the rewrite actually replaced the base file(s). + */ + private def latestBaseFileNames(mc: HoodieTableMetaClient, tablePath: String): Set[String] = { + val engineContext = new HoodieLocalEngineContext(mc.getStorageConf) + val metadataConfig = HoodieMetadataConfig.newBuilder.build + val viewManager = FileSystemViewManager.createViewManager( + engineContext, metadataConfig, FileSystemViewStorageConfig.newBuilder.build, + HoodieCommonConfig.newBuilder.build, + (m: HoodieTableMetaClient) => mc.getTableFormat + .getMetadataFactory.create(engineContext, m.getStorage, metadataConfig, tablePath)) + val fsView = viewManager.getFileSystemView(mc) + try { + fsView.getLatestBaseFiles("") + .collect(Collectors.toList[org.apache.hudi.common.model.HoodieBaseFile]) + .asScala.map(_.getFileName).toSet + } finally { + fsView.close() + } + } + /** * Shared implementation for OUT_OF_LINE blob tests. Writes rows with external references, * reads them back (optionally with a specific read mode), and asserts the reference survives From 443685080964a026f3261b9ca9b217a87f330b57 Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 01:22:49 +0800 Subject: [PATCH 155/255] fix(build): import lock result enums from their release-branch package TestLockResultEnums, added by 86871e04030c (#19224), references LockGetResult and LockUpsertResult unqualified. On master those enums sit in org.apache.hudi.client.transaction.lock, the same package as the test, so no import was needed. On release-1.2.1 they are still in the .models subpackage (org.apache.hudi.client.transaction.lock.models), so the references did not resolve and hudi-client-common failed to test-compile with "package LockGetResult does not exist". Add both imports. The enum shapes match what the test asserts: LockGetResult NOT_EXISTS(0)/SUCCESS(1)/UNKNOWN_ERROR(2) and LockUpsertResult SUCCESS(0)/ACQUIRED_BY_OTHERS(1)/UNKNOWN_ERROR(2)/ THROTTLED(3), with getCode() generated by Lombok @Getter. Same root cause as 46d3c8494aa2. --- .../hudi/client/transaction/lock/TestLockResultEnums.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java index 51e944c214a95..41caaa78575ce 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestLockResultEnums.java @@ -18,6 +18,9 @@ package org.apache.hudi.client.transaction.lock; +import org.apache.hudi.client.transaction.lock.models.LockGetResult; +import org.apache.hudi.client.transaction.lock.models.LockUpsertResult; + import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; From 9049cc59dbc4277393a7c0f2a7d5be21ec31941f Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 01:28:11 +0800 Subject: [PATCH 156/255] fix(build): import Spark Metadata/StructField types in TestHoodieSparkLanceWriter The createBlobSchema() helper added by 120d73fcc4db (#19236) uses Metadata, MetadataBuilder and StructField, but the commit did not add their imports: master's copy of this test already carried them from earlier commits, so upstream had nothing to add. release-1.2.1's copy only imported DataTypes and StructType, so the backported helper did not compile ("cannot find symbol: class Metadata"). Add the three org.apache.spark.sql.types imports. This is a context dependency the cherry-pick could not carry, not a conflict-resolution error. --- .../org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java index 7f245f65ef6df..122834c46870a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java @@ -48,6 +48,9 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; import org.junit.jupiter.api.AfterEach; From 04b5c0ec2fb54b79019637cc46beb6f563f4c6cc Mon Sep 17 00:00:00 2001 From: Lin Liu <141371752+linliu-code@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:42:02 -0700 Subject: [PATCH 157/255] Remove spaces around partition columns (#18423) (cherry picked from commit 541ab01b9f5a2ef039cfb90738541329fbb91bc6) --- .../java/org/apache/hudi/common/table/HoodieTableConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index aef84c9f8adc9..76f687b7963a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -730,6 +730,7 @@ public static Option getPartitionFieldProp(HoodieConfig config) { public static Option getPartitionFields(HoodieConfig config) { if (contains(PARTITION_FIELDS, config)) { return Option.of(Arrays.stream(config.getString(PARTITION_FIELDS).split(BaseKeyGenerator.FIELD_SEPARATOR)) + .map(String::trim) .filter(p -> !p.isEmpty()) .map(p -> getPartitionFieldWithoutKeyGenPartitionType(p, config)) .collect(Collectors.toList()).toArray(new String[] {})); From 0068f21c085ff5b93eb462a883f3a55ea2dedce4 Mon Sep 17 00:00:00 2001 From: Surya Prasanna Date: Mon, 13 Jul 2026 03:08:27 -0700 Subject: [PATCH 158/255] fix: align log4j2 and slf4j versions to resolve IntelliJ test failures (#18177) * Fix no classdef errors on intellij * fix: bind slf4j 2.x by using the log4j-slf4j2-impl binding Raising slf4j.version to 2.0.7 in the root properties leaves every build that does not activate a spark profile (a plain `mvn` build, and all -Dflink* builds) pairing slf4j-api 2.0.7 with log4j-slf4j-impl. That artifact only ships org/slf4j/impl/StaticLoggerBinder, which is the slf4j 1.7 binding; slf4j 2.x discovers providers through META-INF/services/org.slf4j.spi.SLF4JServiceProvider and therefore finds none: SLF4J: No SLF4J providers were found. SLF4J: Defaulting to no-operation (NOP) logger implementation SLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier. All logging would silently go to the NOP logger, which no test asserts on. Swap the binding to log4j-slf4j2-impl (the log4j2 binding for slf4j 2.x, available since log4j 2.19) in the root dependencyManagement, in every module that declares it, and in the hudi-metaserver-server-bundle shade includes. Spark builds already pull log4j-slf4j2-impl transitively, so this only makes the declared binding match what slf4j 2.x can actually load. No source change is needed: Hudi uses no slf4j 2.x-only API. * fix: keep hive's slf4j 1.x binding off the classpath hive-shims-common pulls in log4j-slf4j-impl 2.17.2, and that artifact ships the same classes as log4j-slf4j2-impl (org.apache.logging.slf4j.Log4jLoggerFactory, Log4jLogger, Log4jMarkerFactory), so both jars on one classpath means the shared classes are resolved from whichever jar comes first, with no Maven mediation. This was masked until now: the root dependencyManagement pinned log4j-slf4j-impl to ${log4j2.version}, so hive's copy was force-upgraded and its duplicate Log4jLoggerFactory happened to carry the same (Log4jMarkerFactory) constructor. Once that entry became log4j-slf4j2-impl the pin was gone, hive's 2.17.2 copy came in with only a no-arg constructor, and SLF4JServiceProvider.initialize() failed with NoSuchMethodError: Log4jLoggerFactory.(Log4jMarkerFactory) surfacing as ExceptionInInitializerError / NoClassDefFoundError on every class with a static logger, e.g. HoodieConfig. Exclude log4j-slf4j-impl from the four hive deps that leak it (hive-shims, hive-jdbc, hive-serde, hive-metastore); hive-exec, hive-common and hive-service already exclude org.apache.logging.log4j:* wholesale. This mirrors the existing exclusions for slf4j-log4j12, hive's other slf4j 1.x binding. * fix: keep spark 3.3's slf4j 1.x binding off the classpath Spark only moved to log4j-slf4j2-impl in 3.4, so spark-core 3.3.x still brings log4j-slf4j-impl. That is the same duplicate-class collision already fixed for hive: both binding artifacts ship org.apache.logging.slf4j.Log4jLoggerFactory, so the older copy shadows the one log4j-slf4j2-impl expects and SLF4JServiceProvider.initialize() dies with NoSuchMethodError: Log4jLoggerFactory.(Log4jMarkerFactory) Exclude log4j-slf4j-impl from both spark-core entries in dependencyManagement (the main one and the tests classifier), alongside the org.slf4j:* and log4j:log4j exclusions already there. Spark 3.4+ is unaffected: it ships the slf4j2 binding already. Verified there is no remaining log4j-slf4j-impl on the tree for spark3.3, spark3.4, spark3.5, spark4.0, spark4.1, spark4.2, flink1.18, flink1.19, flink1.20, flink2.0, flink2.1 and the default profile. --------- Co-authored-by: voon (cherry picked from commit bbe73d3e2ee4771b0fdaae39c7f5250e626c6c74) --- hudi-client/hudi-flink-client/pom.xml | 2 +- hudi-flink-datasource/hudi-flink/pom.xml | 2 +- .../hudi-flink1.18.x/pom.xml | 2 +- .../hudi-flink1.19.x/pom.xml | 2 +- .../hudi-flink1.20.x/pom.xml | 2 +- hudi-flink-datasource/hudi-flink2.0.x/pom.xml | 2 +- hudi-flink-datasource/hudi-flink2.1.x/pom.xml | 2 +- hudi-io/pom.xml | 2 +- hudi-platform-service/hudi-metaserver/pom.xml | 2 +- hudi-sync/hudi-adb-sync/pom.xml | 2 +- hudi-tests-common/pom.xml | 2 +- .../hudi-metaserver-server-bundle/pom.xml | 4 +-- pom.xml | 34 +++++++++++++++++-- 13 files changed, 45 insertions(+), 15 deletions(-) diff --git a/hudi-client/hudi-flink-client/pom.xml b/hudi-client/hudi-flink-client/pom.xml index 5ec3dfb9c16c4..4be3da9a3d5f7 100644 --- a/hudi-client/hudi-flink-client/pom.xml +++ b/hudi-client/hudi-flink-client/pom.xml @@ -42,7 +42,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink/pom.xml b/hudi-flink-datasource/hudi-flink/pom.xml index 1923534f475b3..2fb0525c85a51 100644 --- a/hudi-flink-datasource/hudi-flink/pom.xml +++ b/hudi-flink-datasource/hudi-flink/pom.xml @@ -78,7 +78,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink1.18.x/pom.xml b/hudi-flink-datasource/hudi-flink1.18.x/pom.xml index 72adebafa4b1b..025821f2217e7 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.18.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink1.19.x/pom.xml b/hudi-flink-datasource/hudi-flink1.19.x/pom.xml index 983dde06bd55a..de712340cf5c3 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.19.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink1.20.x/pom.xml b/hudi-flink-datasource/hudi-flink1.20.x/pom.xml index 70838282e5ac1..06738ec1b2f7e 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.20.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink2.0.x/pom.xml b/hudi-flink-datasource/hudi-flink2.0.x/pom.xml index 322779f7c1197..86647aa03d2d6 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink2.0.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-flink-datasource/hudi-flink2.1.x/pom.xml b/hudi-flink-datasource/hudi-flink2.1.x/pom.xml index f8d07669b2a8e..5eccf6ad3eaed 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink2.1.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-io/pom.xml b/hudi-io/pom.xml index d431354de4a0e..dbc0d74f46a20 100644 --- a/hudi-io/pom.xml +++ b/hudi-io/pom.xml @@ -185,7 +185,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl ${log4j2.version} provided diff --git a/hudi-platform-service/hudi-metaserver/pom.xml b/hudi-platform-service/hudi-metaserver/pom.xml index 341faa2a01e32..038901de6d8ed 100644 --- a/hudi-platform-service/hudi-metaserver/pom.xml +++ b/hudi-platform-service/hudi-metaserver/pom.xml @@ -77,7 +77,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl ${log4j2.version} compile diff --git a/hudi-sync/hudi-adb-sync/pom.xml b/hudi-sync/hudi-adb-sync/pom.xml index c545aea9b2875..39772a1af1738 100644 --- a/hudi-sync/hudi-adb-sync/pom.xml +++ b/hudi-sync/hudi-adb-sync/pom.xml @@ -103,7 +103,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j diff --git a/hudi-tests-common/pom.xml b/hudi-tests-common/pom.xml index af3ec9c9b21da..e174ba6d554ca 100644 --- a/hudi-tests-common/pom.xml +++ b/hudi-tests-common/pom.xml @@ -68,7 +68,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl compile diff --git a/packaging/hudi-metaserver-server-bundle/pom.xml b/packaging/hudi-metaserver-server-bundle/pom.xml index bfe8c10359ac1..d0f353644c2e6 100644 --- a/packaging/hudi-metaserver-server-bundle/pom.xml +++ b/packaging/hudi-metaserver-server-bundle/pom.xml @@ -67,7 +67,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl ${log4j2.version} compile @@ -106,7 +106,7 @@ org.apache.logging.log4j:log4j-api org.apache.logging.log4j:log4j-core org.apache.logging.log4j:log4j-1.2-api - org.apache.logging.log4j:log4j-slf4j-impl + org.apache.logging.log4j:log4j-slf4j2-impl org.slf4j:slf4j-api org.slf4j:jul-to-slf4j org.mybatis:mybatis diff --git a/pom.xml b/pom.xml index ae01441bf9585..dc95ae3e750d5 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ 1.14.1 5.21.0 2.25.4 - 1.7.36 + 2.0.7 2.9.9 2.10.2 org.apache.hive @@ -833,7 +833,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl ${log4j2.version} provided @@ -1026,6 +1026,11 @@ org.slf4j * + + + org.apache.logging.log4j + log4j-slf4j-impl + log4j log4j @@ -1074,6 +1079,11 @@ org.slf4j * + + + org.apache.logging.log4j + log4j-slf4j-impl + log4j log4j @@ -1467,6 +1477,11 @@ org.pentaho * + + + org.apache.logging.log4j + log4j-slf4j-impl + @@ -1487,6 +1502,11 @@ org.slf4j slf4j-log4j12 + + + org.apache.logging.log4j + log4j-slf4j-impl + log4j log4j @@ -1503,6 +1523,11 @@ javax.mail mail + + + org.apache.logging.log4j + log4j-slf4j-impl + @@ -1531,6 +1556,11 @@ log4j log4j + + + org.apache.logging.log4j + log4j-slf4j-impl + org.apache.hbase * From b57870f1a9479cad85151752b8dab60fe8b1f9a7 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Mon, 13 Jul 2026 21:18:35 +0700 Subject: [PATCH 159/255] fix(spark): match the staged table, not LogicalWriteInfo, in BasicStagedTable.newWriteBuilder (#19251) (cherry picked from commit 770e8aabf367b51b56912e4f9120fe0500334198) --- .../sql/hudi/catalog/BasicStagedTable.scala | 2 +- .../sql/hudi/catalog/HoodieCatalog.scala | 20 +++- .../hudi/catalog/TestBasicStagedTable.scala | 52 ++++++++ .../TestHoodieCatalogStagedTable.scala | 112 ++++++++++++++++++ 4 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/catalog/TestBasicStagedTable.scala create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/catalog/TestHoodieCatalogStagedTable.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala index 2cc94c3e14792..ae2ac079187d6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala @@ -38,7 +38,7 @@ case class BasicStagedTable(ident: Identifier, table: Table, catalog: TableCatalog) extends SupportsWrite with StagedTable { override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - info match { + table match { case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) case _ => throw new HoodieException(s"Table `${ident.name}` does not support writes.") } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala index 2527602653023..f5ef66a04b92c 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala @@ -74,7 +74,7 @@ class HoodieCatalog extends DelegatingCatalogExtension } else { BasicStagedTable( ident, - super.createTable(ident, schema, partitions, properties), + createOrLoadTable(ident, schema, partitions, properties), this) } } @@ -92,7 +92,7 @@ class HoodieCatalog extends DelegatingCatalogExtension super.dropTable(ident) BasicStagedTable( ident, - super.createTable(ident, schema, partitions, properties), + createOrLoadTable(ident, schema, partitions, properties), this) } } @@ -115,11 +115,25 @@ class HoodieCatalog extends DelegatingCatalogExtension } BasicStagedTable( ident, - super.createTable(ident, schema, partitions, properties), + createOrLoadTable(ident, schema, partitions, properties), this) } } + /** + * Creates the table in the delegate catalog and returns it, never null. + * + * A delegate is allowed to return null from `createTable`: `V2SessionCatalog` does so deliberately, to save the + * `loadTable` round-trip for a `CREATE TABLE` without `AS SELECT`. Load the table that was just created in that + * case, so that the staged table is never backed by a null table. Spark guards its own staging the same way. + */ + private def createOrLoadTable(ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): Table = { + Option(super.createTable(ident, schema, partitions, properties)).getOrElse(loadTable(ident)) + } + override def loadTable(ident: Identifier): Table = { super.loadTable(ident) match { case V1Table(catalogTable0) if sparkAdapter.isHoodieTable(catalogTable0) => diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/catalog/TestBasicStagedTable.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/catalog/TestBasicStagedTable.scala new file mode 100644 index 0000000000000..7efc70a82abf8 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/catalog/TestBasicStagedTable.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.catalog + +import org.apache.hudi.exception.HoodieException + +import org.apache.spark.sql.connector.catalog.{Identifier, SupportsWrite, Table, TableCatalog} +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.junit.jupiter.api.Assertions.{assertSame, assertThrows, assertTrue} +import org.junit.jupiter.api.Test +import org.mockito.Mockito.{mock, when} + +class TestBasicStagedTable { + + private val ident = Identifier.of(Array("db"), "tbl") + + @Test + def testNewWriteBuilderDelegatesToWritableTable(): Unit = { + val table = mock(classOf[SupportsWrite]) + val info = mock(classOf[LogicalWriteInfo]) + val writeBuilder = mock(classOf[WriteBuilder]) + when(table.newWriteBuilder(info)).thenReturn(writeBuilder) + + val staged = BasicStagedTable(ident, table, mock(classOf[TableCatalog])) + + assertSame(writeBuilder, staged.newWriteBuilder(info)) + } + + @Test + def testNewWriteBuilderThrowsWhenTableIsNotWritable(): Unit = { + val staged = BasicStagedTable(ident, mock(classOf[Table]), mock(classOf[TableCatalog])) + + val ex = assertThrows(classOf[HoodieException], + () => staged.newWriteBuilder(mock(classOf[LogicalWriteInfo]))) + assertTrue(ex.getMessage.contains("`tbl` does not support writes")) + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/catalog/TestHoodieCatalogStagedTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/catalog/TestHoodieCatalogStagedTable.scala new file mode 100644 index 0000000000000..c607b434580c8 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/catalog/TestHoodieCatalogStagedTable.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.catalog + +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.testutils.HoodieClientTestUtils + +import org.apache.spark.api.java.JavaSparkContext +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.catalog.{Identifier, StagedTable, SupportsWrite, Table, TableCatalog} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.apache.spark.sql.types.{IntegerType, StructField, StructType} +import org.junit.jupiter.api.{AfterAll, BeforeAll, TestInstance} +import org.junit.jupiter.api.Assertions.{assertEquals, assertSame, assertThrows, assertTrue} +import org.junit.jupiter.api.TestInstance.Lifecycle +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.mockito.Mockito.{mock, when} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * Tests the staging methods of {@link HoodieCatalog} for a non-Hudi table, which stage through + * {@link BasicStagedTable} and hand the write off to the delegate catalog's table. + */ +@TestInstance(Lifecycle.PER_CLASS) +class TestHoodieCatalogStagedTable { + + private val ident = Identifier.of(Array("default"), "tbl") + private val schema = StructType(Seq(StructField("id", IntegerType))) + private val partitions = Array.empty[Transform] + // Not a Hudi provider, so the staging methods take the BasicStagedTable branch + private val properties: util.Map[String, String] = Map("provider" -> "parquet").asJava + + private var sparkSession: SparkSession = _ + + @BeforeAll + def setUp(): Unit = { + val jsc = new JavaSparkContext( + HoodieClientTestUtils.getSparkConfForTest(classOf[TestHoodieCatalogStagedTable].getName)) + jsc.setLogLevel("ERROR") + // HoodieCatalog resolves SparkSession.active in its constructor + sparkSession = SparkSession.builder.config(jsc.getConf).getOrCreate + } + + @AfterAll + def tearDown(): Unit = { + sparkSession.close() + } + + @ParameterizedTest + @ValueSource(strings = Array("stageCreate", "stageReplace", "stageCreateOrReplace")) + def testStagedWriteIsDelegatedToWritableTable(stagingMethod: String): Unit = { + val delegateTable = mock(classOf[SupportsWrite]) + val info = mock(classOf[LogicalWriteInfo]) + val writeBuilder = mock(classOf[WriteBuilder]) + when(delegateTable.newWriteBuilder(info)).thenReturn(writeBuilder) + val delegate = mock(classOf[TableCatalog]) + when(delegate.createTable(ident, schema, partitions, properties)).thenReturn(delegateTable) + + val staged = stage(stagingMethod, delegate) + + assertSame(writeBuilder, staged.asInstanceOf[SupportsWrite].newWriteBuilder(info)) + } + + @ParameterizedTest + @ValueSource(strings = Array("stageCreate", "stageReplace", "stageCreateOrReplace")) + def testStagedTableIsLoadedWhenDelegateCreateTableReturnsNull(stagingMethod: String): Unit = { + // V2SessionCatalog, the default delegate, returns null from createTable by design, to save the loadTable call + val delegate = mock(classOf[TableCatalog]) + val loadedTable = mock(classOf[Table]) + when(loadedTable.schema()).thenReturn(schema) + when(delegate.loadTable(ident)).thenReturn(loadedTable) + + val staged = stage(stagingMethod, delegate) + + // The staged table is backed by the table that was just created, rather than by null + assertEquals(schema, staged.schema()) + // It is not writable, so the write is rejected instead of being delegated + val ex = assertThrows(classOf[HoodieException], + () => staged.asInstanceOf[SupportsWrite].newWriteBuilder(mock(classOf[LogicalWriteInfo]))) + assertTrue(ex.getMessage.contains("`tbl` does not support writes")) + } + + private def stage(stagingMethod: String, delegate: TableCatalog): StagedTable = { + val catalog = new HoodieCatalog() + catalog.setDelegateCatalog(delegate) + stagingMethod match { + case "stageCreate" => catalog.stageCreate(ident, schema, partitions, properties) + case "stageReplace" => catalog.stageReplace(ident, schema, partitions, properties) + case "stageCreateOrReplace" => catalog.stageCreateOrReplace(ident, schema, partitions, properties) + } + } +} From d1d385e8a18a96f4156991a4bd3996800f8d600f Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 13 Jul 2026 09:19:25 -0700 Subject: [PATCH 160/255] test(spark): cover the legacy parquet read path with file-group reader disabled (#19133) * test(spark): cover the legacy parquet read path with file-group reader disabled * test(spark): exercise legacy parquet read path on nested struct/array columns Address review feedback on #19133: - Add a nested struct and an array column so the legacy parquet reader is driven through its complex-type branch (the vectorized nested-column path, historically fragile per HUDI-7190/#10265), not just flat scalar columns. - Sort compared rows by the named 'id' column instead of a positional index. * test(spark): keep vectorized nested-column coverage on spark3.3 The nested struct/array columns flip the legacy parquet reader's supportBatch off on spark3.3, where spark.sql.parquet.enableNestedColumnVectorizedReader defaults to false (true only from spark3.4). That made testCowSnapshotReadEqualsFileGroupReader and testCowSnapshotReadWithoutVectorizedReader both fall back to the parquet-mr row path on 3.3, dropping the vectorized nested-column branch the suite targets. Enable nested-column vectorization in setUp so the vectorized branch runs on every Spark profile, and assert supportBatch is true in the vectorized case and false in the non-vectorized case so a Spark default change cannot silently collapse the two paths again. * test(spark): cover legacy reader implicit int->long type-change path The prior cases all wrote and read the same schema, so typeChangeInfos stayed empty and shouldUseInternalSchema never became true in the SparkXXLegacyHoodieParquetFileFormat shims. The suite never reached the branch that swaps Spark's stock VectorizedParquetRecordReader for Hudi's HoodieVectorizedParquetRecordReader -- the Hudi-specific reason these per-version copies exist. Add testCowSnapshotReadWithImplicitTypeChange: commit 1 writes `value` as int32; commit 2 upserts only the p0 rows with a widened long value too large for an int, promoting the table schema to long. In COW the p1/p2 base files keep the narrower physical int, so reading them against the long table schema goes through HoodieParquetFileFormatHelper .buildImplicitSchemaChangeInfo and HoodieVectorizedParquetRecordReader. Asserts the vectorized branch is engaged, the values widen correctly, and the legacy path still matches the file-group reader. * test(spark): cover schema-on-read and remaining type-change branches of the legacy read path Address review on the legacy-read-path suite: - testCowSnapshotReadWithSchemaOnRead: the one production shape in which DefaultSource#resolveBaseFileOnlyRelation returns BaseFileOnlyRelation itself (buildScan ships), flips shouldExtractPartitionValuesFromPartitionPath off (shims built with shouldAppendPartitionValues = false), and drives the explicit internal-schema branch (InternalSchemaCache + InternalSchemaMerger). The InternalSchema is seeded without DDL via hoodie.schema.on.read.enable plus hoodie.datasource.write.reconcile.schema on the writes. - testCowSnapshotReadWithImplicitTypeChangeWithoutVectorizedReader: the row-based Cast/GenerateUnsafeProjection widening the shims run when the vectorized reader is disabled. - testCowSnapshotReadWithNestedTypeChange: int->long inside the nested struct; vectorized reads must fail fast with the documented IllegalArgumentException and the advertised workaround (vectorized reader off) must read the promoted struct correctly through the row-based Cast branch. --------- Co-authored-by: voonhous (cherry picked from commit a4cde3a1d0bcba5c4a179edd0d0a78faa5f0f556) --- .../TestLegacyParquetReadPath.scala | 473 ++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLegacyParquetReadPath.scala diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLegacyParquetReadPath.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLegacyParquetReadPath.scala new file mode 100644 index 0000000000000..fac97a0cbecba --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLegacyParquetReadPath.scala @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{BaseFileOnlyRelation, DataSourceReadOptions, DataSourceWriteOptions, IncrementalRelationV1, IncrementalRelationV2, ScalaAssertionSupport} +import org.apache.hudi.common.config.HoodieReaderConfig +import org.apache.hudi.common.table.HoodieTableConfig +import org.apache.hudi.common.table.log.InstantRange.RangeType +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.testutils.HoodieSparkClientTestBase + +import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession} +import org.apache.spark.sql.functions.{col, lit, struct} +import org.apache.spark.sql.types.{IntegerType, LongType} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} + +/** Row shape written by these tests. A nested struct and an array are included so the legacy + * parquet read path is exercised on complex types -- the historically fragile vectorized + * nested-column branch (e.g. HUDI-7190), not just flat scalar columns. */ +private case class LegacyNested(a: Int, b: String) + +private case class LegacyTestRow(id: String, + ts: Long, + value: Long, + partition: String, + nested: LegacyNested, + tags: Seq[Int]) + +/** + * Functional tests for the legacy (pre-file-group-reader) Spark read path: + * [[BaseFileOnlyRelation]], [[IncrementalRelationV1]], [[IncrementalRelationV2]] and the + * per-Spark-version legacy Hudi parquet file format created via + * `sparkAdapter.createLegacyHoodieParquetFileFormat`. + * + * In the batch datasource, `DefaultSource` routes normal reads to the file-group-reader-based + * relations regardless of `hoodie.file.group.reader.enabled`; the legacy relations still run in + * production for metadata-table reads and for streaming reads with the flag disabled. To exercise + * them functionally here, the legacy relations are constructed directly (with the flag set to + * false in their options, matching how the streaming sources invoke them) and their results are + * compared row-by-row against the file-group-reader-enabled reads of the same table. + * + * `DefaultSource#resolveBaseFileOnlyRelation` returns [[BaseFileOnlyRelation]] itself only under + * schema-on-read and converts it to a `HadoopFsRelation` otherwise, so the relation's own + * `buildScan` ships only in the schema-on-read shape; both scan shapes are exercised below. + */ +class TestLegacyParquetReadPath extends HoodieSparkClientTestBase with ScalaAssertionSupport { + + var spark: SparkSession = _ + + private val writeOpts = Map( + "hoodie.insert.shuffle.parallelism" -> "2", + "hoodie.upsert.shuffle.parallelism" -> "2", + DataSourceWriteOptions.RECORDKEY_FIELD.key -> "id", + DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", + DataSourceWriteOptions.TABLE_TYPE.key -> DataSourceWriteOptions.COW_TABLE_TYPE_OPT_VAL, + DataSourceWriteOptions.HIVE_STYLE_PARTITIONING.key -> "true", + HoodieTableConfig.ORDERING_FIELDS.key -> "ts", + HoodieWriteConfig.TBL_NAME.key -> "legacy_read_path_tbl" + ) + + // Columns compared across the read paths; meta fields are persisted in the base files, so the + // record key and commit time must match exactly between the legacy and new readers. `nested` and + // `tags` force the legacy parquet reader through its complex-type (struct / array) branch. + private val comparedCols = + Seq("_hoodie_commit_time", "_hoodie_record_key", "id", "ts", "value", "partition", "nested", "tags") + + @BeforeEach override def setUp(): Unit = { + setTableName("legacy_read_path_tbl") + initPath() + initSparkContexts() + spark = sqlContext.sparkSession + // The test schema carries a nested struct and an array. On the legacy parquet read path, batch + // (vectorized) support for such complex types is additionally gated on nested-column + // vectorization, which defaults off on spark3.3 and on only from spark3.4. Enable it so the + // vectorized nested-column branch -- the one HUDI-7190 fixed -- is exercised on every Spark + // profile instead of silently falling back to parquet-mr on 3.3 (which would make the vectorized + // and non-vectorized cases below collapse onto the same row-based path there). + spark.conf.set("spark.sql.parquet.enableNestedColumnVectorizedReader", "true") + initHoodieStorage() + } + + @AfterEach override def tearDown(): Unit = { + cleanupResources() + spark = null + } + + private def makeRows(ids: Seq[Int], ts: Long, valueFn: Int => Long): Seq[LegacyTestRow] = + ids.map(i => LegacyTestRow(i.toString, ts, valueFn(i), "p" + (i % 3), + LegacyNested(i, "v" + valueFn(i)), Seq(i, ts.toInt))) + + private def writeBatch(rows: Seq[LegacyTestRow], operation: String, + extraWriteOpts: Map[String, String] = Map.empty): Unit = { + spark.createDataFrame(rows) + .write.format("hudi") + .options(writeOpts ++ extraWriteOpts) + .option(DataSourceWriteOptions.OPERATION.key, operation) + .mode(SaveMode.Append) + .save(basePath) + } + + /** Commit 1: insert 30 rows; commit 2: upsert rows 1-10 with new values. */ + private def writeTwoCommits(): Unit = { + writeBatch(makeRows(1 to 30, ts = 1L, i => i * 10L), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + writeBatch(makeRows(1 to 10, ts = 2L, i => i * 100L), DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL) + } + + // > Int.MaxValue, so a widened value is representable only as a long + private val widenedBase = 10000000000L + + /** + * Commit 1: insert ids 1..30 with `value` written as INT32; commit 2: upsert only the p0 rows + * (id % 3 == 0) with a LONG `value` too large for an int, promoting the table schema to long. + * In COW, commit 2 rewrites just the p0 file group, so the p1/p2 base files keep the narrower + * physical int while the table schema is now long -- reading them exercises the legacy format's + * type-change reconciliation. Updating a single partition is deliberate: upserting across all + * partitions would rewrite every file group and leave no narrow base files behind. + */ + private def writeIntToLongCommits(extraWriteOpts: Map[String, String] = Map.empty): Unit = { + spark.createDataFrame(makeRows(1 to 30, ts = 1L, i => i.toLong)) + .withColumn("value", col("value").cast(IntegerType)) + .write.format("hudi") + .options(writeOpts ++ extraWriteOpts) + .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + writeBatch(makeRows((1 to 30).filter(_ % 3 == 0), ts = 2L, i => widenedBase + i), + DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL, extraWriteOpts) + } + + /** + * Asserts the promoted `value` column of a [[writeIntToLongCommits]] table: p1/p2 ids come from + * int base files widened on read, p0 ids carry the large long values written in commit 2. + */ + private def assertWidenedValues(df: DataFrame): Unit = { + assertEquals(LongType, df.schema("value").dataType, + "Reading the promoted table must surface `value` as long") + val actual = df.select("id", "value").collect() + .map(r => (r.getString(0), r.getLong(1))).toSeq.sortBy(_._1.toInt) + val expected = (1 to 30).map(i => (i.toString, if (i % 3 == 0) widenedBase + i else i.toLong)) + assertEquals(expected, actual) + } + + private def fgReaderDf(extraOpts: Map[String, String] = Map.empty): DataFrame = + spark.read.format("hudi") + .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key, "true") + .options(extraOpts) + .load(basePath) + + private def legacyReadOpts(extraOpts: Map[String, String]): Map[String, String] = + Map( + "path" -> basePath, + DataSourceReadOptions.QUERY_TYPE.key -> DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL, + HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key -> "false" + ) ++ extraOpts + + /** + * Legacy scan through [[BaseFileOnlyRelation]]'s own `PrunedFilteredScan` implementation + * (`HoodieBaseRelation.buildScan` -> base-file readers built on the legacy parquet format). + */ + private def legacyRelationDf(extraOpts: Map[String, String] = Map.empty): DataFrame = { + val metaClient = createMetaClient(spark, basePath) + spark.baseRelationToDataFrame( + BaseFileOnlyRelation(sqlContext, metaClient, legacyReadOpts(extraOpts), None)) + } + + /** + * Legacy scan through the `HadoopFsRelation` conversion that + * `DefaultSource#resolveBaseFileOnlyRelation` applies, executing the per-Spark-version + * legacy Hudi parquet file format inside a regular file-source scan. + */ + private def legacyFileFormatDf(extraOpts: Map[String, String] = Map.empty): DataFrame = { + val metaClient = createMetaClient(spark, basePath) + val hadoopFsRelation = + BaseFileOnlyRelation(sqlContext, metaClient, legacyReadOpts(extraOpts), None).toHadoopFsRelation + assertTrue(hadoopFsRelation.fileFormat.getClass.getSimpleName.contains("LegacyHoodieParquetFileFormat"), + s"Expected the legacy parquet file format but got ${hadoopFsRelation.fileFormat.getClass.getName}") + spark.baseRelationToDataFrame(hadoopFsRelation) + } + + /** + * Whether the legacy parquet format engages its vectorized (batch) reader for the table's + * schema. Because the schema carries a nested struct and an array, batch support additionally + * requires nested-column vectorization; asserting on this pins which branch of the reader a test + * exercises so the vectorized and row-based cases cannot silently collapse onto one path (e.g. if + * a Spark default change dropped nested-column vectorization on some profile). + */ + private def legacyFormatSupportsBatch: Boolean = { + val metaClient = createMetaClient(spark, basePath) + val hadoopFsRelation = + BaseFileOnlyRelation(sqlContext, metaClient, legacyReadOpts(Map.empty), None).toHadoopFsRelation + hadoopFsRelation.fileFormat.supportBatch(spark, hadoopFsRelation.schema) + } + + private def collectSorted(df: DataFrame): Seq[Row] = + df.select(comparedCols.map(col): _*).collect().toSeq.sortBy(_.getAs[String]("id").toInt) + + private def assertSameRows(expected: DataFrame, actual: DataFrame): Unit = { + val expectedRows = collectSorted(expected) + assertTrue(expectedRows.nonEmpty, "Comparison must cover a non-empty result") + assertEquals(expectedRows, collectSorted(actual)) + } + + @Test + def testCowSnapshotReadEqualsFileGroupReader(): Unit = { + writeTwoCommits() + + // With nested-column vectorization enabled (see setUp), the legacy format must take its + // vectorized branch on every profile; otherwise this would degenerate to the same row-based + // path as testCowSnapshotReadWithoutVectorizedReader. + assertTrue(legacyFormatSupportsBatch, + "Legacy parquet format must engage the vectorized reader on the nested-column schema") + + val newReaderDf = fgReaderDf() + assertEquals(30, newReaderDf.count()) + + Seq(legacyRelationDf(), legacyFileFormatDf()).foreach { legacyDf => + assertSameRows(newReaderDf, legacyDf) + // The upserts from the second commit must be visible through the legacy path. + val updatedValues = legacyDf.filter(col("ts") === 2L) + .collect().map(_.getAs[Long]("value")).sorted.toSeq + assertEquals((1 to 10).map(_ * 100L), updatedValues) + } + } + + @Test + def testCowSnapshotReadWithoutVectorizedReader(): Unit = { + writeTwoCommits() + + val vectorizedKey = "spark.sql.parquet.enableVectorizedReader" + val previous = spark.conf.get(vectorizedKey, "true") + spark.conf.set(vectorizedKey, "false") + try { + // Row-based (non-batch) branch of the legacy parquet file format. Pin that the disabled + // vectorized reader really forces the fallback path, so this case stays distinct from the + // vectorized one above on every profile. + assertFalse(legacyFormatSupportsBatch, + "Disabling the vectorized reader must force the legacy parquet format onto the row-based path") + val newReaderDf = fgReaderDf() + assertSameRows(newReaderDf, legacyFileFormatDf()) + assertSameRows(newReaderDf, legacyRelationDf()) + } finally { + spark.conf.set(vectorizedKey, previous) + } + } + + @Test + def testCowSnapshotReadWithPartitionValuesExtractedFromPath(): Unit = { + writeTwoCommits() + + // BaseFileOnlyRelation always appends partition values parsed from the (hive-style) + // partition path; enabling the same extraction on the new reader must yield equal rows. + val extractOpts = Map(DataSourceReadOptions.EXTRACT_PARTITION_VALUES_FROM_PARTITION_PATH.key -> "true") + val newReaderDf = fgReaderDf(extractOpts) + assertSameRows(newReaderDf, legacyFileFormatDf(extractOpts)) + assertSameRows(newReaderDf, legacyRelationDf(extractOpts)) + + val partitions = legacyFileFormatDf(extractOpts) + .select("partition").distinct().collect().map(_.getString(0)).sorted.toSeq + assertEquals(Seq("p0", "p1", "p2"), partitions) + } + + @Test + def testPartitionAndDataFilterPushdown(): Unit = { + writeTwoCommits() + + def applyFilters(df: DataFrame): DataFrame = + df.filter(col("partition") === "p1" && col("value") > 100L) + + // Partition p1 holds ids with id % 3 == 1: updated ids {4, 7, 10} have value > 100 + // (id 1 has value exactly 100) and untouched ids {13, 16, 19, 22, 25, 28} do as well. + val newReaderDf = applyFilters(fgReaderDf()) + assertEquals(9, newReaderDf.count()) + assertSameRows(newReaderDf, applyFilters(legacyFileFormatDf())) + assertSameRows(newReaderDf, applyFilters(legacyRelationDf())) + } + + @Test + def testCowIncrementalReadEqualsFileGroupReader(): Unit = { + writeTwoCommits() + writeBatch(makeRows(11 to 15, ts = 3L, i => i * 1000L), DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL) + + val metaClient = createMetaClient(spark, basePath) + val firstInstant = metaClient.getCommitsTimeline.filterCompletedInstants.firstInstant.get + + // New-reader incremental query; on current table versions the start bound is a completion + // time and the range is start-exclusive, so this returns rows written by commits 2 and 3. + val newReaderDf = spark.read.format("hudi") + .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key, "true") + .option(DataSourceReadOptions.QUERY_TYPE.key, DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL) + .option(DataSourceReadOptions.START_COMMIT.key, firstInstant.getCompletionTime) + .load(basePath) + + // V1 slices the timeline by instant time, V2 by completion time; both are start-exclusive. + val incOptsV1 = Map( + DataSourceReadOptions.QUERY_TYPE.key -> DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL, + DataSourceReadOptions.START_COMMIT.key -> firstInstant.requestedTime, + HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key -> "false") + val incOptsV2 = incOptsV1.updated(DataSourceReadOptions.START_COMMIT.key, firstInstant.getCompletionTime) + + val legacyV1Df = spark.baseRelationToDataFrame( + new IncrementalRelationV1(sqlContext, incOptsV1, None, metaClient)) + val legacyV2Df = spark.baseRelationToDataFrame( + new IncrementalRelationV2(sqlContext, incOptsV2, None, metaClient, RangeType.OPEN_CLOSED)) + + // COW upserts preserve the original commit time of untouched rows in rewritten files, so the + // incremental result is exactly the rows written by commits 2 and 3. + val expected = ((1 to 10).map(i => (i.toString, i * 100L)) ++ (11 to 15).map(i => (i.toString, i * 1000L))) + .sortBy(_._1.toInt) + Seq(newReaderDf, legacyV1Df, legacyV2Df).foreach { df => + val actual = df.select("id", "value").collect() + .map(r => (r.getString(0), r.getLong(1))).toSeq.sortBy(_._1.toInt) + assertEquals(expected, actual) + } + + assertSameRows(newReaderDf, legacyV1Df) + assertSameRows(newReaderDf, legacyV2Df) + } + + @Test + def testCowSnapshotReadWithImplicitTypeChange(): Unit = { + // The Hudi-specific reason these per-version parquet formats exist (vs stock ParquetFileFormat) + // is on-read type reconciliation: when a base file's physical column type is narrower than the + // table schema, HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo records the change + // and the vectorized read runs through Hudi's HoodieVectorizedParquetRecordReader (which widens + // the column vector) instead of Spark's stock VectorizedParquetRecordReader. + writeIntToLongCommits() + + // Vectorization must be on so the read takes the HoodieVectorizedParquetRecordReader branch + // rather than the row-based parquet-mr fallback (which reconciles types via a different path). + assertTrue(legacyFormatSupportsBatch, + "Vectorized reader must be engaged so the implicit type change runs through " + + "HoodieVectorizedParquetRecordReader") + + val newReaderDf = fgReaderDf() + Seq(legacyRelationDf(), legacyFileFormatDf()).foreach { legacyDf => + assertWidenedValues(legacyDf) + // The legacy path must still agree with the file-group reader on the promoted column. + assertSameRows(newReaderDf, legacyDf) + } + } + + @Test + def testCowSnapshotReadWithImplicitTypeChangeWithoutVectorizedReader(): Unit = { + writeIntToLongCommits() + + val vectorizedKey = "spark.sql.parquet.enableVectorizedReader" + val previous = spark.conf.get(vectorizedKey, "true") + spark.conf.set(vectorizedKey, "false") + try { + // With the vectorized reader off, the legacy format reconciles the type change on its + // row-based branch instead: a Cast from the file's narrower type compiled into a + // GenerateUnsafeProjection -- an implementation separate from + // HoodieVectorizedParquetRecordReader, so it needs its own coverage. + assertFalse(legacyFormatSupportsBatch, + "Disabling the vectorized reader must force the legacy parquet format onto the row-based path") + val newReaderDf = fgReaderDf() + Seq(legacyRelationDf(), legacyFileFormatDf()).foreach { legacyDf => + assertWidenedValues(legacyDf) + assertSameRows(newReaderDf, legacyDf) + } + } finally { + spark.conf.set(vectorizedKey, previous) + } + } + + @Test + def testCowSnapshotReadWithNestedTypeChange(): Unit = { + // Same int->long promotion, but inside the `nested` struct. The changed top-level column is + // then non-atomic, which the legacy format cannot reconcile in vectorized mode: it must fail + // fast with the documented IllegalArgumentException instead of returning corrupt columns, and + // the workaround the exception advertises (disabling the vectorized reader) must actually read + // the promoted struct correctly through the row-based Cast branch. + writeBatch(makeRows(1 to 30, ts = 1L, i => i * 10L), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + spark.createDataFrame(makeRows((1 to 30).filter(_ % 3 == 0), ts = 2L, i => i * 100L)) + .withColumn("nested", + struct((col("nested.a") + lit(widenedBase)).as("a"), col("nested.b").as("b"))) + .write.format("hudi") + .options(writeOpts) + .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + + assertTrue(legacyFormatSupportsBatch, + "Vectorized reader must be engaged so the non-atomic type change hits the legacy format's rejection") + Seq(legacyRelationDf(), legacyFileFormatDf()).foreach { legacyDf => + val thrown = assertThrows(classOf[Throwable]) { + legacyDf.collect() + } + val causes = Iterator.iterate(thrown: Throwable)(_.getCause).takeWhile(_ != null).take(10).toSeq + assertTrue(causes.exists(c => c.isInstanceOf[IllegalArgumentException] + && String.valueOf(c.getMessage).contains("cannot be read in vectorized mode")), + s"Expected the non-atomic type-change rejection but got: $thrown") + } + + val vectorizedKey = "spark.sql.parquet.enableVectorizedReader" + val previous = spark.conf.get(vectorizedKey, "true") + spark.conf.set(vectorizedKey, "false") + try { + assertFalse(legacyFormatSupportsBatch, + "Disabling the vectorized reader must force the legacy parquet format onto the row-based path") + val newReaderDf = fgReaderDf() + Seq(legacyRelationDf(), legacyFileFormatDf()).foreach { legacyDf => + val actual = legacyDf.select("id", "nested").collect() + .map(r => (r.getString(0), r.getStruct(1).getLong(0))).toSeq.sortBy(_._1.toInt) + val expected = (1 to 30).map(i => (i.toString, if (i % 3 == 0) widenedBase + i else i.toLong)) + assertEquals(expected, actual) + assertSameRows(newReaderDf, legacyDf) + } + } finally { + spark.conf.set(vectorizedKey, previous) + } + } + + @Test + def testCowSnapshotReadWithSchemaOnRead(): Unit = { + // Schema-on-read is the one production shape in which DefaultSource#resolveBaseFileOnlyRelation + // returns BaseFileOnlyRelation itself instead of converting it to a HadoopFsRelation, making + // the relation's own buildScan the shipped scan path. It also flips + // BaseFileOnlyRelation.shouldExtractPartitionValuesFromPartitionPath (defined as + // internalSchemaOpt.isEmpty) to false -- the only way the legacy parquet format is constructed + // with shouldAppendPartitionValues = false -- and drives the format's explicit internal-schema + // branch (InternalSchemaCache lookup + InternalSchemaMerger) instead of the implicit + // footer-based reconciliation. + // + // No ALTER TABLE is needed to get an InternalSchema into commit metadata: with + // hoodie.schema.on.read.enable plus hoodie.datasource.write.reconcile.schema on the writes, + // HoodieSparkSqlWriter seeds the internal schema on the first commit and evolves it with the + // int->long promotion on the second. + writeIntToLongCommits(Map( + DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key -> "true", + DataSourceWriteOptions.RECONCILE_SCHEMA.key -> "true")) + + val readOpts = Map(DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key -> "true") + val metaClient = createMetaClient(spark, basePath) + val schemaOnReadRelation = BaseFileOnlyRelation(sqlContext, metaClient, legacyReadOpts(readOpts), None) + assertTrue(schemaOnReadRelation.hasSchemaOnRead, + "The writes must have recorded an InternalSchema in commit metadata for schema-on-read to engage") + // Pin the flipped partition-values branch: under schema-on-read the relation reads partition + // columns from the data files (empty partition schema) instead of re-appending them from the + // partition path, unlike every other case in this suite. + assertTrue(schemaOnReadRelation.toHadoopFsRelation.partitionSchema.isEmpty, + "Schema-on-read must flip shouldExtractPartitionValuesFromPartitionPath off") + assertTrue(BaseFileOnlyRelation(sqlContext, metaClient, legacyReadOpts(Map.empty), None) + .toHadoopFsRelation.partitionSchema.nonEmpty, + "Without schema-on-read the converted relation appends partition values from the path") + + assertTrue(legacyFormatSupportsBatch, + "Vectorized reader must be engaged so the explicit type change runs through " + + "HoodieVectorizedParquetRecordReader") + + val newReaderDf = fgReaderDf(readOpts) + Seq(legacyRelationDf(readOpts), legacyFileFormatDf(readOpts)).foreach { legacyDf => + assertWidenedValues(legacyDf) + assertSameRows(newReaderDf, legacyDf) + } + } +} From 2576cec4226931471dde63372071889def1621b7 Mon Sep 17 00:00:00 2001 From: vamsikarnika Date: Tue, 14 Jul 2026 13:52:56 +0530 Subject: [PATCH 161/255] fix: reuse Inflater/Deflater in BitCaskDiskMap to avoid JDK8 finalizer contention (#18818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(common): reuse Inflater/Deflater in BitCaskDiskMap to avoid JDK8 finalizer contention CompressionHandler currently allocates a new Deflater on every compressBytes() and a new Inflater on every decompressBytes(). On JDK 8 both classes register a Finalizer on construction. Under sustained, multi-threaded disk-map traffic (observed during MDT/RLI compaction merging millions of records across several Spark task threads on the same executor), the rate of zlib allocations exceeds the rate at which the single Finalizer thread can drain its queue. Native ZStreamRef handles pile up in old gen, the heap saturates, and G1 enters a mixed-GC death spiral while application threads make no progress. CompressionHandler is already held in a ThreadLocal, so a single Deflater/ Inflater pair per worker thread is sufficient. This change: - Adds transient Deflater/Inflater fields and lazy accessors (transient so the class remains Serializable; lazy so deserialized instances rebuild the codecs on first use). - Calls reset() on the cached codecs at the start of each call. - Passes the user-supplied codecs to DeflaterOutputStream(out, def) and InflaterInputStream(in, inf), which sets usesDefaultDeflater/Inflater to false so close() does not call end() on the codec — the codec survives the try-with-resources for reuse on the next call. On-disk format, compression level, and error semantics are unchanged. Allocation rate drops from O(records) to O(threads). On JDK 9+ this also removes per-call Cleaner registration overhead. * remove extra comments * address comments * address nits * test(common): add micro-benchmark for Inflater/Deflater reuse Adds InflaterDeflaterReuseRLIBenchmark to quantify the gain from reusing Deflater/Inflater per worker thread vs allocating per call, using a realistic RLI HoodieMetadataPayload (kryo-serialized HoodieRecord wrapper, ~156 bytes -- matches what BitCaskDiskMap actually compresses). InspectRLISize captures the per-record serialized + deflated sizes. * chore: re-trigger CI (cherry picked from commit 1dfbdcb265ea470837a30f432079245f3a76fb45) --- .../util/collection/BitCaskDiskMap.java | 33 +- .../InflaterDeflaterReuseRLIBenchmark.java | 327 ++++++++++++++++++ .../util/collection/InspectRLISize.java | 58 ++++ 3 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/collection/InflaterDeflaterReuseRLIBenchmark.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/collection/InspectRLISize.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java index e7b298dc2b8d4..fb22c9869ae26 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java @@ -58,6 +58,7 @@ import java.util.stream.Stream; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; import static org.apache.hudi.common.util.BinaryUtil.generateChecksum; @@ -408,6 +409,11 @@ private static class CompressionHandler implements Serializable { private final ByteArrayOutputStream compressBaos; private final ByteArrayOutputStream decompressBaos; private final byte[] decompressIntermediateBuffer; + // Each CompressionHandler is held in a ThreadLocal, + // so a single Deflater/Inflater pair per worker thread is sufficient and + // avoids per-call construction. + private transient Deflater deflater; + private transient Inflater inflater; CompressionHandler() { compressBaos = new ByteArrayOutputStream(DISK_COMPRESSION_INITIAL_BUFFER_SIZE); @@ -415,22 +421,35 @@ private static class CompressionHandler implements Serializable { decompressIntermediateBuffer = new byte[DECOMPRESS_INTERMEDIATE_BUFFER_SIZE]; } + private Deflater getDeflater() { + if (deflater == null) { + deflater = new Deflater(Deflater.BEST_COMPRESSION); + } + return deflater; + } + + private Inflater getInflater() { + if (inflater == null) { + inflater = new Inflater(); + } + return inflater; + } + private byte[] compressBytes(final byte[] value) throws IOException { compressBaos.reset(); - Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION); - DeflaterOutputStream dos = new DeflaterOutputStream(compressBaos, deflater); - try { + Deflater deflater = getDeflater(); + deflater.reset(); + try (DeflaterOutputStream dos = new DeflaterOutputStream(compressBaos, deflater)) { dos.write(value); - } finally { - dos.close(); - deflater.end(); } return compressBaos.toByteArray(); } private byte[] decompressBytes(final byte[] bytes) throws IOException { decompressBaos.reset(); - try (InputStream in = new InflaterInputStream(new ByteArrayInputStream(bytes))) { + Inflater inflater = getInflater(); + inflater.reset(); + try (InputStream in = new InflaterInputStream(new ByteArrayInputStream(bytes), inflater)) { int len; while ((len = in.read(decompressIntermediateBuffer)) > 0) { decompressBaos.write(decompressIntermediateBuffer, 0, len); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InflaterDeflaterReuseRLIBenchmark.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InflaterDeflaterReuseRLIBenchmark.java new file mode 100644 index 0000000000000..8b780c6ad4e93 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InflaterDeflaterReuseRLIBenchmark.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.collection; + +/* + * Micro-benchmark for the Inflater/Deflater reuse change in BitCaskDiskMap — RLI payload variant. + * + * Same OLD-vs-NEW comparison as InflaterDeflaterReuseBenchmark, but the payload + * fed to compress/decompress is a kryo-serialized HoodieAvroRecord + * carrying a record-index entry. This mirrors what BitCaskDiskMap.compressBytes actually + * sees in production when spilling metadata-table record-index records. + * + * Each "op" compresses + decompresses ONE serialized record by default. To stress the + * codec with larger buffers, pass recordsPerOp > 1 — the benchmark packs that many + * serialized records into a single byte[] per op. + * + * Run via Maven (needs Hudi classes on the test classpath): + * + * JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_192.jdk/Contents/Home \ + * mvn -pl hudi-common -DskipTests test-compile + * JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_192.jdk/Contents/Home \ + * mvn -pl hudi-common exec:java -Dexec.classpathScope=test \ + * -Dexec.mainClass=org.apache.hudi.common.util.collection.InflaterDeflaterReuseRLIBenchmark \ + * -Dexec.args="8 100000 1" + * + * Args: [threads] [opsPerThread] [recordsPerOp] + */ + +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.metadata.HoodieMetadataPayload; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +public class InflaterDeflaterReuseRLIBenchmark { + + private static final int DEFAULT_THREADS = 8; + private static final int DEFAULT_OPS_PER_THREAD = 100_000; + private static final int DEFAULT_RECORDS_PER_OP = 1; + private static final int DECOMPRESS_INTERMEDIATE_BUFFER_SIZE = 8192; + private static final int COMPRESS_INITIAL_BUFFER_SIZE = 1024 * 1024; + + public static void main(String[] args) throws Exception { + int threads = args.length > 0 ? Integer.parseInt(args[0]) : DEFAULT_THREADS; + int opsPerThread = args.length > 1 ? Integer.parseInt(args[1]) : DEFAULT_OPS_PER_THREAD; + int recordsPerOp = args.length > 2 ? Integer.parseInt(args[2]) : DEFAULT_RECORDS_PER_OP; + + // Build a pool of pre-serialized RLI records so the timed loop only measures + // compress/decompress, not record construction + kryo serialization. + byte[][] serializedRecords = buildSerializedRLIRecords(2048); + int singleRecordSize = serializedRecords[0].length; + + System.out.println("=== Inflater/Deflater reuse benchmark — RLI payload ==="); + System.out.println("threads=" + threads + + " opsPerThread=" + opsPerThread + + " recordsPerOp=" + recordsPerOp + + " singleRecordSerializedBytes=" + singleRecordSize + + " payloadBytesPerOp=" + (singleRecordSize * recordsPerOp) + + " totalOps=" + ((long) threads * opsPerThread)); + System.out.println("java.version=" + System.getProperty("java.version") + + " vm=" + System.getProperty("java.vm.name")); + System.out.println(); + + // Warm both paths. + runScenario("warmup-old", 2, 2_000, serializedRecords, recordsPerOp, false); + runScenario("warmup-new", 2, 2_000, serializedRecords, recordsPerOp, true); + System.gc(); + Thread.sleep(200); + + for (int trial = 1; trial <= 3; trial++) { + System.out.println("--- Trial " + trial + " ---"); + if (trial % 2 == 1) { + runScenario("OLD (new Deflater/Inflater per call)", threads, opsPerThread, serializedRecords, recordsPerOp, false); + runScenario("NEW (reuse via ThreadLocal)", threads, opsPerThread, serializedRecords, recordsPerOp, true); + } else { + runScenario("NEW (reuse via ThreadLocal)", threads, opsPerThread, serializedRecords, recordsPerOp, true); + runScenario("OLD (new Deflater/Inflater per call)", threads, opsPerThread, serializedRecords, recordsPerOp, false); + } + System.out.println(); + } + } + + private static byte[][] buildSerializedRLIRecords(int count) throws IOException { + byte[][] out = new byte[count][]; + Random rnd = new Random(42); + String instantTime = "20260520120000"; // any valid yyyyMMddHHmmss; not what we're measuring + for (int i = 0; i < count; i++) { + String recordKey = "user_" + rnd.nextLong() + "_" + i; + String partition = "date=2026-05-" + String.format("%02d", 1 + (i % 28)); + String fileId = UUID.randomUUID().toString() + "-0"; + HoodieRecord rec = + HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partition, fileId, instantTime, 0); + out[i] = SerializationUtils.serialize(rec); + } + return out; + } + + /** + * Build the per-op payload by concatenating `recordsPerOp` serialized records. + * BitCaskDiskMap actually compresses one record at a time; this stitching mode is + * here only to simulate larger-buffer scenarios on demand. + */ + private static byte[] buildPayload(byte[][] pool, int recordsPerOp, int seed) { + if (recordsPerOp == 1) { + return pool[Math.floorMod(seed, pool.length)]; + } + int total = 0; + int start = Math.floorMod(seed, pool.length); + for (int i = 0; i < recordsPerOp; i++) { + total += pool[(start + i) % pool.length].length; + } + byte[] out = new byte[total]; + int pos = 0; + for (int i = 0; i < recordsPerOp; i++) { + byte[] r = pool[(start + i) % pool.length]; + System.arraycopy(r, 0, out, pos, r.length); + pos += r.length; + } + return out; + } + + private static void runScenario(String label, + int threads, + int opsPerThread, + byte[][] pool, + int recordsPerOp, + boolean reuse) throws Exception { + ExecutorService poolExec = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(threads); + + long[][] perThreadLatenciesNs = new long[threads][]; + AtomicLong failures = new AtomicLong(); + + for (int t = 0; t < threads; t++) { + final int idx = t; + poolExec.submit(new Runnable() { + @Override + public void run() { + long[] latencies = new long[opsPerThread]; + ThreadCompressionContext ctx = new ThreadCompressionContext(); + try { + start.await(); + for (int i = 0; i < opsPerThread; i++) { + byte[] payload = buildPayload(pool, recordsPerOp, idx * 1_000_003 + i); + long t0 = System.nanoTime(); + byte[] compressed = reuse ? ctx.compressReuse(payload) : ctx.compressNew(payload); + byte[] decompressed = reuse ? ctx.decompressReuse(compressed) : ctx.decompressNew(compressed); + long t1 = System.nanoTime(); + latencies[i] = t1 - t0; + if (decompressed.length != payload.length) { + failures.incrementAndGet(); + } + } + perThreadLatenciesNs[idx] = latencies; + } catch (Throwable th) { + failures.incrementAndGet(); + th.printStackTrace(); + } finally { + ctx.close(); + done.countDown(); + } + } + }); + } + + long wallStart = System.nanoTime(); + start.countDown(); + done.await(); + long wallEnd = System.nanoTime(); + poolExec.shutdown(); + + long totalOps = (long) threads * opsPerThread; + long wallMs = (wallEnd - wallStart) / 1_000_000; + double opsPerSec = totalOps / ((wallEnd - wallStart) / 1e9); + + long[] all = flatten(perThreadLatenciesNs); + Arrays.sort(all); + + System.out.printf("%-42s wall=%6d ms throughput=%9.0f ops/s" + + " avg=%6.1f us p50=%6.1f us p95=%6.1f us p99=%6.1f us max=%7.1f us failures=%d%n", + label, + wallMs, + opsPerSec, + mean(all) / 1000.0, + percentile(all, 50) / 1000.0, + percentile(all, 95) / 1000.0, + percentile(all, 99) / 1000.0, + all[all.length - 1] / 1000.0, + failures.get()); + } + + /** Mirrors BitCaskDiskMap.CompressionHandler with both pre-fix and post-fix paths. */ + private static final class ThreadCompressionContext { + private final ByteArrayOutputStream compressBaos = + new ByteArrayOutputStream(COMPRESS_INITIAL_BUFFER_SIZE); + private final ByteArrayOutputStream decompressBaos = + new ByteArrayOutputStream(COMPRESS_INITIAL_BUFFER_SIZE); + private final byte[] intermediate = new byte[DECOMPRESS_INTERMEDIATE_BUFFER_SIZE]; + + private Deflater reusedDeflater; + private Inflater reusedInflater; + + byte[] compressNew(byte[] value) throws IOException { + compressBaos.reset(); + Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION); + DeflaterOutputStream dos = new DeflaterOutputStream(compressBaos, deflater); + try { + dos.write(value); + } finally { + dos.close(); + deflater.end(); + } + return compressBaos.toByteArray(); + } + + byte[] decompressNew(byte[] bytes) throws IOException { + decompressBaos.reset(); + try (InputStream in = new InflaterInputStream(new ByteArrayInputStream(bytes))) { + int len; + while ((len = in.read(intermediate)) > 0) { + decompressBaos.write(intermediate, 0, len); + } + } + return decompressBaos.toByteArray(); + } + + byte[] compressReuse(byte[] value) throws IOException { + compressBaos.reset(); + if (reusedDeflater == null) { + reusedDeflater = new Deflater(Deflater.BEST_COMPRESSION); + } + reusedDeflater.reset(); + try (DeflaterOutputStream dos = new DeflaterOutputStream(compressBaos, reusedDeflater)) { + dos.write(value); + } + return compressBaos.toByteArray(); + } + + byte[] decompressReuse(byte[] bytes) throws IOException { + decompressBaos.reset(); + if (reusedInflater == null) { + reusedInflater = new Inflater(); + } + reusedInflater.reset(); + try (InputStream in = new InflaterInputStream(new ByteArrayInputStream(bytes), reusedInflater)) { + int len; + while ((len = in.read(intermediate)) > 0) { + decompressBaos.write(intermediate, 0, len); + } + } + return decompressBaos.toByteArray(); + } + + void close() { + if (reusedDeflater != null) { + reusedDeflater.end(); + } + if (reusedInflater != null) { + reusedInflater.end(); + } + } + } + + private static long[] flatten(long[][] arrs) { + int total = 0; + for (long[] a : arrs) { + total += a.length; + } + long[] out = new long[total]; + int pos = 0; + for (long[] a : arrs) { + System.arraycopy(a, 0, out, pos, a.length); + pos += a.length; + } + return out; + } + + private static double mean(long[] sorted) { + long sum = 0; + for (long v : sorted) { + sum += v; + } + return (double) sum / sorted.length; + } + + private static long percentile(long[] sorted, int p) { + int idx = (int) Math.ceil(p / 100.0 * sorted.length) - 1; + if (idx < 0) { + idx = 0; + } + if (idx >= sorted.length) { + idx = sorted.length - 1; + } + return sorted[idx]; + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InspectRLISize.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InspectRLISize.java new file mode 100644 index 0000000000000..0ae90fcc7da83 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/InspectRLISize.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.collection; + +/* + * Quick utility: print the kryo-serialized size of an RLI record and its deflated size. + * Run: mvn -pl hudi-common exec:java -Dexec.classpathScope=test \ + * -Dexec.mainClass=org.apache.hudi.common.util.collection.InspectRLISize + */ + +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.metadata.HoodieMetadataPayload; + +import java.io.ByteArrayOutputStream; +import java.util.UUID; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; + +public class InspectRLISize { + public static void main(String[] args) throws Exception { + for (int i = 0; i < 5; i++) { + String recordKey = "user_record_" + i; + String partition = "date=2026-05-22"; + String fileId = UUID.randomUUID().toString() + "-0"; + String instantTime = "20260522120000"; + HoodieRecord rec = + HoodieMetadataPayload.createRecordIndexUpdate(recordKey, partition, fileId, instantTime, 0); + byte[] serialized = SerializationUtils.serialize(rec); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Deflater d = new Deflater(Deflater.BEST_COMPRESSION); + try (DeflaterOutputStream dos = new DeflaterOutputStream(baos, d)) { + dos.write(serialized); + } + d.end(); + System.out.printf("record %d: kryoBytes=%d deflatedBytes=%d ratio=%.2fx%n", + i, serialized.length, baos.size(), + (double) serialized.length / baos.size()); + } + } +} From 05957b865c0e4775a5fc7b0879f7e2c0c8732c7d Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Wed, 15 Jul 2026 12:12:54 +0530 Subject: [PATCH 162/255] fix: Improve error message for conflict resolution (#18119) * Improve error message for conflict resolution * Fix build * Address comments (cherry picked from commit 59e62dd488cd964c3125e9cb96d0089f9148c0be) --- ...tFileWritesConflictResolutionStrategy.java | 62 ++++++- ...tFileWritesConflictResolutionStrategy.java | 171 ++++++++++++++++++ .../hudi/common/model/WriteOperationType.java | 11 ++ 3 files changed, 242 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java index 0d6327272a19f..92c6f6f66ba54 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java @@ -200,8 +200,66 @@ public Option resolveConflict(HoodieTable table, return thisOperation.getCommitMetadataOption(); } // just abort the current write if conflicts are found (failed for rollback conflicts). - throw new HoodieWriteConflictException(new ConcurrentModificationException("Cannot resolve conflicts for overlapping writes between first operation = " + thisOperation - + ", second operation = " + otherOperation)); + throw new HoodieWriteConflictException(new ConcurrentModificationException(buildConflictErrorMessage(thisOperation, otherOperation))); + } + + /** + * Builds a detailed error message for write conflicts based on the operation types involved. + */ + private String buildConflictErrorMessage(ConcurrentOperation thisOperation, ConcurrentOperation otherOperation) { + boolean thisIsTableService = WriteOperationType.isTableService(thisOperation.getOperationType()); + boolean otherIsTableService = WriteOperationType.isTableService(otherOperation.getOperationType()); + String thisOperationDescription = formatOperationDescription(thisOperation); + String otherOperationDescription = formatOperationDescription(otherOperation); + // If either operation is a table service, provide specific retry guidance + if (thisIsTableService || otherIsTableService) { + ConcurrentOperation tableServiceOperation = thisIsTableService ? thisOperation : otherOperation; + String tableServiceDescription = thisIsTableService ? thisOperationDescription : otherOperationDescription; + String regularOperationDescription = thisIsTableService ? otherOperationDescription : thisOperationDescription; + String serviceType = getTableServiceDisplayName(tableServiceOperation.getOperationType()); + return String.format( + "Cannot resolve conflicts for overlapping writes. %s is currently running and has overlapping file groups with %s. " + + "Please retry the write operation after the %s completes.", + tableServiceDescription, regularOperationDescription, serviceType.toLowerCase() + ); + } + // For regular write operations conflicting with each other + return String.format( + "Cannot resolve conflicts for overlapping writes. %s has overlapping file groups with %s.", + thisOperationDescription, otherOperationDescription + ); + } + + /** + * Formats a description of an operation including its type, instant, and state. + */ + private String formatOperationDescription(ConcurrentOperation operation) { + String operationName = WriteOperationType.isTableService(operation.getOperationType()) + ? "Table " + getTableServiceDisplayName(operation.getOperationType()) + : operation.getOperationType().value() + " operation"; + + return String.format("%s (instant: %s, state: %s)", + operationName, + operation.getInstantTimestamp(), + operation.getInstantActionState()); + } + + /** + * Returns a user-friendly display name for table service operations. + */ + private String getTableServiceDisplayName(WriteOperationType operationType) { + switch (operationType) { + case COMPACT: + return "Compaction"; + case CLUSTER: + return "Clustering"; + case LOG_COMPACT: + return "Log Compaction"; + case INDEX: + return "Indexing"; + default: + return operationType.value(); + } } @Override diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java index 4e11940686ca6..5232dc444b448 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java @@ -515,4 +515,175 @@ public void testConcurrentWritesWithPendingInstants() throws Exception { } } } + + @Test + public void testErrorMessageForConflictWithCompaction() throws Exception { + initMetaClient(true, HoodieTableType.MERGE_ON_READ); + createCommit(WriteClientTestUtils.createNewInstantTime(), metaClient); + HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); + Option lastSuccessfulInstant = timeline.getCommitsTimeline().filterCompletedInstants().lastInstant(); + + // writer 1 starts + String currentWriterInstant = WriteClientTestUtils.createNewInstantTime(); + createInflightCommit(currentWriterInstant, metaClient); + + // compaction gets scheduled and runs + String compactionInstant = WriteClientTestUtils.createNewInstantTime(); + createCompactionRequested(compactionInstant, metaClient); + + Option currentInstant = Option.of(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, currentWriterInstant)); + SimpleConcurrentFileWritesConflictResolutionStrategy strategy = new SimpleConcurrentFileWritesConflictResolutionStrategy(); + HoodieCommitMetadata currentMetadata = createCommitMetadata(currentWriterInstant); + metaClient.reloadActiveTimeline(); + + List candidateInstants = strategy.getCandidateInstants(metaClient, currentInstant.get(), lastSuccessfulInstant) + .collect(Collectors.toList()); + Assertions.assertEquals(1, candidateInstants.size()); + + ConcurrentOperation thatCompactionOperation = new ConcurrentOperation(candidateInstants.get(0), metaClient); + ConcurrentOperation thisCommitOperation = new ConcurrentOperation(currentInstant.get(), currentMetadata); + Assertions.assertTrue(strategy.hasConflict(thisCommitOperation, thatCompactionOperation)); + + HoodieWriteConflictException exception = Assertions.assertThrows(HoodieWriteConflictException.class, + () -> strategy.resolveConflict(null, thisCommitOperation, thatCompactionOperation)); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("Table Compaction"), + "Error message should mention 'Table Compaction', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("is currently running"), + "Error message should contain 'is currently running', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("Please retry the write operation after the compaction completes"), + "Error message should contain retry guidance, but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains(compactionInstant), + "Error message should contain compaction instant time, but was: " + errorMessage); + } + + @Test + public void testErrorMessageForConflictWithClustering() throws Exception { + initMetaClient(); + createCommit(WriteClientTestUtils.createNewInstantTime(), metaClient); + HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); + Option lastSuccessfulInstant = timeline.getCommitsTimeline().filterCompletedInstants().lastInstant(); + + // writer 1 starts + String currentWriterInstant = WriteClientTestUtils.createNewInstantTime(); + createInflightCommit(currentWriterInstant, metaClient); + + // clustering gets scheduled + String clusteringInstant = WriteClientTestUtils.createNewInstantTime(); + createClusterRequested(clusteringInstant, metaClient); + + Option currentInstant = Option.of(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, currentWriterInstant)); + SimpleConcurrentFileWritesConflictResolutionStrategy strategy = new SimpleConcurrentFileWritesConflictResolutionStrategy(); + HoodieCommitMetadata currentMetadata = createCommitMetadata(currentWriterInstant); + metaClient.reloadActiveTimeline(); + + List candidateInstants = strategy.getCandidateInstants(metaClient, currentInstant.get(), lastSuccessfulInstant) + .collect(Collectors.toList()); + Assertions.assertEquals(1, candidateInstants.size()); + + ConcurrentOperation thatClusteringOperation = new ConcurrentOperation(candidateInstants.get(0), metaClient); + ConcurrentOperation thisCommitOperation = new ConcurrentOperation(currentInstant.get(), currentMetadata); + Assertions.assertTrue(strategy.hasConflict(thisCommitOperation, thatClusteringOperation)); + + HoodieWriteConflictException exception = Assertions.assertThrows(HoodieWriteConflictException.class, + () -> strategy.resolveConflict(null, thisCommitOperation, thatClusteringOperation)); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("Table Clustering"), + "Error message should mention 'Table Clustering', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("is currently running"), + "Error message should contain 'is currently running', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("Please retry the write operation after the clustering completes"), + "Error message should contain retry guidance, but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains(clusteringInstant), + "Error message should contain clustering instant time, but was: " + errorMessage); + } + + @Test + public void testErrorMessageForConflictBetweenRegularWrites() throws Exception { + initMetaClient(); + createCommit(WriteClientTestUtils.createNewInstantTime(), metaClient); + HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); + Option lastSuccessfulInstant = timeline.getCommitsTimeline().filterCompletedInstants().lastInstant(); + + // writer 1 starts + String currentWriterInstant = WriteClientTestUtils.createNewInstantTime(); + createInflightCommit(currentWriterInstant, metaClient); + + // writer 2 starts and finishes + String writer2Instant = WriteClientTestUtils.createNewInstantTime(); + createCommit(writer2Instant, metaClient); + + Option currentInstant = Option.of(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, currentWriterInstant)); + SimpleConcurrentFileWritesConflictResolutionStrategy strategy = new SimpleConcurrentFileWritesConflictResolutionStrategy(); + HoodieCommitMetadata currentMetadata = createCommitMetadata(currentWriterInstant); + metaClient.reloadActiveTimeline(); + + List candidateInstants = strategy.getCandidateInstants(metaClient, currentInstant.get(), lastSuccessfulInstant) + .collect(Collectors.toList()); + Assertions.assertEquals(1, candidateInstants.size()); + + ConcurrentOperation thatCommitOperation = new ConcurrentOperation(candidateInstants.get(0), metaClient); + ConcurrentOperation thisCommitOperation = new ConcurrentOperation(currentInstant.get(), currentMetadata); + Assertions.assertTrue(strategy.hasConflict(thisCommitOperation, thatCommitOperation)); + + HoodieWriteConflictException exception = Assertions.assertThrows(HoodieWriteConflictException.class, + () -> strategy.resolveConflict(null, thisCommitOperation, thatCommitOperation)); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("Cannot resolve conflicts for overlapping writes"), + "Error message should mention overlapping writes, but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("has overlapping file groups"), + "Error message should contain 'has overlapping file groups', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains(currentWriterInstant), + "Error message should contain current writer instant time, but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains(writer2Instant), + "Error message should contain other writer instant time, but was: " + errorMessage); + // Should NOT contain table service specific messaging + Assertions.assertFalse(errorMessage.contains("Table Compaction"), + "Error message should not mention table services for regular writes, but was: " + errorMessage); + Assertions.assertFalse(errorMessage.contains("Please retry"), + "Error message should not contain retry guidance for regular writes, but was: " + errorMessage); + } + + @Test + public void testErrorMessageForConflictWithCompletedClustering() throws Exception { + initMetaClient(); + createCommit(WriteClientTestUtils.createNewInstantTime(), metaClient); + HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); + Option lastSuccessfulInstant = timeline.getCommitsTimeline().filterCompletedInstants().lastInstant(); + + // writer 1 starts + String currentWriterInstant = WriteClientTestUtils.createNewInstantTime(); + createInflightCommit(currentWriterInstant, metaClient); + + // clustering completes + String clusteringInstant = WriteClientTestUtils.createNewInstantTime(); + createCluster(clusteringInstant, WriteOperationType.CLUSTER, metaClient); + + Option currentInstant = Option.of(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, currentWriterInstant)); + SimpleConcurrentFileWritesConflictResolutionStrategy strategy = new SimpleConcurrentFileWritesConflictResolutionStrategy(); + HoodieCommitMetadata currentMetadata = createCommitMetadata(currentWriterInstant); + metaClient.reloadActiveTimeline(); + + List candidateInstants = strategy.getCandidateInstants(metaClient, currentInstant.get(), lastSuccessfulInstant) + .collect(Collectors.toList()); + Assertions.assertEquals(1, candidateInstants.size()); + + ConcurrentOperation thatClusteringOperation = new ConcurrentOperation(candidateInstants.get(0), metaClient); + ConcurrentOperation thisCommitOperation = new ConcurrentOperation(currentInstant.get(), currentMetadata); + Assertions.assertTrue(strategy.hasConflict(thisCommitOperation, thatClusteringOperation)); + + HoodieWriteConflictException exception = Assertions.assertThrows(HoodieWriteConflictException.class, + () -> strategy.resolveConflict(null, thisCommitOperation, thatClusteringOperation)); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("Table Clustering"), + "Error message should mention 'Table Clustering', but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains(clusteringInstant), + "Error message should contain clustering instant time, but was: " + errorMessage); + Assertions.assertTrue(errorMessage.contains("COMPLETED"), + "Error message should indicate the state of the clustering operation, but was: " + errorMessage); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java b/hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java index e794aad60b606..db288339a1b0a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java @@ -188,6 +188,17 @@ public static boolean isCompactionOrClustering(WriteOperationType operationType) return operationType == COMPACT || operationType == CLUSTER; } + /** + * Checks if the given operation type is a table service operation. + * Table service operations include compaction, clustering, log compaction, and indexing. + */ + public static boolean isTableService(WriteOperationType operationType) { + return operationType == COMPACT + || operationType == CLUSTER + || operationType == LOG_COMPACT + || operationType == INDEX; + } + /** * @return true if streaming writes to metadata table is supported for a given {@link WriteOperationType}. false otherwise. */ From e6a28c8c1d21a30e10016326d5b33178d8cd1a5c Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Fri, 17 Jul 2026 07:51:08 +0700 Subject: [PATCH 163/255] fix(flink): remove the Source V2 read teardown race via materialized bounded minibatches (#19202) * fix(flink): make CDC read iterator teardown thread-safe to de-flake stream-read ITs CdcFileSplitsIterator is drained on the Flink task thread but can be closed from the split-fetcher thread during job teardown, racing on its non-volatile recordIterator/imageManager fields (and the nested BaseImageIterator.cdcItr) and surfacing an NPE that flakes testStreamReadFromSpecifiedCommitWithChangelog. Synchronize hasNext()/next()/close() on the iterator and add a closed flag so a concurrent close() cannot null out state mid-read and the iterator reports a well-defined end-of-stream afterwards. Extend the test's isAcceptableTerminalFailure teardown-race tolerance to the CDC-iterator frame, since BatchRecords drives hasNext()/next() as separate calls and a close() between them still ends a force-terminated drain with a benign NoSuchElementException. * addressed review comments: split next() guard and use class refs for CDC teardown-frame check * test(flink): retry a bare await timeout in the CDC stream-read IT The flink2.1 CI shard failed testStreamReadFromSpecifiedCommitWithChangelog with "Unexpected job failure" caused by a TimeoutException from tableResult.await(): the streaming read had not collected its expected rows within the window (CI-load slowness, not the teardown race this PR already tolerates and retries). A bare TimeoutException was rethrown as an AssertionError, bypassing submitAndFetchWithRetry, which only retries short results. Treat a bare await timeout as a retryable short read: cancel the still-running job and return the rows collected so far so the retry loop re-submits a fresh job. Also widen the await window from 30s to 60s so a slow shard is less likely to time out at all. Test-only; no production change. * addressed review comments: use equals for CDC teardown-frame class match * addressed review comments: redesign Source V2 read path to materialize bounded minibatches The reviewers asked for a root-cause fix rather than the CdcFileSplitsIterator synchronization mitigation. The real flaw was that BatchRecords did lazy reading: it held a live ClosableIterator drained on the Flink task thread, while a forced-cancel teardown closed that same iterator on the split-fetcher thread, so read and close genuinely ran on different threads. This reworks the Source V2 read path so a reader function is a per-split cursor that owns the record iterator, the CdcImageManager and the file-group readers. HoodieSourceSplitReader.fetch() opens a split, reads one bounded minibatch (DEFAULT_MINI_BATCH_SIZE), and closes the split's resources on the same split-fetcher thread on EOF, failure or cancellation. BatchRecords now carries a materialized list instead of a live iterator, so read and close happen on one thread and the cross-thread teardown race is removed at its source; this also covers the MOR/COW Parquet teardown race, not just the CDC one. Because Flink's columnar readers and the CDC/MOR row projections return the same reused RowData on every next(), readBatch deep-copies each record before buffering it and re-applies the RowKind, which the serializer does not round-trip. With read and close single-threaded, the CdcFileSplitsIterator synchronization and closed-flag guards are reverted, and the test-side teardown-race tolerance in ITTestHoodieDataSource is removed: isAcceptableTerminalFailure now accepts only the SuccessException happy path. The independent await-timeout retry is kept. * addressed review comments: narrow the await-timeout retry to a bare top-level TimeoutException TableResult.await(long, TimeUnit) throws its own timeout bare, so only a top-level TimeoutException means the await window elapsed with the job still running. A genuine job failure arrives wrapped in an ExecutionException that may itself embed a TimeoutException (checkpoint expiry, RPC timeout); walking the whole cause chain misclassified such a failure as a slow shard, cancelled and retried it, and then reported a row-count mismatch with the real cause discarded. isAwaitTimeout now inspects only the top-level exception. * addressed review comments: regroup copySerializer field and rename test open-count accessor * addressed review comments: drop redundant RowKind re-apply after RowDataSerializer.copy * addressed review comments: unblock fetch() on wakeUp and close the file-group reader on init failure P1: fetch() now drains an I/O-backed minibatch, so wakeUp() sets a volatile flag the drain polls between records; on a wake-up fetch() returns promptly without finishing/closing, keeping the split-close on the fetcher thread. P2: HoodieSplitReaderFunction.createRecordIterator retains the HoodieFileGroupReader in a local and closes it when getClosableIterator() fails, suppressing the close error onto the original exception. --------- Co-authored-by: Vova Kolmakov (cherry picked from commit d10b868d90c97bd7297c0ac18fcb7fcf3deac8af) --- .../hudi/source/reader/BatchRecords.java | 64 ++- .../reader/HoodieSourceSplitReader.java | 80 +++- .../function/AbstractSplitReaderFunction.java | 107 ++++- .../HoodieCdcSplitReaderFunction.java | 27 +- .../function/HoodieSplitReaderFunction.java | 43 +- .../reader/function/SplitReaderFunction.java | 46 +- .../hudi/table/format/cdc/CdcIterators.java | 5 + .../hudi/source/reader/TestBatchRecords.java | 249 +++-------- .../reader/TestHoodieSourceSplitReader.java | 418 ++++++++++++++++-- .../TestAbstractSplitReaderFunction.java | 153 ++++++- .../TestHoodieCdcSplitReaderFunction.java | 11 +- .../TestHoodieSplitReaderFunction.java | 92 +++- .../hudi/table/ITTestHoodieDataSource.java | 148 ++----- 13 files changed, 992 insertions(+), 451 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/BatchRecords.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/BatchRecords.java index 99e01e45aabf1..ea0fce5644ded 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/BatchRecords.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/BatchRecords.java @@ -20,46 +20,54 @@ import org.apache.flink.connector.base.source.reader.RecordsBySplits; import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.common.util.collection.ClosableIterator; import java.util.Collections; +import java.util.List; import java.util.Set; import javax.annotation.Nullable; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; /** - * Implementation of RecordsWithSplitIds with a list record inside. + * Implementation of {@link RecordsWithSplitIds} backed by a materialized, bounded minibatch of + * records for a single split. * - * Type parameters: – record type + *

    The records are already drained (and copied) on the split-fetcher thread before this batch is + * enqueued, so {@link #nextRecordFromSplit()} only walks an in-memory list and holds no live I/O + * resource. Record reading and resource teardown therefore stay on the same (split-fetcher) thread, + * which removes the cross-thread teardown race a live-iterator batch would be exposed to. The + * underlying iterator and its I/O resources are owned and closed by the reader function (see + * {@code AbstractSplitReaderFunction#closeCurrentSplit}). + * + * @param record type */ public class BatchRecords implements RecordsWithSplitIds> { private String splitId; private String nextSplitId; - private final ClosableIterator recordIterator; + private final List records; private final Set finishedSplits; private final HoodieRecordWithPosition recordAndPosition; - // point to current read position within the records list - private int position; + // points to the current read position within the records list + private int index; BatchRecords( String splitId, - ClosableIterator recordIterator, + List records, int fileOffset, long startingRecordOffset, Set finishedSplits) { ValidationUtils.checkArgument( finishedSplits != null, "finishedSplits can be empty but not null"); ValidationUtils.checkArgument( - recordIterator != null, "recordIterator can be empty but not null"); + records != null, "records can be empty but not null"); this.splitId = splitId; this.nextSplitId = splitId; - this.recordIterator = recordIterator; + this.records = records; this.finishedSplits = finishedSplits; this.recordAndPosition = new HoodieRecordWithPosition<>(); this.recordAndPosition.set(null, fileOffset, startingRecordOffset); - this.position = 0; + this.index = 0; } @Nullable @@ -67,7 +75,7 @@ public class BatchRecords implements RecordsWithSplitIds nextRecordFromSplit() { - if (recordIterator.hasNext()) { - recordAndPosition.record(recordIterator.next()); - position = position + 1; + if (index < records.size()) { + recordAndPosition.record(records.get(index)); + index++; return recordAndPosition; } else { - recordIterator.close(); return null; } } @@ -95,29 +102,14 @@ public Set finishedSplits() { @Override public void recycle() { - if (recordIterator != null) { - recordIterator.close(); - } - } - - public void seek(long startingRecordOffset) { - for (long i = 0; i < startingRecordOffset; ++i) { - if (recordIterator.hasNext()) { - position = position + 1; - recordIterator.next(); - } else { - throw new IllegalStateException( - String.format( - "Invalid starting record offset %d for split %s", - startingRecordOffset, - splitId)); - } - } + // No-op: the minibatch is fully materialized, so there is no live iterator or I/O resource to + // release here. The underlying reader is owned and closed by the reader function on the + // split-fetcher thread (AbstractSplitReaderFunction#closeCurrentSplit). } public static BatchRecords forRecords( - String splitId, ClosableIterator recordIterator, int fileOffset, long startingRecordOffset) { - return new BatchRecords<>(splitId, recordIterator, fileOffset, startingRecordOffset, Set.of()); + String splitId, List records, int fileOffset, long startingRecordOffset) { + return new BatchRecords<>(splitId, records, fileOffset, startingRecordOffset, Set.of()); } public static RecordsWithSplitIds> lastBatchRecords(String splitId) { @@ -128,4 +120,4 @@ public static RecordsWithSplitIds> lastBatchReco // in SourceReaderBase for bounded (batch) reads. return new RecordsBySplits<>(Collections.emptyMap(), Set.of(splitId)); } -} \ No newline at end of file +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java index c916dc21dadb6..52cc54db017af 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java @@ -46,22 +46,32 @@ /** * The split reader of Hoodie source. * - *

    Each call to {@link #fetch()} reads one split and returns it as a single - * {@link RecordsWithSplitIds} batch. Flink's {@code SourceReaderBase} is responsible for - * draining all records from the batch (via {@code nextRecordFromSplit()}) and marking - * the split finished (via {@code finishedSplits()}) before calling {@link #fetch()} again. + *

    Each call to {@link #fetch()} returns one bounded minibatch of the currently open split, so a + * single split spans multiple {@code fetch()} calls. When the open split is exhausted its resources + * are closed on this (split-fetcher) thread and a finish signal ({@code finishedSplits()}) is + * returned so Flink's {@code SourceReaderBase} can advance / reach end-of-input. All record reading + * and resource teardown for a split therefore happen on the same thread, which removes the + * cross-thread teardown race a live-iterator batch would be exposed to. * * @param record type */ public class HoodieSourceSplitReader implements SplitReader, HoodieSourceSplit> { private static final Logger LOG = LoggerFactory.getLogger(HoodieSourceSplitReader.class); + // Upper bound on the number of records materialized per fetch() call (one minibatch). Kept as a + // fixed constant for now (mirrors RecordIterators.DEFAULT_BATCH_SIZE); it can be promoted to a + // Flink option later if a tunable per-fetch bound is ever needed. + private static final int DEFAULT_MINI_BATCH_SIZE = 2048; + private final SerializableComparator splitComparator; private final Queue splits; private final FlinkStreamReadMetrics readerMetrics; private final SplitReaderFunction readerFunction; private final Option recordLimiter; private transient HoodieSourceSplit currentSplit; + // Set by wakeUp() (possibly from another thread) to stop the in-flight minibatch drain promptly. + // Reset at the start of every fetch(), so it only ever means "a wakeUp() landed during THIS fetch". + private volatile boolean wokenUp; public HoodieSourceSplitReader( String tableName, @@ -79,27 +89,47 @@ public HoodieSourceSplitReader( @Override public RecordsWithSplitIds> fetch() throws IOException { - // finish current split. - if (currentSplit != null) { - return finishSplit(); + // A wakeUp() only needs to unblock an in-progress fetch(); Flink's SplitFetcher drives shutdown + // off its own 'closed' flag (set before wakeUp() and checked before the next fetch()), not off a + // lasting wakeUp effect. Start each cycle from a clean flag so the drain below reacts only to a + // wakeUp that lands during THIS fetch. + wokenUp = false; + if (currentSplit == null) { + // Limit already satisfied: drain any remaining locally-queued splits as immediately finished + // so that Flink's SourceReaderBase can reach end-of-input cleanly. + if (recordLimiter.map(RecordLimiter::isLimitReached).orElse(false)) { + return drainRemainingAsSplitsFinished(); + } + HoodieSourceSplit nextSplit = splits.poll(); + if (nextSplit == null) { + // return an empty result, which will lead to split fetch to be idle. + // SplitFetcherManager will then close idle fetcher. + return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); + } + currentSplit = nextSplit; + readerFunction.open(currentSplit); } - // Limit already satisfied: drain any remaining locally-queued splits as immediately finished - // so that Flink's SourceReaderBase can reach end-of-input cleanly. - if (recordLimiter.map(RecordLimiter::isLimitReached).orElse(false)) { - return drainRemainingAsSplitsFinished(); + // Read the next bounded minibatch of the open split, unless the global limit is already reached. + if (!recordLimiter.map(RecordLimiter::isLimitReached).orElse(false)) { + BatchRecords batch = readerFunction.readBatch(currentSplit, DEFAULT_MINI_BATCH_SIZE, () -> wokenUp); + if (batch != null) { + // Partial (woken) or full minibatch; the split is not finished either way. + return recordLimiter.map(rl -> rl.wrap(batch)).orElse(batch); + } + if (wokenUp) { + // Woken before any record was buffered: return promptly WITHOUT finishing or closing the + // split, so it resumes on the next fetch(), or the fetcher observes shutdown and closes it + // on this (split-fetcher) thread. This branch must stay inside the !isLimitReached block: + // a wakeUp coinciding with the limit-reached path below must still finish the split. + return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); + } } - HoodieSourceSplit nextSplit = splits.poll(); - if (nextSplit != null) { - currentSplit = nextSplit; - RecordsWithSplitIds> records = readerFunction.read(nextSplit); - return recordLimiter.map(rl -> rl.wrap(records)).orElse(records); - } else { - // return an empty result, which will lead to split fetch to be idle. - // SplitFetcherManager will then close idle fetcher. - return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); - } + // Split exhausted (or the limit was reached mid-split): close its resources on this + // (split-fetcher) thread first, then emit the finish signal so SourceReaderBase can advance. + readerFunction.closeCurrentSplit(); + return finishSplit(); } @Override @@ -122,13 +152,17 @@ public void handleSplitsChanges(SplitsChange splitsChange) { @Override public void wakeUp() { - // Nothing to do + // Flink calls this (while holding SplitFetcher.lock) to unblock a fetch() that is draining a + // minibatch, e.g. on shutdown. Keep it a non-blocking plain volatile write; the drain loop in + // readBatch polls the flag between records and returns promptly. The actual resource teardown + // still happens on the split-fetcher thread via close()/closeCurrentSplit(). + wokenUp = true; } /** * SourceSplitReader only reads splits sequentially. When waiting for watermark alignment * the SourceOperator will stop processing and recycling the fetched batches. Based on this the - * {@code pauseOrResumeSplits} and the {@code wakeUp} are left empty. + * {@code pauseOrResumeSplits} is left empty. * @param splitsToPause splits to pause * @param splitsToResume splits to resume */ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/AbstractSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/AbstractSplitReaderFunction.java index 6bef94ff0709d..f3ca8f5c98af1 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/AbstractSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/AbstractSplitReaderFunction.java @@ -20,16 +20,37 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.RowType; +import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.configuration.HadoopConfigurations; import org.apache.hudi.source.ExpressionPredicates; +import org.apache.hudi.source.reader.BatchRecords; +import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.table.format.InternalSchemaManager; import org.apache.hudi.util.FlinkWriteClients; +import java.util.ArrayList; import java.util.List; +import java.util.function.BooleanSupplier; /** - * Abstract implementation of SplitReaderFunction. + * Abstract implementation of {@link SplitReaderFunction} that provides the per-split cursor + * machinery shared by all reader functions. + * + *

    A subclass only supplies how to create the record iterator for a split + * ({@link #createRecordIterator(HoodieSourceSplit)}) and the {@link RowType} of the records it + * produces ({@link #producedRowType()}); this base drives {@link #open(HoodieSourceSplit)}, + * {@link #readBatch(HoodieSourceSplit, int, java.util.function.BooleanSupplier)}, {@link #closeCurrentSplit()} and {@link #close()}, + * all on the single split-fetcher thread. + * + *

    Because Flink's columnar readers (and the CDC/MOR row projections) return the same reused + * {@link RowData} object on every {@code next()}, {@link #readBatch} copies each record before + * buffering it - materializing raw references would make every entry in a minibatch alias the last + * row. {@link RowDataSerializer#copy(RowData)} preserves the record's {@code RowKind}, so no + * re-apply is needed. */ public abstract class AbstractSplitReaderFunction implements SplitReaderFunction { @@ -39,6 +60,11 @@ public abstract class AbstractSplitReaderFunction implements SplitReaderFunction protected final boolean emitDelete; private transient HoodieWriteConfig writeConfig; private transient org.apache.hadoop.conf.Configuration hadoopConf; + private transient RowDataSerializer copySerializer; + + // Per-split cursor state (split-fetcher thread only). + private transient ClosableIterator currentIterator; + private transient long nextRecordOffset; public AbstractSplitReaderFunction( Configuration conf, @@ -51,6 +77,85 @@ public AbstractSplitReaderFunction( this.emitDelete = emitDelete; } + /** + * Creates the record iterator (and its underlying I/O resources) for {@code split}. Closing the + * returned iterator must release all of those resources. + */ + protected abstract ClosableIterator createRecordIterator(HoodieSourceSplit split); + + /** The {@link RowType} of the records produced by {@link #createRecordIterator}. */ + protected abstract RowType producedRowType(); + + @Override + public void open(HoodieSourceSplit split) { + this.currentIterator = createRecordIterator(split); + try { + // Skip the records already emitted before the last checkpoint so a recovered split resumes at + // the right position; matches the validation the old BatchRecords#seek performed. + long consumed = split.getConsumed(); + for (long i = 0; i < consumed; i++) { + if (currentIterator.hasNext()) { + currentIterator.next(); + } else { + throw new IllegalStateException( + String.format("Invalid starting record offset %d for split %s", consumed, split.splitId())); + } + } + this.nextRecordOffset = consumed; + } catch (RuntimeException | Error e) { + // Close on failure so the split's I/O resources are released even if the resume-skip fails. + closeCurrentSplit(); + throw e; + } + } + + @Override + public BatchRecords readBatch(HoodieSourceSplit split, int batchSize, BooleanSupplier wakeupSignal) { + ValidationUtils.checkState(currentIterator != null, + "readBatch called before open for split " + split.splitId()); + RowDataSerializer serializer = getCopySerializer(); + List buffer = new ArrayList<>(); + try { + // Poll wakeupSignal between records so a wakeUp() lands promptly: materialization stops early + // and whatever is buffered so far is returned as a partial minibatch (null if nothing yet). + while (buffer.size() < batchSize && !wakeupSignal.getAsBoolean() && currentIterator.hasNext()) { + RowData next = currentIterator.next(); + buffer.add(serializer.copy(next)); + } + } catch (RuntimeException | Error e) { + // Close on failure so the split's I/O resources are released even if a mid-drain read fails. + closeCurrentSplit(); + throw e; + } + if (buffer.isEmpty()) { + return null; + } + long startingRecordOffset = nextRecordOffset; + nextRecordOffset += buffer.size(); + return BatchRecords.forRecords(split.splitId(), buffer, split.getFileOffset(), startingRecordOffset); + } + + @Override + public void closeCurrentSplit() { + if (currentIterator != null) { + currentIterator.close(); + currentIterator = null; + } + nextRecordOffset = 0; + } + + @Override + public void close() throws Exception { + closeCurrentSplit(); + } + + private RowDataSerializer getCopySerializer() { + if (copySerializer == null) { + copySerializer = new RowDataSerializer(producedRowType()); + } + return copySerializer; + } + protected HoodieWriteConfig getWriteConfig() { if (writeConfig == null) { writeConfig = FlinkWriteClients.getHoodieClientConfig(conf); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java index 0b0d4f6191bc0..266c8ababa791 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java @@ -39,8 +39,6 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.source.ExpressionPredicates; -import org.apache.hudi.source.reader.BatchRecords; -import org.apache.hudi.source.reader.HoodieRecordWithPosition; import org.apache.hudi.source.split.HoodieCdcSourceSplit; import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.table.format.FilePathUtils; @@ -55,9 +53,9 @@ import org.apache.hudi.util.StreamerUtil; import lombok.extern.slf4j.Slf4j; -import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; import org.apache.hadoop.fs.Path; import java.io.IOException; @@ -80,7 +78,6 @@ public class HoodieCdcSplitReaderFunction extends AbstractSplitReaderFunction { private final List fieldTypes; private final MergeOnReadTableState tableState; private transient HoodieTableMetaClient metaClient; - private transient ClosableIterator currentIterator; // Fallback reader for non-CDC splits (e.g. snapshot reads when read.start-commit='earliest') private transient HoodieSplitReaderFunction fallbackReaderFunction; @@ -109,12 +106,12 @@ public HoodieCdcSplitReaderFunction( } @Override - public RecordsWithSplitIds> read(HoodieSourceSplit split) { + protected ClosableIterator createRecordIterator(HoodieSourceSplit split) { if (!(split instanceof HoodieCdcSourceSplit)) { // Non-CDC splits arrive when reading from 'earliest' with no prior CDC history // (i.e. instantRange is empty → snapshot path). Fall back to the standard MOR reader // which emits all records as INSERT rows, matching the expected snapshot behaviour. - return getFallbackReaderFunction().read(split); + return getFallbackReaderFunction().createRecordIterator(split); } HoodieCdcSourceSplit cdcSplit = (HoodieCdcSourceSplit) split; @@ -131,21 +128,15 @@ public RecordsWithSplitIds> read(HoodieSourceS mode, imageManager); - currentIterator = new CdcIterators.CdcFileSplitsIterator(cdcSplit.getChanges(), imageManager, recordIteratorFunc); - BatchRecords records = BatchRecords.forRecords( - split.splitId(), currentIterator, split.getFileOffset(), split.getConsumed()); - records.seek(split.getConsumed()); - return records; + // The CdcFileSplitsIterator owns the imageManager and its per-split record iterators; closing it + // (via the base class closeCurrentSplit) releases them. The base class handles the consumed-offset + // skip and the minibatch materialization uniformly with the MOR/COW path. + return new CdcIterators.CdcFileSplitsIterator(cdcSplit.getChanges(), imageManager, recordIteratorFunc); } @Override - public void close() throws Exception { - if (currentIterator != null) { - currentIterator.close(); - } - if (fallbackReaderFunction != null) { - fallbackReaderFunction.close(); - } + protected RowType producedRowType() { + return tableState.getRequiredRowType(); } // ------------------------------------------------------------------------- diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java index ac7827bfef7de..2e88d6163a52d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java @@ -30,14 +30,13 @@ import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.source.ExpressionPredicates; -import org.apache.hudi.source.reader.BatchRecords; -import org.apache.hudi.source.reader.HoodieRecordWithPosition; import org.apache.hudi.source.split.HoodieSourceSplit; -import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; import org.apache.hudi.table.format.FormatUtils; import org.apache.hudi.table.format.InternalSchemaManager; +import org.apache.hudi.util.HoodieSchemaConverter; import org.apache.hudi.util.StreamerUtil; import java.io.IOException; @@ -52,7 +51,6 @@ public class HoodieSplitReaderFunction extends AbstractSplitReaderFunction { private final HoodieSchema tableSchema; private final HoodieSchema requiredSchema; private final String mergeType; - private transient HoodieFileGroupReader fileGroupReader; public HoodieSplitReaderFunction( Configuration configuration, @@ -72,28 +70,39 @@ public HoodieSplitReaderFunction( } @Override - public RecordsWithSplitIds> read(HoodieSourceSplit split) { - final String splitId = split.splitId(); + protected ClosableIterator createRecordIterator(HoodieSourceSplit split) { HoodieTableMetaClient metaClient = StreamerUtil.metaClientForReader(conf, getHadoopConf()); - + // Closing the returned iterator cascade-closes the whole HoodieFileGroupReader, so the base + // class only has to close the iterator in closeCurrentSplit(). But getClosableIterator() runs + // initRecordIterators(), which opens the reader's base-file iterator / record buffer before the + // wrapping iterator is returned; if it throws, the reader is only a local here and nothing else + // would close it. Keep it in a local and close it in the failure path. + HoodieFileGroupReader fileGroupReader = createFileGroupReader(split, metaClient); try { - this.fileGroupReader = createFileGroupReader(split, metaClient); - final ClosableIterator recordIterator = fileGroupReader.getClosableIterator(); - BatchRecords records = BatchRecords.forRecords(splitId, recordIterator, split.getFileOffset(), split.getConsumed()); - records.seek(split.getConsumed()); - return records; + return fileGroupReader.getClosableIterator(); } catch (IOException e) { + closeSuppressing(fileGroupReader, e); throw new HoodieIOException("Failed to read from file group: " + split.getFileId(), e); + } catch (RuntimeException | Error e) { + closeSuppressing(fileGroupReader, e); + throw e; } } - @Override - public void close() throws Exception { - if (fileGroupReader != null) { - fileGroupReader.close(); + /** Closes {@code reader}, attaching any close failure to {@code primary} as a suppressed exception. */ + private static void closeSuppressing(HoodieFileGroupReader reader, Throwable primary) { + try { + reader.close(); + } catch (Exception closeError) { + primary.addSuppressed(closeError); } } + @Override + protected RowType producedRowType() { + return HoodieSchemaConverter.convertToRowType(requiredSchema); + } + /** * Creates a {@link HoodieFileGroupReader} for the given split. * @@ -101,7 +110,7 @@ public void close() throws Exception { * @param metaClient The table meta client for schema and config resolution * @return A {@link HoodieFileGroupReader} instance */ - private HoodieFileGroupReader createFileGroupReader(HoodieSourceSplit split, HoodieTableMetaClient metaClient) { + protected HoodieFileGroupReader createFileGroupReader(HoodieSourceSplit split, HoodieTableMetaClient metaClient) { // Create FileSlice from split information FileSlice fileSlice = new FileSlice( new HoodieFileGroupId(split.getPartitionPath(), split.getFileId()), diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/SplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/SplitReaderFunction.java index 6f7bf0f18ebe2..46fb343aa3d4d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/SplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/SplitReaderFunction.java @@ -18,19 +18,55 @@ package org.apache.hudi.source.reader.function; -import org.apache.hudi.source.reader.HoodieRecordWithPosition; +import org.apache.hudi.source.reader.BatchRecords; import org.apache.hudi.source.split.HoodieSourceSplit; -import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; - import java.io.Serializable; +import java.util.function.BooleanSupplier; /** - * Interface for split read function. + * Interface for a split read function. + * + *

    A reader function is a stateful, per-split cursor driven entirely on the Flink split-fetcher + * thread by {@link org.apache.hudi.source.reader.HoodieSourceSplitReader#fetch()}: + * {@link #open(HoodieSourceSplit)} creates the record iterator and its underlying I/O resources for + * a split, {@link #readBatch(HoodieSourceSplit, int, java.util.function.BooleanSupplier)} drains the next bounded minibatch, and + * {@link #closeCurrentSplit()} releases the split's resources once it is exhausted. Because open, + * read and close all run on the same thread, no record or I/O resource is ever touched concurrently. + * + * @param record type */ public interface SplitReaderFunction extends Serializable { - RecordsWithSplitIds> read(HoodieSourceSplit split); + /** + * Opens {@code split} for reading: creates the record iterator and its underlying I/O resources, + * and skips the records already consumed ({@link HoodieSourceSplit#getConsumed()}) so a recovered + * split resumes at the right position. + */ + void open(HoodieSourceSplit split); + + /** + * Drains up to {@code batchSize} records from the currently open split into a materialized + * {@link BatchRecords} minibatch. Returns {@code null} once the split is exhausted. + * + *

    {@code wakeupSignal} is polled between records: once it returns {@code true} materialization + * stops early and the records buffered so far are returned as a (non-finishing) partial minibatch; + * if nothing has been buffered yet {@code null} is returned. This lets a blocking {@code fetch()} + * unblock promptly on {@link org.apache.hudi.source.reader.HoodieSourceSplitReader#wakeUp()} + * without touching any resource off the split-fetcher thread. + */ + BatchRecords readBatch(HoodieSourceSplit split, int batchSize, BooleanSupplier wakeupSignal); + + /** + * Closes the currently open split's iterator and I/O resources. Called when the split is + * exhausted, a read fails, or the read is stopped early. Safe to call when no split is open. + */ + void closeCurrentSplit(); + /** + * Closes the reader function entirely (idempotent). Invoked by + * {@link org.apache.hudi.source.reader.HoodieSourceSplitReader#close()} on the split-fetcher + * thread. + */ void close() throws Exception; } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java index 5f47814942173..20f6801aec0f9 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java @@ -92,6 +92,11 @@ private CdcIterators() { /** * Iterates over an ordered sequence of {@link HoodieCDCFileSplit}s, delegating * per-split record reading to a user-supplied factory function. + * + *

    Not thread-safe by design: in the Source V2 read path this iterator is created, drained into a + * materialized minibatch, and closed entirely on the single split-fetcher thread (see + * {@code AbstractSplitReaderFunction}); the legacy {@link CdcInputFormat} path likewise reads and + * closes it on one thread. No method is ever invoked concurrently, so no synchronization is needed. */ public static class CdcFileSplitsIterator implements ClosableIterator { private CdcImageManager imageManager; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestBatchRecords.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestBatchRecords.java index 69af5702f1f42..5b111dddcc035 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestBatchRecords.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestBatchRecords.java @@ -18,14 +18,11 @@ package org.apache.hudi.source.reader; -import org.apache.hudi.common.util.collection.ClosableIterator; - import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -36,16 +33,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Test cases for {@link BatchRecords}. + * Test cases for {@link BatchRecords}, which now holds a materialized, bounded minibatch of records. */ public class TestBatchRecords { @Test - public void testForRecordsWithEmptyIterator() { + public void testForRecordsWithEmptyList() { String splitId = "test-split-1"; - ClosableIterator emptyIterator = createClosableIterator(Collections.emptyList()); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, emptyIterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, Collections.emptyList(), 0, 0L); assertNotNull(batchRecords); assertEquals(splitId, batchRecords.nextSplit()); @@ -57,9 +53,8 @@ public void testForRecordsWithEmptyIterator() { public void testForRecordsWithMultipleRecords() { String splitId = "test-split-2"; List records = Arrays.asList("record1", "record2", "record3"); - ClosableIterator iterator = createClosableIterator(records); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, 0, 0L); // Verify split ID assertEquals(splitId, batchRecords.nextSplit()); @@ -86,46 +81,13 @@ public void testForRecordsWithMultipleRecords() { assertNull(batchRecords.nextRecordFromSplit()); } - @Test - public void testSeekToStartingOffset() { - String splitId = "test-split-3"; - List records = Arrays.asList("record1", "record2", "record3", "record4", "record5"); - ClosableIterator iterator = createClosableIterator(records); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 2L); - batchRecords.seek(2L); - - // After seeking to offset 2, we should start from record3 - batchRecords.nextSplit(); - - HoodieRecordWithPosition record = batchRecords.nextRecordFromSplit(); - assertNotNull(record); - assertEquals("record3", record.record()); - } - - @Test - public void testSeekBeyondAvailableRecords() { - String splitId = "test-split-4"; - List records = Arrays.asList("record1", "record2"); - ClosableIterator iterator = createClosableIterator(records); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); - - IllegalStateException exception = assertThrows(IllegalStateException.class, () -> { - batchRecords.seek(10L); - }); - - assertTrue(exception.getMessage().contains("Invalid starting record offset")); - } - @Test public void testFileOffsetPersistence() { String splitId = "test-split-5"; int fileOffset = 5; List records = Arrays.asList("record1", "record2"); - ClosableIterator iterator = createClosableIterator(records); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, fileOffset, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, fileOffset, 0L); batchRecords.nextSplit(); HoodieRecordWithPosition record1 = batchRecords.nextRecordFromSplit(); @@ -140,12 +102,11 @@ public void testFileOffsetPersistence() { @Test public void testConstructorWithFinishedSplits() { String splitId = "test-split-7"; - List records = Arrays.asList("record1"); - ClosableIterator iterator = createClosableIterator(records); + List records = Collections.singletonList("record1"); Set finishedSplits = new HashSet<>(Arrays.asList("split1", "split2")); BatchRecords batchRecords = new BatchRecords<>( - splitId, iterator, 0, 0L, finishedSplits); + splitId, records, 0, 0L, finishedSplits); assertEquals(2, batchRecords.finishedSplits().size()); assertTrue(batchRecords.finishedSplits().contains("split1")); @@ -157,10 +118,9 @@ public void testRecordOffsetIncrementsCorrectly() { String splitId = "test-split-8"; long startingRecordOffset = 10L; List records = Arrays.asList("A", "B", "C"); - ClosableIterator iterator = createClosableIterator(records); BatchRecords batchRecords = BatchRecords.forRecords( - splitId, iterator, 0, startingRecordOffset); + splitId, records, 0, startingRecordOffset); batchRecords.nextSplit(); // First record should be at startingRecordOffset + 1 @@ -176,13 +136,37 @@ public void testRecordOffsetIncrementsCorrectly() { assertEquals(startingRecordOffset + 3, record3.recordOffset()); } + @Test + public void testOffsetContinuityAcrossMinibatches() { + // Two consecutive minibatches of the same split: the reader function threads the running + // record offset so that offsets stay monotonic across batch boundaries (batch 2 starts where + // batch 1 left off). This mirrors AbstractSplitReaderFunction#readBatch. + String splitId = "test-split-continuity"; + + BatchRecords batch1 = BatchRecords.forRecords(splitId, Arrays.asList("a", "b", "c"), 0, 0L); + batch1.nextSplit(); + assertEquals(1L, batch1.nextRecordFromSplit().recordOffset()); + assertEquals(2L, batch1.nextRecordFromSplit().recordOffset()); + assertEquals(3L, batch1.nextRecordFromSplit().recordOffset()); + assertNull(batch1.nextRecordFromSplit()); + + // batch 2 opens at startingRecordOffset = 3 (the count already consumed by batch 1) + BatchRecords batch2 = BatchRecords.forRecords(splitId, Arrays.asList("d", "e"), 0, 3L); + batch2.nextSplit(); + HoodieRecordWithPosition d = batch2.nextRecordFromSplit(); + assertEquals("d", d.record()); + assertEquals(4L, d.recordOffset()); + HoodieRecordWithPosition e = batch2.nextRecordFromSplit(); + assertEquals("e", e.record()); + assertEquals(5L, e.recordOffset()); + } + @Test public void testSplitIdReturnedOnlyOnce() { String splitId = "test-split-9"; - List records = Arrays.asList("record1"); - ClosableIterator iterator = createClosableIterator(records); + List records = Collections.singletonList("record1"); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, 0, 0L); assertEquals(splitId, batchRecords.nextSplit()); assertNull(batchRecords.nextSplit()); @@ -191,187 +175,64 @@ public void testSplitIdReturnedOnlyOnce() { } @Test - public void testRecycleClosesIterator() { + public void testRecycleIsNoOp() { + // The minibatch is fully materialized, so recycle() releases nothing and is a harmless no-op: + // it must not throw and must not affect the still-readable records. String splitId = "test-split-10"; List records = Arrays.asList("record1", "record2"); - MockClosableIterator mockIterator = new MockClosableIterator<>(records); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, mockIterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, 0, 0L); + batchRecords.nextSplit(); batchRecords.recycle(); - assertTrue(mockIterator.isClosed(), "Iterator should be closed after recycle"); - } - - @Test - public void testRecycleWithNullIterator() { - // Test that recycle handles null iterator gracefully (though in practice this shouldn't happen) - // This tests the null check in recycle() method - String splitId = "test-split-11"; - ClosableIterator emptyIterator = createClosableIterator(Collections.emptyList()); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, emptyIterator, 0, 0L); - - // Should not throw exception - batchRecords.recycle(); + assertNotNull(batchRecords.nextRecordFromSplit(), "records remain readable after recycle"); + assertNotNull(batchRecords.nextRecordFromSplit()); + assertNull(batchRecords.nextRecordFromSplit()); } @Test public void testNextRecordFromSplitAfterExhaustion() { String splitId = "test-split-12"; - List records = Arrays.asList("record1"); - ClosableIterator iterator = createClosableIterator(records); + List records = Collections.singletonList("record1"); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, 0, 0L); batchRecords.nextSplit(); // Read the only record assertNotNull(batchRecords.nextRecordFromSplit()); - // After exhaustion, should return null + // After exhaustion, should return null (and stay null) assertNull(batchRecords.nextRecordFromSplit()); assertNull(batchRecords.nextRecordFromSplit()); } - @Test - public void testSeekWithZeroOffset() { - String splitId = "test-split-13"; - List records = Arrays.asList("record1", "record2", "record3"); - ClosableIterator iterator = createClosableIterator(records); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); - - // Seeking to 0 should not skip any records - batchRecords.seek(0L); - batchRecords.nextSplit(); - - HoodieRecordWithPosition record = batchRecords.nextRecordFromSplit(); - assertNotNull(record); - assertEquals("record1", record.record()); - } - @Test public void testConstructorNullValidation() { String splitId = "test-split-14"; - List records = Arrays.asList("record1"); - ClosableIterator iterator = createClosableIterator(records); + List records = Collections.singletonList("record1"); // Test null finishedSplits - assertThrows(IllegalArgumentException.class, () -> { - new BatchRecords<>(splitId, iterator, 0, 0L, null); - }); - - // Test null recordIterator - assertThrows(IllegalArgumentException.class, () -> { - new BatchRecords<>(splitId, null, 0, 0L, new HashSet<>()); - }); + assertThrows(IllegalArgumentException.class, () -> + new BatchRecords<>(splitId, records, 0, 0L, null)); + + // Test null records + assertThrows(IllegalArgumentException.class, () -> + new BatchRecords<>(splitId, null, 0, 0L, new HashSet<>())); } @Test public void testRecordPositionReusability() { String splitId = "test-split-15"; List records = Arrays.asList("A", "B", "C"); - ClosableIterator iterator = createClosableIterator(records); - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 0, 0L); + BatchRecords batchRecords = BatchRecords.forRecords(splitId, records, 0, 0L); batchRecords.nextSplit(); HoodieRecordWithPosition pos1 = batchRecords.nextRecordFromSplit(); HoodieRecordWithPosition pos2 = batchRecords.nextRecordFromSplit(); - // Should reuse the same object + // Should reuse the same object (safe because SourceReaderBase emits each record before the next) assertTrue(pos1 == pos2, "Should reuse the same HoodieRecordWithPosition object"); } - - @Test - public void testSeekUpdatesPosition() { - String splitId = "test-split-16"; - List records = Arrays.asList("r1", "r2", "r3", "r4", "r5"); - ClosableIterator iterator = createClosableIterator(records); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, iterator, 5, 10L); - - // Seek to offset 3 - batchRecords.seek(3L); - - batchRecords.nextSplit(); - - // After seeking 3, next record should be r4 (4th record) - HoodieRecordWithPosition record = batchRecords.nextRecordFromSplit(); - assertNotNull(record); - assertEquals("r4", record.record()); - } - - @Test - public void testIteratorClosedAfterExhaustion() { - String splitId = "test-split-17"; - List records = Arrays.asList("record1"); - MockClosableIterator mockIterator = new MockClosableIterator<>(records); - - BatchRecords batchRecords = BatchRecords.forRecords(splitId, mockIterator, 0, 0L); - batchRecords.nextSplit(); - - // Read records - batchRecords.nextRecordFromSplit(); - - // Trigger close operation - batchRecords.nextRecordFromSplit(); - - // After exhaustion, nextRecordFromSplit should close the iterator - assertTrue(mockIterator.isClosed(), "Iterator should be closed after exhaustion"); - } - - /** - * Helper method to create a ClosableIterator from a list of items. - */ - private ClosableIterator createClosableIterator(List items) { - Iterator iterator = items.iterator(); - return new ClosableIterator() { - @Override - public void close() { - // No-op for test - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public T next() { - return iterator.next(); - } - }; - } - - /** - * Mock closable iterator for testing close behavior. - */ - private static class MockClosableIterator implements ClosableIterator { - private final Iterator iterator; - private boolean closed = false; - - public MockClosableIterator(List items) { - this.iterator = items.iterator(); - } - - @Override - public void close() { - closed = true; - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public T next() { - return iterator.next(); - } - - public boolean isClosed() { - return closed; - } - } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java index 4ceb09caee43d..2ea14b0ac0060 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java @@ -21,7 +21,6 @@ import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.source.reader.function.SplitReaderFunction; import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.source.split.SerializableComparator; @@ -33,11 +32,17 @@ import org.mockito.Mockito; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -184,13 +189,235 @@ public void testClose() throws Exception { } @Test - public void testWakeUp() { + public void testWakeUp() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); - // wakeUp is a no-op, should not throw any exception + // wakeUp() now sets a flag but must not throw, and the flag is reset at the top of each fetch() + // so a wakeUp with no in-flight drain leaves the next fetch() unaffected. reader.wakeUp(); + RecordsWithSplitIds> result = reader.fetch(); + assertNotNull(result); + assertNull(result.nextSplit()); + } + + // ------------------------------------------------------------------------- + // wakeUp — cooperative cancellation of the minibatch drain + // ------------------------------------------------------------------------- + + @Test + public void testWakeUpMidDrainReturnsPartialBatchAndResumes() throws IOException { + // A wakeUp() landing after the first record stops the drain between records: fetch() returns the + // 1 record buffered so far as a NON-finishing batch without closing the split; a later fetch() + // resumes the rest with continuous offsets, and the split is closed only once at true EOF. + List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + boolean[] fired = {false}; + readerFunction.setDrainProbe((buffered, hasNext) -> { + if (buffered == 1 && !fired[0]) { + fired[0] = true; + reader.wakeUp(); + } + }); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + // First fetch: woken after r1 -> partial batch of 1, split not finished, not closed. + RecordsWithSplitIds> b1 = reader.fetch(); + assertEquals(split.splitId(), b1.nextSplit()); + HoodieRecordWithPosition first = b1.nextRecordFromSplit(); + assertNotNull(first); + assertEquals("r1", first.record()); + assertEquals(1L, first.recordOffset()); + assertNull(b1.nextRecordFromSplit(), "drain must stop at the first record on wake-up"); + assertTrue(b1.finishedSplits().isEmpty(), "woken split must not be finished"); + assertEquals(0, readerFunction.getCloseCurrentSplitCount(), "woken split must not be closed"); + + // Second fetch: resumes the remaining records with continuous offsets (2..5). + RecordsWithSplitIds> b2 = reader.fetch(); + assertEquals(split.splitId(), b2.nextSplit()); + HoodieRecordWithPosition next = b2.nextRecordFromSplit(); + assertNotNull(next); + assertEquals("r2", next.record()); + assertEquals(2L, next.recordOffset(), "offset continues across the wake boundary"); + assertEquals(3, drainRecordCount(b2), "r3, r4, r5 remain"); + assertTrue(b2.finishedSplits().isEmpty()); + + // Third fetch: true EOF -> finish signal, split closed exactly once, opened exactly once. + RecordsWithSplitIds> b3 = reader.fetch(); + assertTrue(b3.finishedSplits().contains(split.splitId())); + assertEquals(1, readerFunction.getCloseCurrentSplitCount()); + assertEquals(1, readerFunction.getOpenCount()); + } + + @Test + public void testWakeUpBeforeAnyRecordReturnsEmptyNonFinishingBatch() throws IOException { + // A wakeUp() landing before any record is buffered must return an empty NON-finishing batch, not + // a finish signal: the split stays open (not closed) and resumes on the next fetch(). + List testData = Arrays.asList("r1", "r2", "r3"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + boolean[] fired = {false}; + readerFunction.setDrainProbe((buffered, hasNext) -> { + // Wake at the start of the drain (count 0) while data is still available. + if (buffered == 0 && hasNext && !fired[0]) { + fired[0] = true; + reader.wakeUp(); + } + }); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + RecordsWithSplitIds> b1 = reader.fetch(); + assertNull(b1.nextSplit(), "empty non-finishing batch carries no split records"); + assertTrue(b1.finishedSplits().isEmpty(), "must not finish the split on wake-up"); + assertEquals(0, readerFunction.getCloseCurrentSplitCount(), "split must stay open"); + + // Next fetch resumes and returns all records (the one-shot wake has fired). + RecordsWithSplitIds> b2 = reader.fetch(); + assertEquals(split.splitId(), b2.nextSplit()); + assertEquals(3, drainRecordCount(b2)); + } + + @Test + public void testWakeUpCoincidingWithEofDefersFinishByOneFetch() throws IOException { + // readBatch returning null is ambiguous between true-EOF and woken-empty. When a wakeUp lands + // exactly at genuine exhaustion, fetch() returns an empty non-finishing batch once (deferring the + // finish), and the next fetch() finishes the split - it must never stall. + List testData = Arrays.asList("r1", "r2"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + boolean[] fired = {false}; + readerFunction.setDrainProbe((buffered, hasNext) -> { + // Wake only at the start of a drain that finds the cursor already exhausted. + if (buffered == 0 && !hasNext && !fired[0]) { + fired[0] = true; + reader.wakeUp(); + } + }); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + // First fetch drains both records (cursor still had data at drain start, so no wake). + RecordsWithSplitIds> b1 = reader.fetch(); + assertEquals(2, drainRecordCount(b1)); + assertEquals(0, readerFunction.getCloseCurrentSplitCount()); + + // Second fetch: cursor is exhausted at drain start -> wake fires -> empty non-finishing, deferred. + RecordsWithSplitIds> b2 = reader.fetch(); + assertTrue(b2.finishedSplits().isEmpty(), "finish deferred by the coinciding wake-up"); + assertEquals(0, readerFunction.getCloseCurrentSplitCount()); + + // Third fetch: no wake now -> true EOF finishes and closes the split. + RecordsWithSplitIds> b3 = reader.fetch(); + assertTrue(b3.finishedSplits().contains(split.splitId())); + assertEquals(1, readerFunction.getCloseCurrentSplitCount()); + } + + @Test + public void testWakeUpWithLimitReachedStillFinishesSplit() throws IOException { + // Guard: the woken-resume short-circuit must live strictly on the readBatch-returned-null path. + // A wakeUp coinciding with the limit-reached path must still finish the split, never loop forever + // returning non-finishing batches. The limiter wakes the reader exactly when the limit is reached. + List testData = Arrays.asList("r1", "r2", "r3"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + AtomicReference> holder = new AtomicReference<>(); + RecordLimiter wakingLimiter = new RecordLimiter(2L) { + @Override + public boolean isLimitReached() { + boolean reached = super.isLimitReached(); + if (reached && holder.get() != null) { + holder.get().wakeUp(); + } + return reached; + } + }; + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(wakingLimiter)); + holder.set(reader); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + // First fetch returns the split data; the limit wrapper caps consumption at 2 records. + RecordsWithSplitIds> b1 = reader.fetch(); + assertEquals(split.splitId(), b1.nextSplit()); + assertEquals(2, drainRecordCount(b1), "limit caps drained records at 2"); + + // Second fetch hits the limit-reached path (which wakes the reader); it must finish, not defer. + RecordsWithSplitIds> b2 = reader.fetch(); + assertTrue(b2.finishedSplits().contains(split.splitId()), "limit-reached path must finish the split"); + assertEquals(1, readerFunction.getCloseCurrentSplitCount()); + } + + @Test + public void testWakeUpPartialBatchRespectsLimit() throws IOException { + // A partial (woken) batch must not double-count against the pushed-down limit: with limit=3 over a + // 5-record split and a wake after the first record, the total emitted across batches stays at 3. + List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(3L))); + boolean[] fired = {false}; + readerFunction.setDrainProbe((buffered, hasNext) -> { + if (buffered == 1 && !fired[0]) { + fired[0] = true; + reader.wakeUp(); + } + }); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + int total = 0; + // Drain fetches until the split is finished; assert the limit caps the total at 3. + for (int i = 0; i < 10; i++) { + RecordsWithSplitIds> batch = reader.fetch(); + if (batch.nextSplit() != null) { + total += drainRecordCount(batch); + } + if (batch.finishedSplits().contains(split.splitId())) { + break; + } + } + assertEquals(3, total, "pushed-down limit must cap the total across partial woken batches"); + } + + @Test + public void testCloseReleasesWokenStillOpenSplit() throws Exception { + // Unit proxy for the real shutdown path (SplitFetcher.run() -> splitReader.close() on the fetcher + // thread): a split left open by a wake-up is released when the reader is closed. + List testData = Arrays.asList("r1", "r2", "r3"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + boolean[] fired = {false}; + readerFunction.setDrainProbe((buffered, hasNext) -> { + if (buffered == 1 && !fired[0]) { + fired[0] = true; + reader.wakeUp(); + } + }); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + // Woken fetch leaves the split open (not closed). + reader.fetch(); + assertEquals(0, readerFunction.getCloseCurrentSplitCount()); + + // close() on the (split-fetcher) thread releases the still-open split. + reader.close(); + assertEquals(1, readerFunction.getCloseCurrentSplitCount()); + assertTrue(readerFunction.isClosed()); } @Test @@ -222,7 +449,7 @@ public void testReaderFunctionCalledCorrectly() throws IOException { reader.fetch(); - assertEquals(1, readerFunction.getReadCount()); + assertEquals(1, readerFunction.getOpenCount()); assertEquals(split, readerFunction.getLastReadSplit()); } @@ -247,7 +474,7 @@ public void testFetchEmptyResultWhenNoSplitsAdded() throws IOException { assertNotNull(result); assertNull(result.nextSplit()); - assertEquals(0, readerFunction.getReadCount(), "Should not read any splits"); + assertEquals(0, readerFunction.getOpenCount(), "Should not read any splits"); } @Test @@ -348,8 +575,8 @@ public void testLimitDrainsRemainingSplitsWhenLimitReached() throws IOException assertTrue(drainBatch.finishedSplits().contains(split2.splitId()), "split2 should be drained as finished once limit is reached"); assertNull(drainBatch.nextSplit()); - // readerFunction.read() was called only once (for split1, never for split2) - assertEquals(1, readerFunction.getReadCount()); + // readerFunction was opened only once (for split1, never for split2) + assertEquals(1, readerFunction.getOpenCount()); } @Test @@ -369,8 +596,8 @@ public void testLimitZeroDrainsAllSplitsImmediately() throws IOException { assertTrue(finished.contains(split1.splitId())); assertTrue(finished.contains(split2.splitId())); assertNull(batch.nextSplit()); - // readerFunction.read() was never called - assertEquals(0, readerFunction.getReadCount()); + // readerFunction was never opened + assertEquals(0, readerFunction.getOpenCount()); } @Test @@ -450,6 +677,78 @@ public void testNoLimitSentinelReturnsAllRecords() throws IOException { assertEquals(5, drainRecordCount(batch)); } + // ------------------------------------------------------------------------- + // Minibatch / resume tests + // ------------------------------------------------------------------------- + + @Test + public void testSplitSpanningMultipleMinibatches() throws IOException { + // A split larger than the mini-batch bound (2048) is emitted as multiple non-finished batches, + // then one finish signal. The reader function is opened once and closed once across the split, + // and the record offset stays continuous across the minibatch boundary. + int n = 2049; + List testData = IntStream.range(0, n).mapToObj(i -> "r" + i).collect(Collectors.toList()); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + // First minibatch: 2048 records, offsets 1..2048, split not yet finished. + RecordsWithSplitIds> b1 = reader.fetch(); + assertEquals(split.splitId(), b1.nextSplit()); + long lastOffset = 0L; + int c1 = 0; + HoodieRecordWithPosition rec; + while ((rec = b1.nextRecordFromSplit()) != null) { + lastOffset = rec.recordOffset(); + c1++; + } + assertEquals(2048, c1); + assertEquals(2048L, lastOffset); + assertTrue(b1.finishedSplits().isEmpty(), "split not finished after the first minibatch"); + + // Second minibatch: the remaining record, offset continues at 2049. + RecordsWithSplitIds> b2 = reader.fetch(); + assertEquals(split.splitId(), b2.nextSplit()); + HoodieRecordWithPosition only = b2.nextRecordFromSplit(); + assertNotNull(only); + assertEquals(2049L, only.recordOffset()); + assertNull(b2.nextRecordFromSplit()); + assertTrue(b2.finishedSplits().isEmpty()); + + // Third fetch: split exhausted -> finish signal. + RecordsWithSplitIds> b3 = reader.fetch(); + assertTrue(b3.finishedSplits().contains(split.splitId())); + + assertEquals(1, readerFunction.getOpenCount(), "split opened exactly once across minibatches"); + assertEquals(1, readerFunction.getCloseCurrentSplitCount(), "split closed exactly once at EOF"); + } + + @Test + public void testResumeSkipsConsumedRecords() throws IOException { + // A recovered split carries a consumed offset; open() skips that many records and the emitted + // offsets resume at consumed+1. + List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); + HoodieSourceSplitReader reader = + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + + HoodieSourceSplit split = createTestSplit(1, "file1"); + split.updatePosition(0, 2L); // 2 records already consumed before recovery + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + RecordsWithSplitIds> batch = reader.fetch(); + assertEquals(split.splitId(), batch.nextSplit()); + HoodieRecordWithPosition first = batch.nextRecordFromSplit(); + assertNotNull(first); + assertEquals("r3", first.record(), "should resume past the 2 consumed records"); + assertEquals(3L, first.recordOffset(), "offset resumes at consumed + 1"); + // r4, r5 remain + assertEquals(2, drainRecordCount(batch)); + } + /** * Fetches the next batch that contains actual split data, skipping split-finish signal batches. * Split-finish batches have non-empty {@code finishedSplits()} but no records. @@ -496,14 +795,29 @@ private HoodieSourceSplit createTestSplit(int splitNum, String fileId) { } /** - * Test implementation of SplitReaderFunction. + * Test implementation of the stateful {@link SplitReaderFunction} cursor contract: {@code open} + * materializes the split's data and honors the consumed-offset skip, {@code readBatch} drains a + * bounded minibatch, and {@code closeCurrentSplit}/{@code close} release it. */ private static class TestSplitReaderFunction implements SplitReaderFunction { private final List testData; - private int readCount = 0; + private int openCount = 0; + private int closeCurrentSplitCount = 0; private HoodieSourceSplit lastReadSplit = null; private boolean closed = false; + // per-split cursor + private Iterator cursor; + private long nextRecordOffset; + + // Optional hook invoked as (bufferedCount, cursorHasNext) at readBatch start (count 0) and after + // each buffered record, so a test can call reader.wakeUp() at a precise point in the drain. + private BiConsumer drainProbe; + + void setDrainProbe(BiConsumer drainProbe) { + this.drainProbe = drainProbe; + } + public TestSplitReaderFunction() { this(Collections.emptyList()); } @@ -513,25 +827,63 @@ public TestSplitReaderFunction(List testData) { } @Override - public RecordsWithSplitIds> read(HoodieSourceSplit split) { - readCount++; + public void open(HoodieSourceSplit split) { + openCount++; lastReadSplit = split; - ClosableIterator iterator = createClosableIterator(testData); - return BatchRecords.forRecords( - split.splitId(), - iterator, - split.getFileOffset(), - split.getConsumed() - ); + cursor = testData.iterator(); + long consumed = split.getConsumed(); + for (long i = 0; i < consumed; i++) { + if (cursor.hasNext()) { + cursor.next(); + } else { + throw new IllegalStateException( + "Invalid starting record offset " + consumed + " for split " + split.splitId()); + } + } + nextRecordOffset = consumed; } @Override - public void close() throws Exception { + public BatchRecords readBatch(HoodieSourceSplit split, int batchSize, BooleanSupplier wakeupSignal) { + List buffer = new ArrayList<>(); + if (drainProbe != null) { + drainProbe.accept(0, cursor.hasNext()); + } + while (buffer.size() < batchSize && !wakeupSignal.getAsBoolean() && cursor.hasNext()) { + buffer.add(cursor.next()); + if (drainProbe != null) { + drainProbe.accept(buffer.size(), cursor.hasNext()); + } + } + if (buffer.isEmpty()) { + return null; + } + long startingRecordOffset = nextRecordOffset; + nextRecordOffset += buffer.size(); + return BatchRecords.forRecords(split.splitId(), buffer, split.getFileOffset(), startingRecordOffset); + } + + @Override + public void closeCurrentSplit() { + closeCurrentSplitCount++; + cursor = null; + } + + @Override + public void close() { + // Mirror AbstractSplitReaderFunction#close(): releasing the reader also releases any split + // still open (e.g. one left open by a wake-up), on the split-fetcher thread. + closeCurrentSplit(); closed = true; } - public int getReadCount() { - return readCount; + // Number of splits opened; mirrors the old per-split read() count. + public int getOpenCount() { + return openCount; + } + + public int getCloseCurrentSplitCount() { + return closeCurrentSplitCount; } public HoodieSourceSplit getLastReadSplit() { @@ -541,25 +893,5 @@ public HoodieSourceSplit getLastReadSplit() { public boolean isClosed() { return closed; } - - private ClosableIterator createClosableIterator(List items) { - Iterator iterator = items.iterator(); - return new ClosableIterator() { - @Override - public void close() { - // No-op - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public String next() { - return iterator.next(); - } - }; - } } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestAbstractSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestAbstractSplitReaderFunction.java index 8b48772ab5f7d..ae95089dd8b0d 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestAbstractSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestAbstractSplitReaderFunction.java @@ -19,14 +19,20 @@ package org.apache.hudi.source.reader.function; import org.apache.flink.configuration.Configuration; -import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.expressions.FieldReferenceExpression; import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.types.AtomicDataType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.types.RowKind; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.source.ExpressionPredicates; +import org.apache.hudi.source.reader.BatchRecords; import org.apache.hudi.source.reader.HoodieRecordWithPosition; import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.table.format.InternalSchemaManager; @@ -39,10 +45,13 @@ import java.io.File; import java.util.Collections; import java.util.List; +import java.util.function.BooleanSupplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -70,8 +79,8 @@ public void setUp() { /** * Minimal concrete implementation that exposes the protected helper methods - * ({@code getWriteConfig()} / {@code getHadoopConf()}) for testing, and leaves - * {@code read()} / {@code close()} as no-ops. + * ({@code getWriteConfig()} / {@code getHadoopConf()}) for testing. The template methods return an + * empty iterator / a trivial row type; the constructor and singleton tests never drive the cursor. */ private static class MinimalSplitReaderFunction extends AbstractSplitReaderFunction { @@ -84,12 +93,13 @@ private static class MinimalSplitReaderFunction extends AbstractSplitReaderFunct } @Override - public RecordsWithSplitIds> read(HoodieSourceSplit split) { - return null; + protected ClosableIterator createRecordIterator(HoodieSourceSplit split) { + return ClosableIterator.wrap(Collections.emptyIterator()); } @Override - public void close() throws Exception { + protected RowType producedRowType() { + return RowType.of(new IntType()); } HoodieWriteConfig writeConfigForTest() { @@ -243,4 +253,135 @@ public void testTwoInstancesHaveStableIndependentWriteConfig() { assertSame(wc2, fn2.writeConfigForTest(), "fn2's writeConfig must remain the same singleton across calls"); } + + // ----------------------------------------------------------------------- + // readBatch — copy-on-materialize (object-reuse regression guard) + // ----------------------------------------------------------------------- + + @Test + public void testReadBatchCopiesReusedRecordObjects() { + // Columnar readers return the SAME mutable RowData object on every next(); readBatch must copy + // each record, otherwise every entry in a materialized minibatch would alias the last row. + ReusedObjectSplitReaderFunction fn = + new ReusedObjectSplitReaderFunction(conf, mockInternalSchemaManager, 3); + HoodieSourceSplit split = createSplit(); + + fn.open(split); + BatchRecords batch = fn.readBatch(split, 10, () -> false); + assertNotNull(batch); + batch.nextSplit(); + + RowData r0 = batch.nextRecordFromSplit().record(); + RowData r1 = batch.nextRecordFromSplit().record(); + RowData r2 = batch.nextRecordFromSplit().record(); + assertNull(batch.nextRecordFromSplit()); + + // Distinct values despite the source reusing a single object. + assertEquals(0, r0.getInt(0)); + assertEquals(1, r1.getInt(0)); + assertEquals(2, r2.getInt(0)); + assertNotSame(r0, r1, "records must be copies, not the reused source object"); + assertNotSame(r1, r2); + // RowKind is preserved across the copy. + assertEquals(RowKind.DELETE, r0.getRowKind()); + assertEquals(RowKind.INSERT, r1.getRowKind()); + assertEquals(RowKind.DELETE, r2.getRowKind()); + } + + // ----------------------------------------------------------------------- + // readBatch — wake-up signal (cooperative cancellation between records) + // ----------------------------------------------------------------------- + + @Test + public void testReadBatchStopsOnWakeupSignal() { + // The wakeupSignal is polled between records; once it trips, materialization stops early and the + // records buffered so far are returned as a partial minibatch with continuous offsets. + ReusedObjectSplitReaderFunction fn = + new ReusedObjectSplitReaderFunction(conf, mockInternalSchemaManager, 5); + HoodieSourceSplit split = createSplit(); + fn.open(split); + + // Returns false, false, true: the loop buffers 2 records, then the 3rd poll stops it. + int[] polls = {0}; + BooleanSupplier signal = () -> (++polls[0]) > 2; + + BatchRecords batch = fn.readBatch(split, 10, signal); + assertNotNull(batch); + batch.nextSplit(); + + // nextRecordFromSplit() returns the same reused position wrapper each call, so read each record's + // value/offset before advancing. Offsets are 1-based and continuous from the starting offset (0). + HoodieRecordWithPosition rec = batch.nextRecordFromSplit(); + assertNotNull(rec); + assertEquals(0, rec.record().getInt(0)); + assertEquals(1L, rec.recordOffset()); + + rec = batch.nextRecordFromSplit(); + assertNotNull(rec); + assertEquals(1, rec.record().getInt(0)); + assertEquals(2L, rec.recordOffset()); + + assertNull(batch.nextRecordFromSplit(), "materialization must stop at 2 records on wake-up"); + } + + @Test + public void testReadBatchReturnsNullWhenWokenBeforeAnyRecord() { + // A wake-up that lands before the first record is buffered yields an empty batch, signalled as + // null (the same sentinel as EOF); HoodieSourceSplitReader.fetch() disambiguates the two. + ReusedObjectSplitReaderFunction fn = + new ReusedObjectSplitReaderFunction(conf, mockInternalSchemaManager, 5); + HoodieSourceSplit split = createSplit(); + fn.open(split); + + assertNull(fn.readBatch(split, 10, () -> true)); + } + + private HoodieSourceSplit createSplit() { + return new HoodieSourceSplit( + 1, "base", Option.of(Collections.emptyList()), "/tbl", "/part", + "read_optimized", "19700101000000000", "file1", Option.empty()); + } + + /** + * Reader function whose iterator returns the SAME mutable {@link GenericRowData} instance on every + * {@code next()} (mimicking a columnar reader), used to prove readBatch copies each record. + */ + private static class ReusedObjectSplitReaderFunction extends AbstractSplitReaderFunction { + private final int count; + + ReusedObjectSplitReaderFunction(Configuration conf, InternalSchemaManager ism, int count) { + super(conf, Collections.emptyList(), ism, false); + this.count = count; + } + + @Override + protected ClosableIterator createRecordIterator(HoodieSourceSplit split) { + return new ClosableIterator() { + private final GenericRowData reused = new GenericRowData(1); + private int i = 0; + + @Override + public boolean hasNext() { + return i < count; + } + + @Override + public RowData next() { + reused.setField(0, i); + reused.setRowKind(i % 2 == 0 ? RowKind.DELETE : RowKind.INSERT); + i++; + return reused; // same object every call + } + + @Override + public void close() { + } + }; + } + + @Override + protected RowType producedRowType() { + return RowType.of(new IntType()); + } + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieCdcSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieCdcSplitReaderFunction.java index b0d0579da0846..d5dc3eca57707 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieCdcSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieCdcSplitReaderFunction.java @@ -156,11 +156,11 @@ public void testReadWithNonCdcSplitDelegatesToFallback() { 1, "base.parquet", Option.empty(), tempDir.getAbsolutePath(), "", "read_optimized", "20230101000000000", "file-1", Option.empty()); - Exception ex = assertThrows(Exception.class, () -> function.read(nonCdcSplit)); + Exception ex = assertThrows(Exception.class, () -> function.open(nonCdcSplit)); assertNotNull(ex); // Must not be IllegalArgumentException (which the old type-guard wrongly threw) if (ex instanceof IllegalArgumentException) { - throw new AssertionError("read() should not throw IllegalArgumentException for non-CDC split; " + throw new AssertionError("open() should not throw IllegalArgumentException for non-CDC split; " + "it should fall through to the fallback reader", ex); } } @@ -223,7 +223,7 @@ public void testConstructorWithLimitZeroIsAccepted() { // ------------------------------------------------------------------------- @Test - public void testReadAcceptsCdcSourceSplitType() { + public void testReadAcceptsCdcSourceSplitType() throws Exception { // Verify that HoodieCdcSourceSplit is accepted (cast doesn't throw). // Actual I/O would require a real Hoodie table, so we only check the // type-guard passes by catching the downstream I/O error rather than @@ -237,7 +237,8 @@ public void testReadAcceptsCdcSourceSplitType() { 1, tempDir.getAbsolutePath(), 128 * 1024 * 1024L, "file-cdc", EMPTY_PARTITION_PATH, changes, "read_optimized", "20230101000000000"); - // Should not throw exception - function.read(cdcSplit); + // Opening the split creates the CDC iterator lazily (no I/O yet); it must not throw. + function.open(cdcSplit); + function.close(); } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieSplitReaderFunction.java index 2d6522d7941e9..59cde8d85afad 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/function/TestHoodieSplitReaderFunction.java @@ -23,9 +23,15 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.source.ExpressionPredicates; +import org.apache.hudi.source.split.HoodieSourceSplit; +import org.apache.hudi.util.StreamerUtil; import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.data.RowData; import org.apache.flink.table.expressions.FieldReferenceExpression; import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.types.logical.VarCharType; @@ -34,14 +40,23 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -283,7 +298,8 @@ public void testConfigurationIsStored() { @Test public void testReadMethodSignature() { - // Verify that the read method returns CloseableIterator + // Verify the reader function constructs via its public signature (the read path is driven + // through open/readBatch on the split-fetcher thread). HoodieSplitReaderFunction function = new HoodieSplitReaderFunction( conf, @@ -443,4 +459,78 @@ public void testDefaultConstructor() { assertNotNull(function); } + + // ------------------------------------------------------------------------- + // createRecordIterator — file-group reader cleanup when iterator init fails + // ------------------------------------------------------------------------- + + @Test + public void testCreateRecordIteratorClosesReaderWhenInitFails() throws Exception { + // getClosableIterator() runs initRecordIterators(), which can open the reader's I/O resources and + // then throw. The reader is only a local in createRecordIterator, so the failure path must close + // it (nothing else would), preserving the original failure as the cause. + HoodieFileGroupReader reader = mockReader(); + IOException initFailure = new IOException("init failed"); + when(reader.getClosableIterator()).thenThrow(initFailure); + + HoodieSplitReaderFunction function = readerFunctionReturning(reader); + HoodieSourceSplit split = createSplit(); + + try (MockedStatic mockedStreamerUtil = mockStatic(StreamerUtil.class)) { + mockedStreamerUtil.when(() -> StreamerUtil.metaClientForReader(any(), any())) + .thenReturn(mockMetaClient); + + HoodieIOException thrown = + assertThrows(HoodieIOException.class, () -> function.createRecordIterator(split)); + assertSame(initFailure, thrown.getCause(), "the original init failure must be the cause"); + } + verify(reader, times(1)).close(); + } + + @Test + public void testCreateRecordIteratorSuppressesCloseError() throws Exception { + // A close() failure on the cleanup path must not mask the init failure: it is attached as a + // suppressed exception on the original. + HoodieFileGroupReader reader = mockReader(); + IOException initFailure = new IOException("init failed"); + IOException closeFailure = new IOException("close failed"); + when(reader.getClosableIterator()).thenThrow(initFailure); + doThrow(closeFailure).when(reader).close(); + + HoodieSplitReaderFunction function = readerFunctionReturning(reader); + HoodieSourceSplit split = createSplit(); + + try (MockedStatic mockedStreamerUtil = mockStatic(StreamerUtil.class)) { + mockedStreamerUtil.when(() -> StreamerUtil.metaClientForReader(any(), any())) + .thenReturn(mockMetaClient); + + HoodieIOException thrown = + assertThrows(HoodieIOException.class, () -> function.createRecordIterator(split)); + assertSame(initFailure, thrown.getCause()); + assertEquals(1, initFailure.getSuppressed().length, "close failure must be suppressed, not lost"); + assertSame(closeFailure, initFailure.getSuppressed()[0]); + } + } + + @SuppressWarnings("unchecked") + private static HoodieFileGroupReader mockReader() { + return mock(HoodieFileGroupReader.class); + } + + private HoodieSplitReaderFunction readerFunctionReturning(HoodieFileGroupReader reader) { + return new HoodieSplitReaderFunction( + conf, tableSchema, requiredSchema, mockInternalSchemaManager, + "AVRO_PAYLOAD", Collections.emptyList(), false) { + @Override + protected HoodieFileGroupReader createFileGroupReader(HoodieSourceSplit split, HoodieTableMetaClient metaClient) { + return reader; + } + }; + } + + private static HoodieSourceSplit createSplit() { + return new HoodieSourceSplit( + 1, "base", Option.of(Collections.emptyList()), "/tbl", "/part", + "read_optimized", "19700101000000000", "file1", Option.empty()); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index c5e38bde48838..b48afb7d96b81 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -100,6 +100,7 @@ import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -3990,10 +3991,11 @@ private List execSelectSqlWithExpectedNum(TableEnvironment tEnv, String sel * the collected rows. * *

    The streaming job is terminated by a forced {@link CollectSinkTableFactory.SuccessException} once - * {@code expectedNum} rows are collected. A benign teardown race (see {@link #isAcceptableTerminalFailure}) - * can instead end the job before all rows are emitted, leaving a short result. Re-reading the already - * committed table is idempotent, so retry up to {@link #MAX_STREAM_READ_ATTEMPTS} times when the result - * is short; this keeps the race from surfacing as a confusing row-count assertion failure. + * {@code expectedNum} rows are collected. On a slow CI shard the {@code await} window can elapse before + * the sink reaches {@code expectedNum} (a bare timeout - see {@link #isAwaitTimeout}), leaving a short + * result. Re-reading the already committed table is idempotent, so retry up to + * {@link #MAX_STREAM_READ_ATTEMPTS} times when the result is short; this keeps a slow shard from + * surfacing as a confusing row-count (or "Unexpected job failure") assertion failure. */ private List submitAndFetchWithRetry(TableEnvironment tEnv, String select, String sinkDDL, int expectedNum) { List rows = Collections.emptyList(); @@ -4031,46 +4033,35 @@ private List execSelectSql(TableEnvironment tEnv, String select, long timeo private List fetchResultWithExpectedNum(TableEnvironment tEnv, TableResult tableResult) { try { // wait the continuous streaming query to be terminated by forced exception with expected row number - // and max waiting timeout is 30s - tableResult.await(30, TimeUnit.SECONDS); + // and max waiting timeout is 60s (kept generous so a slow CI shard does not time out before the + // sink collects its rows; a bare timeout is still handled as a retryable short read below) + tableResult.await(60, TimeUnit.SECONDS); } catch (Throwable e) { - // Acceptable terminal causes: - // 1. SuccessException: the sink reached its expected row count and intentionally - // threw to terminate the streaming job. This is the happy path. - // 2. IOException("Stream is closed!") wrapped as HoodieIOException: a benign - // error-attribution race between the source-side cascading-shutdown path and - // the sink-side SuccessException terminator. When the sink throws - // SuccessException to end the job, the chained source's SplitFetcher can close - // the underlying Hadoop FSDataInputStream while the mailbox is still draining - // a BatchRecords queued earlier; the next row-group read on the now-closed - // stream surfaces an IOException("Stream is closed!"). With - // restart-strategy.fixed-delay.attempts=0 (set in beforeEach to keep tests - // deterministic) that IOException becomes the job's reported failure cause - // instead of the sink's SuccessException, even though the sink has already - // collected the expected rows by then - i.e. the functional outcome is - // unchanged, only the error-attribution differs. Production paths correctly - // fail the job on stream-closed-mid-read (the right behavior for real I/O - // failures), so this tolerance is scoped to the SuccessException-based test - // pattern below and is NOT mirrored in production code. - // 3. NullPointerException from ParquetColumnarRowSplitReader#readNextRowGroup: the - // same benign teardown race as (2), observed with different timing. When the - // SplitFetcher's close() fully completes first, ParquetColumnarRowSplitReader#close - // nulls out its `reader` field, so the in-flight row-group read on the task thread - // surfaces as a NullPointerException (reader.readNextRowGroup() on a null reader) - // instead of an IOException("Stream is closed!"). Same functional outcome - the - // sink has already collected the expected rows - only the error symptom differs. - // Tolerated narrowly (an NPE originating from that exact frame) for the same - // reason as (2), and likewise NOT mirrored in production code. - if (!isAcceptableTerminalFailure(e)) { - throw new AssertionError("Unexpected job failure", e); - } - // The races (2)/(3) usually fire after the sink has collected its expected rows, but can also fire - // before - ending the read with a short result. Log the tolerated cause so an incomplete read is - // diagnosable; submitAndFetchWithRetry re-reads when the collected count is below the expectation. - if (!isSuccessException(e)) { - LOG.warn("Streaming read terminated by a tolerated teardown race ({}); collected {} rows so far.", - describeTerminalCause(e), + // The only acceptable terminal cause is the sink reaching its expected row count and throwing + // SuccessException to terminate the streaming job (the happy path). The Source V2 read path now + // reads and closes each split's I/O on a single (split-fetcher) thread, so the former teardown + // races (a closed Parquet stream / a closed CDC iterator surfacing on the task thread) can no + // longer happen; any other terminal failure is a real error and fails the test. + // + // A bare await-window TimeoutException is not a terminal failure at all - the sink simply had not + // reached expectedNum yet (typically CI-load slowness), so the job is still running. Cancel it and + // let submitAndFetchWithRetry re-submit a fresh job, rather than treating a slow shard as a hard + // failure. + if (isAwaitTimeout(e)) { + // Cancel the still-running job (best-effort, bounded) so it cannot keep writing to the shared + // CollectSinkTableFactory.RESULT after the retry's re-submit clears it. + tableResult.getJobClient().ifPresent(jobClient -> { + try { + jobClient.cancel().get(30, TimeUnit.SECONDS); + } catch (Exception ignored) { + // best-effort cancel; the subsequent re-submit clears RESULT and starts a fresh job + } + }); + LOG.warn("Streaming read did not reach the expected row count within the await window; " + + "cancelled the job and will retry. Collected {} rows so far.", CollectSinkTableFactory.RESULT.values().stream().mapToInt(List::size).sum()); + } else if (!isSuccessException(e)) { + throw new AssertionError("Unexpected job failure", e); } } tEnv.executeSql("DROP TABLE IF EXISTS sink"); @@ -4080,56 +4071,20 @@ private List fetchResultWithExpectedNum(TableEnvironment tEnv, TableResult } /** - * Whether {@code e} (or any of its causes) is one of the terminal failures that - * {@link #fetchResultWithExpectedNum} is allowed to swallow. See the comment at the call - * site for the rationale. - */ - private static boolean isAcceptableTerminalFailure(Throwable e) { - Throwable cur = e; - while (cur != null) { - if (cur instanceof CollectSinkTableFactory.SuccessException) { - return true; - } - String msg = cur.getMessage(); - if (msg != null && msg.contains("Stream is closed")) { - return true; - } - // The NPE twin of the "Stream is closed!" teardown race (cause #3 at the call site): - // a NullPointerException whose own stack trace originates from - // ParquetColumnarRowSplitReader#readNextRowGroup, i.e. reader.readNextRowGroup() ran on a - // null `reader` that ParquetColumnarRowSplitReader#close had just nulled out. Scoped to - // that exact frame so genuine NPEs - and the legitimate IOException("expecting more - // rows...") thrown from the same method - still fail the test. - if (isNullPointerException(cur) && containsReadNextRowGroupFrame(cur)) { - return true; - } - cur = cur.getCause(); - } - return false; - } - - /** - * True for a real {@link NullPointerException} as well as one wrapped in Flink's - * {@code SerializedThrowable} when the failure is propagated back from the cluster (its - * {@code toString()} preserves the original {@code java.lang.NullPointerException} prefix). + * Whether {@code e} is a bare {@link TimeoutException} thrown directly by + * {@link org.apache.flink.table.api.TableResult#await(long, TimeUnit)} - i.e. the await window elapsed + * before the sink reached its expected row count and threw + * {@link CollectSinkTableFactory.SuccessException}, leaving the job still running (never terminated), + * so the caller cancels it and retries the read rather than swallowing it. + * + *

    Only the top-level exception is inspected, never the cause chain: {@code await} throws its own + * timeout bare, whereas a genuine job failure arrives wrapped in an {@link ExecutionException} that + * may itself embed a {@link TimeoutException} (checkpoint expiry, RPC timeout). Walking the chain + * would misclassify such a real failure as a slow shard - cancel it, retry, and finally report a + * row-count mismatch with the true cause discarded. */ - private static boolean isNullPointerException(Throwable t) { - return t instanceof NullPointerException - || t.toString().startsWith(NullPointerException.class.getName()); - } - - /** - * Whether {@code t}'s stack trace (preserved even through {@code SerializedThrowable}) - * contains a {@code ParquetColumnarRowSplitReader#readNextRowGroup} frame. - */ - private static boolean containsReadNextRowGroupFrame(Throwable t) { - for (StackTraceElement frame : t.getStackTrace()) { - if (frame.getClassName().endsWith("ParquetColumnarRowSplitReader") - && "readNextRowGroup".equals(frame.getMethodName())) { - return true; - } - } - return false; + private static boolean isAwaitTimeout(Throwable e) { + return e instanceof TimeoutException; } /** @@ -4144,15 +4099,4 @@ private static boolean isSuccessException(Throwable e) { } return false; } - - /** - * Short description of {@code e}'s root cause, for logging which tolerated terminal failure fired. - */ - private static String describeTerminalCause(Throwable e) { - Throwable root = e; - while (root.getCause() != null) { - root = root.getCause(); - } - return root.getClass().getSimpleName() + ": " + root.getMessage(); - } } From 814c294fd3e83651721932782dee3b185c1d6f20 Mon Sep 17 00:00:00 2001 From: ashokkumar-allu <65997235+ashokkumar-allu@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:05:55 +0800 Subject: [PATCH 164/255] docs: claim RFC-108 Multi-dataset incremental reads in Hudi Streamer (#19308) Add the RFC-108 reservation to rfc/README.md. Co-authored-by: gallu (cherry picked from commit 79908efc59c4b4231cf1405252ed58ea89bcca88) --- rfc/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rfc/README.md b/rfc/README.md index 512ee1bc9b012..6e059e2ad0be8 100644 --- a/rfc/README.md +++ b/rfc/README.md @@ -142,3 +142,6 @@ The list of all RFCs can be found here. | 104 | [Unify schema evolution on schema-on-read](./rfc-104/rfc-104.md) | :eyes: `UNDER REVIEW` | | 105 | [Trino Hudi Connector — Shim/Bundle Refactor](./rfc-105/rfc-105.md) | :eyes: `UNDER REVIEW` | | 106 | [Record Level and Secondary Index Support for Flink Writers](./rfc-106/rfc-106.md) | :white_check_mark: `COMPLETED` | +| 107 | Dynamic Partitioned Cache for Flink upsert | :hammer_and_wrench: `IN PROGRESS` | +| 108 | [Multi-dataset incremental reads in Hudi Streamer](./rfc-108/rfc-108.md) | :eyes: `UNDER REVIEW` | +| 109 | Hudi Native Vector Index | :eyes: `UNDER REVIEW` | From f32a5fea1b465e92c73348f5a8d3d0613e01a0ca Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Fri, 17 Jul 2026 17:16:01 +0530 Subject: [PATCH 165/255] fix(test): stabilize flaky testReattemptOfFailedClusteringCommit (#19120) TestJavaHoodieBackedMetadata.testReattemptOfFailedClusteringCommit was intermittently failing in CI. The post-clustering insert used hardcoded commit time "0000003", which sorts lexicographically before the real-timestamp clusteringCommitTime. Writing an out-of-order commit after clustering corrupts the metadata table's file listing state, causing validateMetadata to fail on affected partitions. Fix: use WriteClientTestUtils.createNewInstantTime() for monotonically increasing commit times. (cherry picked from commit 5f7f7660a08bc8f11fd1904fbd9abccf983eb50b) --- .../apache/hudi/client/TestJavaHoodieBackedMetadata.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index 52b5563ac93ef..57086019a8d1b 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -1867,7 +1867,7 @@ public void testReattemptOfFailedClusteringCommit() throws Exception { HoodieJavaWriteClient client = getHoodieWriteClient(config); // Write 1 (Bulk insert) - String newCommitTime = "0000001"; + String newCommitTime = WriteClientTestUtils.createNewInstantTime(); List records = dataGen.generateInserts(newCommitTime, 20); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); List writeStatuses = client.insert(records, newCommitTime); @@ -1876,7 +1876,7 @@ public void testReattemptOfFailedClusteringCommit() throws Exception { validateMetadata(client); // Write 2 (inserts) - newCommitTime = "0000002"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); @@ -1909,7 +1909,7 @@ public void testReattemptOfFailedClusteringCommit() throws Exception { replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file)))); // trigger new write to mimic other writes succeeding before re-attempt. - newCommitTime = "0000003"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); From ff098567e39e3cd3422f5b394ba185e6cc3c2359 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Sun, 19 Jul 2026 10:09:30 -0700 Subject: [PATCH 166/255] test(common): add log-format reader and scanner coverage (#19223) * test(common): add log-format reader and scanner coverage * test(common): fix scanner close/lifecycle and restrict BRAF tests to read path - Drop non-existent HoodieUnMergedLogRecordScanner.close() calls that caused Azure CI to fail compilation of hudi-hadoop-common tests. - Refactor TestBufferedRandomAccessFile to seed fixtures via Files.write and only exercise the read path. Production only opens the class in read mode ("r") (BitCaskDiskMap / LazyFileIterable). Exercising its write path across buffer boundaries hangs in an infinite loop (dormant writer-side bug in expandBufferToCapacityIfNeeded not addressed here). --------- Co-authored-by: sivabalan Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 4394513a8cb7a96d03e301255d6da39310d86b47) --- .../util/TestBufferedRandomAccessFile.java | 132 ++++++++++++ .../functional/TestHoodieLogFormat.java | 202 ++++++++++++++++++ 2 files changed, 334 insertions(+) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestBufferedRandomAccessFile.java diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestBufferedRandomAccessFile.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBufferedRandomAccessFile.java new file mode 100644 index 0000000000000..5dc1fab9c86d0 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBufferedRandomAccessFile.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Unit tests for {@link BufferedRandomAccessFile}, the buffered wrapper used to + * back the log-file read path (via BitCaskDiskMap / LazyFileIterable). The buffer + * capacity is clamped to a 64K minimum, so the payloads here are deliberately + * larger than that to force reads and seeks that cross buffer boundaries. + * + *

    All fixtures are seeded via {@link Files#write} rather than the class's own + * write path since Hudi opens this class only in read mode ("r") in production. + */ +public class TestBufferedRandomAccessFile { + + // Larger than the internal 64K minimum buffer so that access crosses buffer boundaries. + private static final int PAYLOAD_SIZE = (1 << 16) * 3 + 517; + + private static byte[] deterministicBytes(int size) { + byte[] bytes = new byte[size]; + new Random(42).nextBytes(bytes); + return bytes; + } + + private static File seedFile(Path dir, String name, byte[] contents) throws Exception { + File file = new File(dir.toFile(), name); + Files.write(file.toPath(), contents); + return file; + } + + @Test + public void testReadFullyAcrossBufferBoundaries(@TempDir Path tempDir) throws Exception { + byte[] expected = deterministicBytes(PAYLOAD_SIZE); + File file = seedFile(tempDir, "read.bin", expected); + + byte[] readBack = new byte[PAYLOAD_SIZE]; + try (BufferedRandomAccessFile raf = new BufferedRandomAccessFile(file, "r")) { + // readFully loops over read(...) so partial reads at buffer boundaries are handled. + raf.readFully(readBack); + assertEquals(PAYLOAD_SIZE, raf.getFilePointer(), "Read should consume the whole file"); + assertEquals(-1, raf.read(), "Reading past EOF should return -1"); + } + assertArrayEquals(expected, readBack, "Bytes read back should match bytes written"); + } + + @Test + public void testSeekRandomAccessCrossingBoundaries(@TempDir Path tempDir) throws Exception { + byte[] expected = deterministicBytes(PAYLOAD_SIZE); + File file = seedFile(tempDir, "seek.bin", expected); + + try (BufferedRandomAccessFile raf = new BufferedRandomAccessFile(file, "r")) { + // Probe positions that fall in the first, second, third and last logical buffer blocks, + // out of order, so both forward and backward seeks are exercised. + int[] probes = {0, (1 << 16) + 7, (1 << 16) * 2 + 100, 5, PAYLOAD_SIZE - 1}; + for (int pos : probes) { + raf.seek(pos); + assertEquals(pos, raf.getFilePointer(), "getFilePointer should reflect the seek target"); + int actual = raf.read(); + assertEquals(expected[pos] & 0xFF, actual, "Byte at position " + pos + " should match"); + } + + // A ranged read that starts mid-buffer and spans a boundary must return the correct slice. + int start = (1 << 16) - 10; + int len = 40; + raf.seek(start); + byte[] slice = new byte[len]; + raf.readFully(slice); + byte[] expectedSlice = new byte[len]; + System.arraycopy(expected, start, expectedSlice, 0, len); + assertArrayEquals(expectedSlice, slice, "Ranged read across a buffer boundary should match"); + } + } + + @Test + public void testReadIntoOffsetAndLength(@TempDir Path tempDir) throws Exception { + byte[] expected = deterministicBytes(1024); + File file = seedFile(tempDir, "offset.bin", expected); + + try (BufferedRandomAccessFile raf = new BufferedRandomAccessFile(file, "r")) { + byte[] target = new byte[1024 + 8]; + // Leave a 4-byte prefix untouched and read the payload into the middle of the array. + raf.readFully(target, 4, 1024); + byte[] payload = new byte[1024]; + System.arraycopy(target, 4, payload, 0, 1024); + assertArrayEquals(expected, payload, "readFully into an offset should place bytes at that offset"); + assertEquals(0, target[0], "Bytes before the offset must remain untouched"); + assertEquals(0, target[3], "Byte just before the offset must remain untouched"); + assertEquals(0, target[1028], "Byte just after the range must remain untouched"); + assertEquals(0, target[1031], "Last byte after the range must remain untouched"); + } + } + + @Test + public void testLengthAndEofAfterFullRead(@TempDir Path tempDir) throws Exception { + byte[] expected = deterministicBytes(PAYLOAD_SIZE); + File file = seedFile(tempDir, "length.bin", expected); + + try (BufferedRandomAccessFile raf = new BufferedRandomAccessFile(file, "r")) { + assertEquals(PAYLOAD_SIZE, raf.length(), "length should report the on-disk size"); + raf.seek(PAYLOAD_SIZE); + assertEquals(-1, raf.read(), "Reading at EOF should return -1"); + assertEquals(PAYLOAD_SIZE, raf.getFilePointer(), "File pointer should stay at EOF"); + } + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index c24f928b53f1e..2a74e4b73856b 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -47,6 +47,8 @@ import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; +import org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner; +import org.apache.hudi.common.table.log.InstantRange; import org.apache.hudi.common.table.log.TestLogReaderUtils; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; @@ -2955,6 +2957,206 @@ public void testGetRecordPositions(boolean recordWithPositions, TestLogReaderUtils.assertPositionEquals(expectedPositions, dataBlock.getRecordPositions()); } + @Test + public void testUnMergedLogRecordScannerCallbacksWithDeletes() + throws IOException, URISyntaxException, InterruptedException { + HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); + + SchemaTestUtil testUtil = new SchemaTestUtil(); + List records = testUtil.generateHoodieTestRecords(0, 100); + List copyOfRecords = records.stream() + .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())) + .collect(Collectors.toList()); + Map header = new HashMap<>(); + header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); + header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); + writer.appendBlock(getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header)); + + // Delete the first 20 keys via a delete block. + List allKeys = copyOfRecords.stream() + .map(r -> ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()) + .collect(Collectors.toList()); + List deletedKeys = new ArrayList<>(allKeys.subList(0, 20)); + List> deleteRecordList = copyOfRecords.subList(0, 20).stream() + .map(r -> Pair.of(DeleteRecord.create( + ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(), + ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString()), + -1L)) + .collect(Collectors.toList()); + header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); + writer.appendBlock(new HoodieDeleteBlock(deleteRecordList, header)); + writer.close(); + + FileCreateUtilsLegacy.createDeltaCommit(basePath, "100", storage); + + List insertedKeys = new ArrayList<>(); + List deletedKeysSeen = new ArrayList<>(); + HoodieUnMergedLogRecordScanner scanner = HoodieUnMergedLogRecordScanner.newBuilder() + .withStorage(storage) + .withBasePath(basePath) + .withLogFilePaths(Collections.singletonList(writer.getLogFile().getPath().toString())) + .withReaderSchema(schema) + .withLatestInstantTime("100") + .withReverseReader(false) + .withBufferSize(BUFFER_SIZE) + .withLogRecordScannerCallback(record -> insertedKeys.add(record.getRecordKey())) + .withRecordDeletionCallback(key -> deletedKeysSeen.add(key.getRecordKey())) + .build(); + scanner.scan(); + + // The un-merged scanner streams every data record and every deleted key through the callbacks + // without merging them, so all 100 inserts and all 20 deletes should be observed exactly once. + assertEquals(100, insertedKeys.size(), "Callback should see every appended data record"); + assertEquals(new HashSet<>(allKeys), new HashSet<>(insertedKeys), + "Callback keys should match every appended record key"); + assertEquals(20, deletedKeysSeen.size(), "Deletion callback should see every deleted key"); + assertEquals(new HashSet<>(deletedKeys), new HashSet<>(deletedKeysSeen), + "Deletion callback keys should match the delete block keys"); + } + + @Test + public void testUnMergedLogRecordScannerInstantRangeFiltering() + throws IOException, URISyntaxException, InterruptedException { + HoodieSchema schema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema()); + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); + + SchemaTestUtil testUtil = new SchemaTestUtil(); + // First block belongs to instant 100, second block to instant 101. + List recordsAt100 = testUtil.generateHoodieTestRecords(0, 60); + List keysAt100 = recordsAt100.stream() + .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())) + .map(r -> ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()) + .collect(Collectors.toList()); + Map header = new HashMap<>(); + header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); + header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); + writer.appendBlock(getDataBlock(DEFAULT_DATA_BLOCK_TYPE, recordsAt100, header)); + + List recordsAt101 = testUtil.generateHoodieTestRecords(0, 40); + header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101"); + writer.appendBlock(getDataBlock(DEFAULT_DATA_BLOCK_TYPE, recordsAt101, header)); + writer.close(); + + FileCreateUtilsLegacy.createDeltaCommit(basePath, "100", storage); + FileCreateUtilsLegacy.createDeltaCommit(basePath, "101", storage); + + // A closed range of [100, 100] should keep only the first block and drop the instant 101 block. + InstantRange instantRange = InstantRange.builder() + .startInstant("100") + .endInstant("100") + .rangeType(InstantRange.RangeType.CLOSED_CLOSED) + .build(); + + List seenKeys = new ArrayList<>(); + HoodieUnMergedLogRecordScanner scanner = HoodieUnMergedLogRecordScanner.newBuilder() + .withStorage(storage) + .withBasePath(basePath) + .withLogFilePaths(Collections.singletonList(writer.getLogFile().getPath().toString())) + .withReaderSchema(schema) + .withLatestInstantTime("101") + .withReverseReader(false) + .withBufferSize(BUFFER_SIZE) + .withInstantRange(Option.of(instantRange)) + .withLogRecordScannerCallback(record -> seenKeys.add(record.getRecordKey())) + .build(); + scanner.scan(); + + assertEquals(60, seenKeys.size(), "Only the instant 100 block should pass the instant range"); + assertEquals(new HashSet<>(keysAt100), new HashSet<>(seenKeys), + "Records outside the instant range should be filtered out"); + } + + @Test + public void testLogFileReaderReadsPastCorruptBlock() + throws IOException, URISyntaxException, InterruptedException { + HoodieSchema schema = getSimpleSchema(); + HoodieLogFormat.Writer writer = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); + List records1 = SchemaTestUtil.generateTestRecords(0, 100); + List copyOfRecords1 = records1.stream() + .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())) + .collect(Collectors.toList()); + Map header = new HashMap<>(); + header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100"); + header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); + writer.appendBlock(getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records1, header)); + writer.close(); + + // Append a block whose declared length does not match its content, mimicking a partial write. + FSDataOutputStream outputStream = (FSDataOutputStream) storage.append(writer.getLogFile().getPath()); + outputStream.write(HoodieLogFormat.MAGIC); + outputStream.writeLong(474); + outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal()); + outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION); + outputStream.writeLong(400); + outputStream.write(getUTF8Bytes("truncated-block-content")); + outputStream.flush(); + outputStream.close(); + + // Append a valid trailing block so the reader has a real block to recover to after the corruption. + HoodieLogFormat.Writer appendWriter = + HoodieLogFormatWriter.builder().withParentPath(partitionPath) + .withFileExtension(HoodieLogFile.DELTA_EXTENSION) + .withLogFileId("test-fileid1") + .withInstantTime("100") + .withStorage(storage) + .build(); + ((HoodieLogFormatWriter) appendWriter).withOutputStream( + (FSDataOutputStream) storage.append(writer.getLogFile().getPath())); + List records2 = SchemaTestUtil.generateTestRecords(0, 10); + List copyOfRecords2 = records2.stream() + .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())) + .collect(Collectors.toList()); + appendWriter.appendBlock(getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header)); + appendWriter.close(); + + HoodieLogFile logFile = new HoodieLogFile(appendWriter.getLogFile().getPath(), + storage.getPathInfo(appendWriter.getLogFile().getPath()).getLength()); + try (HoodieLogFileReader reader = + new HoodieLogFileReader(storage, logFile, SchemaTestUtil.getSimpleSchema(), BUFFER_SIZE)) { + // First a valid data block. + assertTrue(reader.hasNext(), "First data block should be available"); + HoodieLogBlock firstBlock = reader.next(); + assertEquals(HoodieLogBlockType.AVRO_DATA_BLOCK, firstBlock.getBlockType(), "First block should be a data block"); + List firstRead = getRecords((HoodieDataBlock) firstBlock); + assertEquals(convertAvroToSerializableIndexedRecords(copyOfRecords1), firstRead, + "First block contents should match the written records"); + + // The reader seeks past the bad magic/length and surfaces a corrupt block. + assertTrue(reader.hasNext(), "Corrupt block should be surfaced"); + HoodieLogBlock corruptBlock = reader.next(); + assertEquals(HoodieLogBlockType.CORRUPT_BLOCK, corruptBlock.getBlockType(), "Second block should be a corrupt block"); + + // The valid trailing block should still be readable after recovery. + assertTrue(reader.hasNext(), "Trailing data block should be available after the corrupt block"); + HoodieLogBlock lastBlock = reader.next(); + assertEquals(HoodieLogBlockType.AVRO_DATA_BLOCK, lastBlock.getBlockType(), "Third block should be a data block"); + List lastRead = getRecords((HoodieDataBlock) lastBlock); + assertEquals(convertAvroToSerializableIndexedRecords(copyOfRecords2), lastRead, + "Trailing block contents should match the written records"); + + assertFalse(reader.hasNext(), "There should be no more blocks"); + } + } + private static Stream testArguments() { // Arg1: ExternalSpillableMap Type, Arg2: isDiskMapCompressionEnabled return Stream.of( From ab4a6d6f6c1a241f9aaacdf1c62c7bad8d397080 Mon Sep 17 00:00:00 2001 From: Nada Date: Sun, 19 Jul 2026 14:11:08 -0400 Subject: [PATCH 167/255] fix(metadata-table): add config to skip zero-size data files on MDT initialization (#18611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in config `hoodie.metadata.skip.zero.size.files.on.initialize` (default false, advanced) that skips zero-size data files while listing the data table during MDT initialization. Prevents init failures on tables carrying leftover zero-byte files, and emits `skipped_zero_size_files_on_initialize` with the skipped count. Skip is scoped to the initialize path only — the restore-sync caller of `listAllPartitionsFromFilesystem` passes `skipZeroSizeFiles=false` unconditionally, so files already tracked in MDT are not spuriously deleted. Existing `DirectoryInfo` constructors delegate with `false` to preserve current behavior. (cherry picked from commit 5178a3bedae848e1322842b1db5f1b03ee439b83) --- .../HoodieBackedTableMetadataWriter.java | 16 ++++++++--- .../TestHoodieMetadataBootstrap.java | 28 +++++++++++++++++++ .../common/config/HoodieMetadataConfig.java | 17 +++++++++++ .../metadata/HoodieTableMetadataUtil.java | 15 ++++++++-- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 2185418fe8eba..afb5ec210db0f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -431,7 +431,8 @@ private boolean initializeFromFilesystem(String dataTableInstantTime, List listAllPartitionsFromFilesystem(String initializationTime, Set pendingDataInstants) { + private List listAllPartitionsFromFilesystem(String initializationTime, Set pendingDataInstants, boolean skipZeroSizeFiles) { if (dataMetaClient.getActiveTimeline().countInstants() == 0) { return Collections.emptyList(); } @@ -1103,6 +1107,7 @@ private List listAllPartitionsFromFilesystem(String initializatio StorageConfiguration storageConf = dataMetaClient.getStorageConf(); final String dirFilterRegex = dataWriteConfig.getMetadataConfig().getDirectoryFilterRegex(); StoragePath storageBasePath = dataMetaClient.getBasePath(); + long totalZeroSizeFiles = 0; while (!pathsToList.isEmpty()) { // In each round we will list a section of directories @@ -1116,12 +1121,13 @@ private List listAllPartitionsFromFilesystem(String initializatio List processedDirectories = engineContext.map(pathsToProcess, path -> { HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf); String relativeDirPath = FSUtils.getRelativePartitionPath(storageBasePath, path); - return new DirectoryInfo(relativeDirPath, storage.listDirectEntries(path), initializationTime, pendingDataInstants); + return new DirectoryInfo(relativeDirPath, storage.listDirectEntries(path), initializationTime, pendingDataInstants, true, skipZeroSizeFiles); }, numDirsToList); // If the listing reveals a directory, add it to queue. If the listing reveals a hoodie partition, add it to // the results. for (DirectoryInfo dirInfo : processedDirectories) { + totalZeroSizeFiles += dirInfo.getZeroSizeFileCount(); if (!dirFilterRegex.isEmpty()) { final String relativePath = dirInfo.getRelativePath(); if (!relativePath.isEmpty() && relativePath.matches(dirFilterRegex)) { @@ -1140,6 +1146,8 @@ private List listAllPartitionsFromFilesystem(String initializatio } } + final long zeroSizeCount = totalZeroSizeFiles; + metrics.ifPresent(m -> m.incrementMetric("skipped_zero_size_files_on_initialize", zeroSizeCount)); return partitionsToBootstrap; } @@ -1772,7 +1780,7 @@ public void update(HoodieRestoreMetadata restoreMetadata, String instantTime) { // Restore requires the existing pipelines to be shutdown. So we can safely scan the dataset to find the current // list of files in the filesystem. - List dirInfoList = listAllPartitionsFromFilesystem(instantTime, Collections.emptySet()); + List dirInfoList = listAllPartitionsFromFilesystem(instantTime, Collections.emptySet(), false); Map dirInfoMap = dirInfoList.stream().collect(Collectors.toMap(DirectoryInfo::getRelativePath, Function.identity())); dirInfoList.clear(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java index 28b325889072a..742e14874377b 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java @@ -108,6 +108,34 @@ public void testMetadataBootstrapWithExtraFiles() throws Exception { validateMetadata(testTable); } + @Test + public void testMetadataSkipsZeroSizeFilesOnInitialize() throws Exception { + HoodieTableType tableType = COPY_ON_WRITE; + init(tableType, false); + doPreBootstrapWriteOperation(testTable, INSERT, "0000001"); + doPreBootstrapWriteOperation(testTable, "0000002"); + // Add a zero-size base file — bootstrap should skip it without failing. + String fileName = UUID.randomUUID().toString(); + Path zeroSizeFilePath = FileCreateUtilsLegacy.getBaseFilePath(basePath, "p1", "0000003", fileName); + FileCreateUtilsLegacy.createBaseFile(basePath, "p1", "0000003", fileName, 0); + + writeConfig = getWriteConfigBuilder(true, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withSkipZeroSizeFilesOnInitialize(true) + .build()) + .build(); + initWriteConfigAndMetatableWriter(writeConfig, true); + syncTableMetadata(writeConfig); + + // Delete the zero-size file before validation — it was skipped in MDT and must not + // exist on disk for the filesystem-vs-MDT consistency check to pass. + Files.delete(zeroSizeFilePath); + validateMetadata(testTable); + doWriteInsertAndUpsert(testTable); + validateMetadata(testTable); + } + @ParameterizedTest @EnumSource(HoodieTableType.class) public void testMetadataBootstrapInsertUpsertRollback(HoodieTableType tableType) throws Exception { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index 7ce1e447e65f5..f6757a5b71258 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -195,6 +195,14 @@ public final class HoodieMetadataConfig extends HoodieConfig { .sinceVersion("0.7.0") .withDocumentation("Directories matching this regex, will be filtered out when initializing metadata table from lake storage for the first time."); + public static final ConfigProperty SKIP_ZERO_SIZE_FILES_ON_INITIALIZE = ConfigProperty + .key(METADATA_PREFIX + ".skip.zero.size.files.on.initialize") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.2.0") + .withDocumentation("When enabled, zero-size data files encountered while listing the data table during " + + "metadata table initialization are skipped instead of being recorded in the metadata table."); + public static final ConfigProperty FILE_LISTING_PARALLELISM_VALUE = ConfigProperty .key("hoodie.file.listing.parallelism") .defaultValue(200) @@ -800,6 +808,10 @@ public String getDirectoryFilterRegex() { return getString(DIR_FILTER_REGEX); } + public boolean shouldSkipZeroSizeFilesOnInitialize() { + return getBoolean(SKIP_ZERO_SIZE_FILES_ON_INITIALIZE); + } + public boolean shouldIgnoreSpuriousDeletes() { return getBoolean(IGNORE_SPURIOUS_DELETES); } @@ -1173,6 +1185,11 @@ public Builder withDirectoryFilterRegex(String regex) { return this; } + public Builder withSkipZeroSizeFilesOnInitialize(boolean skipZeroSizeFiles) { + metadataConfig.setValue(SKIP_ZERO_SIZE_FILES_ON_INITIALIZE, String.valueOf(skipZeroSizeFiles)); + return this; + } + public Builder ignoreSpuriousDeletes(boolean validateMetadataPayloadConsistency) { metadataConfig.setValue(IGNORE_SPURIOUS_DELETES, String.valueOf(validateMetadataPayloadConsistency)); return this; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index cfdeb45b84ed6..f0a0fe3f98176 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -3126,9 +3126,10 @@ public static class DirectoryInfo implements Serializable { private final List subDirectories = new ArrayList<>(); // Is this a hoodie partition private boolean isHoodiePartition = false; + private int zeroSizeFileCount = 0; public DirectoryInfo(String relativePath, List pathInfos, String maxInstantTime, Set pendingDataInstants) { - this(relativePath, pathInfos, maxInstantTime, pendingDataInstants, true); + this(relativePath, pathInfos, maxInstantTime, pendingDataInstants, true, false); } /** @@ -3136,6 +3137,11 @@ public DirectoryInfo(String relativePath, List pathInfos, Strin */ public DirectoryInfo(String relativePath, List pathInfos, String maxInstantTime, Set pendingDataInstants, boolean validateHoodiePartitions) { + this(relativePath, pathInfos, maxInstantTime, pendingDataInstants, validateHoodiePartitions, false); + } + + public DirectoryInfo(String relativePath, List pathInfos, String maxInstantTime, Set pendingDataInstants, + boolean validateHoodiePartitions, boolean skipZeroSizeFiles) { this.relativePath = relativePath; // Pre-allocate with the maximum length possible @@ -3156,7 +3162,12 @@ public DirectoryInfo(String relativePath, List pathInfos, Strin String dataFileCommitTime = FSUtils.getCommitTime(pathInfo.getPath().getName()); // Limit the file listings to files which were created by successful commits before the maxInstant time. if (!pendingDataInstants.contains(dataFileCommitTime) && compareTimestamps(dataFileCommitTime, LESSER_THAN_OR_EQUALS, maxInstantTime)) { - filenameToSizeMap.put(pathInfo.getPath().getName(), pathInfo.getLength()); + if (pathInfo.getLength() > 0 || !skipZeroSizeFiles) { + filenameToSizeMap.put(pathInfo.getPath().getName(), pathInfo.getLength()); + } else { + log.warn("Skipping zero-size data file during MDT bootstrap: {}", pathInfo.getPath()); + zeroSizeFileCount++; + } } } } From 3760a3c8cb7a4288b84429ffe094f0972bdeadae Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 13:23:55 +0800 Subject: [PATCH 168/255] fix(build): use CURRENT_VERSION for the inline log-format header on release-1.2.1 TestHoodieLogFormat, added by 9e65e525e906 (#19223), writes an inline log block header using HoodieLogFormat.INLINE_LOG_FORMAT_VERSION. That constant does not exist on this branch. Master introduced it when the log format was bumped for native logs: there CURRENT_VERSION = 2 (native) and INLINE_LOG_FORMAT_VERSION = 1 (legacy inline). release-1.2.1 has no native log format, so it only declares CURRENT_VERSION = 1, and hudi-hadoop-common failed to test-compile with "cannot find symbol: variable INLINE_LOG_FORMAT_VERSION". Use CURRENT_VERSION, which is the inline format version on this branch and carries the same value (1), so the written header bytes are unchanged. --- .../org/apache/hudi/common/functional/TestHoodieLogFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index 2a74e4b73856b..81d5c71aadc50 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -3105,7 +3105,7 @@ public void testLogFileReaderReadsPastCorruptBlock() outputStream.write(HoodieLogFormat.MAGIC); outputStream.writeLong(474); outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal()); - outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION); + outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION); outputStream.writeLong(400); outputStream.write(getUTF8Bytes("truncated-block-content")); outputStream.flush(); From d8d369990f171513afb59ab7b804381606e4b68a Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Mon, 20 Jul 2026 15:40:38 +0530 Subject: [PATCH 169/255] fix(hive-sync): sync column and partition column comments to HMS (#19289) When hoodie.datasource.hive_sync.sync_comment is enabled, comments of partition columns were never synced to the Hive metastore: - In hms sync mode, HMSDDLExecutor.updateTableComments only updated the columns in the storage descriptor and ignored the table's partition keys, silently dropping partition column comments. - In jdbc/hiveql sync modes, the generated ALTER TABLE ... CHANGE COLUMN statement targeted the partition column, which Hive rejects with "Invalid column reference", failing the whole sync. - createTable never populated any comments, so even regular column comments only appeared from the second sync onwards. - The spark schema serialized into the spark.sql.sources.schema table property never carried the field docs, so spark DESCRIBE showed no comments even when they were present in HMS. Changes: - HMSDDLExecutor.createTable populates column and partition column comments from the storage schema field docs when sync_comment is on. - HMSDDLExecutor.updateTableComments also updates the partition keys, matching AWSGlueCatalogSyncClient.updateTableComments. - HiveSchemaUtil.generateCreateDDL (jdbc/hiveql modes) appends COMMENT clauses to regular and partition column definitions when sync_comment is on. - QueryBasedDDLExecutor.updateTableComments skips partition columns with a warning since HiveQL has no DDL to alter partition column comments, instead of failing the sync. - SparkSchemaUtils.convertToSparkSchemaJson can now include field docs as the comment of the field metadata; HiveSyncTool enables it for the spark.sql.sources.schema table property when sync_comment is on. Fixes #17359 (HUDI-8843), follow-up of #11922 which fixed the same problem for AWS Glue. (cherry picked from commit 59ed745c08fd37cd0a43bde4a4bd5650508e128d) --- .../org/apache/hudi/hive/HiveSyncTool.java | 3 +- .../hudi/hive/HoodieHiveSyncClient.java | 11 ++ .../org/apache/hudi/hive/ddl/DDLExecutor.java | 7 + .../apache/hudi/hive/ddl/HMSDDLExecutor.java | 34 +++- .../hudi/hive/ddl/QueryBasedDDLExecutor.java | 10 +- .../apache/hudi/hive/util/HiveSchemaUtil.java | 51 ++++- .../apache/hudi/hive/TestHiveSyncTool.java | 180 ++++++++++++++++++ .../hudi/hive/TestSparkSchemaUtils.java | 25 +++ .../hudi/hive/testutils/HiveTestUtil.java | 6 +- .../util/SparkDataSourceTableUtils.java | 13 +- .../sync/common/util/SparkSchemaUtils.java | 79 ++++++-- 11 files changed, 390 insertions(+), 29 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index fc07512c353b1..35bced6497e64 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -393,7 +393,8 @@ private Map getTableProperties(HoodieSchema schema) { Map tableProperties = ConfigUtils.toMap(config.getString(HIVE_TABLE_PROPERTIES)); if (config.getBoolean(HIVE_SYNC_AS_DATA_SOURCE_TABLE)) { Map sparkTableProperties = SparkDataSourceTableUtils.getSparkTableProperties(config.getSplitStrings(META_SYNC_PARTITION_FIELDS), - config.getStringOrDefault(META_SYNC_SPARK_VERSION), config.getIntOrDefault(HIVE_SYNC_SCHEMA_STRING_LENGTH_THRESHOLD), schema); + config.getStringOrDefault(META_SYNC_SPARK_VERSION), config.getIntOrDefault(HIVE_SYNC_SCHEMA_STRING_LENGTH_THRESHOLD), schema, + config.getBoolean(HIVE_SYNC_COMMENT)); tableProperties.putAll(sparkTableProperties); } return tableProperties; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java index 1bb34bf3d7840..9ad9070a465ef 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java @@ -697,6 +697,17 @@ public boolean updateTableComments(String tableName, List fromMetas } } }); + if (!ddlExecutor.supportsUpdatingPartitionColumnComments()) { + List skippedPartitionFields = config.getSplitStrings(META_SYNC_PARTITION_FIELDS).stream() + .map(partitionField -> partitionField.toLowerCase(Locale.ROOT)) + .filter(alterComments::containsKey) + .collect(Collectors.toList()); + if (!skippedPartitionFields.isEmpty()) { + log.debug("Cannot update comments of partition columns {} of {} in query based sync modes, use hms sync mode instead", + skippedPartitionFields, tableName); + skippedPartitionFields.forEach(alterComments::remove); + } + } if (alterComments.isEmpty()) { log.info("No comment difference of {} ", tableName); return false; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/DDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/DDLExecutor.java index d1e0e03aca3f3..a520bf6c1e945 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/DDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/DDLExecutor.java @@ -108,4 +108,11 @@ void createTable(String tableName, HoodieSchema storageSchema, String inputForma * @param newSchema Map key: field name, Map value: [field type, field comment] */ void updateTableComments(String tableName, Map> newSchema); + + /** + * @return whether this executor can update the comments of partition columns of an existing table. + */ + default boolean supportsUpdatingPartitionColumnComments() { + return true; + } } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java index 6b7bd3af00963..fa0d31a74cfae 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java @@ -52,12 +52,14 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_COMMENT; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS; @@ -110,6 +112,12 @@ public void createTable(String tableName, HoodieSchema storageSchema, String inp String partitionKeyType = HiveSchemaUtil.getPartitionKeyType(mapSchema, partitionKey); return new FieldSchema(partitionKey, partitionKeyType.toLowerCase(), ""); }).collect(Collectors.toList()); + + if (syncConfig.getBoolean(HIVE_SYNC_COMMENT)) { + Map fieldDocs = HiveSchemaUtil.getFieldDocs(storageSchema); + applyFieldDocs(fieldSchema, fieldDocs); + applyFieldDocs(partitionSchema, fieldDocs); + } Table newTb = new Table(); newTb.setDbName(databaseName); newTb.setTableName(tableName); @@ -266,13 +274,9 @@ public void updateTableComments(String tableName, Map fields, Map> alterSchema) { + for (FieldSchema fieldSchema : fields) { + if (alterSchema.containsKey(fieldSchema.getName())) { + String comment = alterSchema.get(fieldSchema.getName()).getRight(); + fieldSchema.setComment(comment); + } + } + } + + private static void applyFieldDocs(List fields, Map fieldDocs) { + for (FieldSchema fieldSchema : fields) { + String doc = fieldDocs.get(fieldSchema.getName().toLowerCase(Locale.ROOT)); + if (doc != null) { + fieldSchema.setComment(doc); + } + } + } + @Override public void close() { if (client != null) { diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java index a4e3ee3091543..78b8e684e49a4 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java @@ -142,8 +142,7 @@ public void updateTableComments(String tableName, Map constructAddPartitions(String tableName, List partitions) { List result = new ArrayList<>(); int batchSyncPartitionNum = config.getIntOrDefault(HIVE_BATCH_SYNC_PARTITION_NUM); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java index 50cf61f3ce218..065d3515614af 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -40,6 +41,7 @@ import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BUCKET_SYNC_SPEC; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_COMMENT; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; @@ -336,12 +338,23 @@ public static String generateSchemaString(HoodieSchema storageSchema, List colsToSkip, boolean supportTimestamp) throws IOException { + return generateSchemaString(storageSchema, colsToSkip, supportTimestamp, Collections.emptyMap()); + } + + public static String generateSchemaString(HoodieSchema storageSchema, List colsToSkip, boolean supportTimestamp, + Map fieldDocs) throws IOException { Map hiveSchema = convertSchemaToHiveSchema(storageSchema, supportTimestamp); StringBuilder columns = new StringBuilder(); for (Map.Entry hiveSchemaEntry : hiveSchema.entrySet()) { - if (!colsToSkip.contains(removeSurroundingTick(hiveSchemaEntry.getKey()))) { + String fieldName = removeSurroundingTick(hiveSchemaEntry.getKey()); + if (!colsToSkip.contains(fieldName)) { columns.append(hiveSchemaEntry.getKey()).append(" "); - columns.append(hiveSchemaEntry.getValue()).append(", "); + columns.append(hiveSchemaEntry.getValue()); + String doc = fieldDocs.get(fieldName.toLowerCase(Locale.ROOT)); + if (doc != null) { + columns.append(" COMMENT '").append(escapeSqlString(doc)).append("'"); + } + columns.append(", "); } } // Remove the last ", " @@ -349,17 +362,43 @@ public static String generateSchemaString(HoodieSchema storageSchema, List getFieldDocs(HoodieSchema schema) { + return schema.getFields().stream() + .filter(field -> field.doc().map(doc -> !doc.isEmpty()).orElse(false)) + .collect(Collectors.toMap(field -> field.name().toLowerCase(Locale.ROOT), field -> field.doc().get(), (existing, duplicate) -> existing)); + } + public static String generateCreateDDL(String tableName, HoodieSchema storageSchema, HiveSyncConfig config, String inputFormatClass, String outputFormatClass, String serdeClass, Map serdeProperties, Map tableProperties) throws IOException { Map hiveSchema = convertSchemaToHiveSchema(storageSchema, config.getBoolean(HIVE_SUPPORT_TIMESTAMP_TYPE)); - String columns = generateSchemaString(storageSchema, config.getSplitStrings(META_SYNC_PARTITION_FIELDS), config.getBoolean(HIVE_SUPPORT_TIMESTAMP_TYPE)); + Map fieldDocs = config.getBoolean(HIVE_SYNC_COMMENT) ? getFieldDocs(storageSchema) : Collections.emptyMap(); + String columns = generateSchemaString(storageSchema, config.getSplitStrings(META_SYNC_PARTITION_FIELDS), + config.getBoolean(HIVE_SUPPORT_TIMESTAMP_TYPE), fieldDocs); List partitionFields = new ArrayList<>(); for (String partitionKey : config.getSplitStrings(META_SYNC_PARTITION_FIELDS)) { String partitionKeyWithTicks = tickSurround(partitionKey); - partitionFields.add(partitionKeyWithTicks + " " - + getPartitionKeyType(hiveSchema, partitionKeyWithTicks)); + String partitionField = partitionKeyWithTicks + " " + + getPartitionKeyType(hiveSchema, partitionKeyWithTicks); + String doc = fieldDocs.get(partitionKey.toLowerCase(Locale.ROOT)); + if (doc != null) { + partitionField += " COMMENT '" + escapeSqlString(doc) + "'"; + } + partitionFields.add(partitionField); } String partitionsStr = String.join(",", partitionFields); @@ -401,7 +440,7 @@ private static String propertyToString(Map properties) { if (!first) { sb.append(","); } - sb.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("'"); + sb.append("'").append(escapeSqlString(entry.getKey())).append("'='").append(escapeSqlString(entry.getValue())).append("'"); first = false; } return sb.toString(); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index 6d971bcc52263..9a7dceef0e4ec 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -1032,6 +1032,186 @@ public void testSyncWithCommentedSchema(String syncMode) throws Exception { assertEquals(2, commentCnt, "hive schema field comment numbers should match the avro schema field doc numbers"); } + private static final String STANDARD_COLUMNS = + "{\"name\": \"name\", \"type\": \"string\", \"doc\": \"name_comment\"}," + + "{\"name\": \"favorite_number\", \"type\": \"int\", \"doc\": \"favorite_number_comment\"}," + + "{\"name\": \"favorite_color\", \"type\": \"string\", \"doc\": \"the person's favorite color\\\\\"}"; + + private static final String COMPLEX_COLUMNS = + "{\"name\": \"address\", \"type\": {\"type\": \"record\", \"name\": \"Address\", \"fields\": [" + + "{\"name\": \"city\", \"type\": \"string\", \"doc\": \"city_comment\"}," + + "{\"name\": \"zip\", \"type\": \"string\"}]}, \"doc\": \"address_comment\"}," + + "{\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}, \"doc\": \"tags_comment\"}," + + "{\"name\": \"scores\", \"type\": {\"type\": \"map\", \"values\": \"int\"}}"; + + private static final String SINGLE_PARTITION_COLUMN = + "{\"name\": \"datestr\", \"type\": \"string\", \"doc\": \"partition_datestr_comment\"}"; + + private static final String MULTI_PARTITION_COLUMNS = + "{\"name\": \"year\", \"type\": \"string\", \"doc\": \"partition_year_comment\"}," + + "{\"name\": \"month\", \"type\": \"string\"}," + + "{\"name\": \"day\", \"type\": \"string\", \"doc\": \"partition_day_comment\"}"; + + @ParameterizedTest + @MethodSource("syncMode") + public void testSyncCommentsForStandardColumnsWithSinglePartitionColumn(String syncMode) throws Exception { + Map commentsByField = syncTableWithCommentedSchema(syncMode, STANDARD_COLUMNS, SINGLE_PARTITION_COLUMN); + assertStandardColumnComments(commentsByField); + assertEquals("partition_datestr_comment", commentsByField.get("datestr"), + "comment of the partition column should be synced on table creation"); + assertSparkSchemaPropertyContainsComments("\"comment\":\"name_comment\"", "\"comment\":\"partition_datestr_comment\""); + } + + @ParameterizedTest + @MethodSource("syncMode") + public void testSyncCommentsForComplexColumnsWithSinglePartitionColumn(String syncMode) throws Exception { + Map commentsByField = syncTableWithCommentedSchema(syncMode, STANDARD_COLUMNS, COMPLEX_COLUMNS, SINGLE_PARTITION_COLUMN); + assertStandardColumnComments(commentsByField); + assertComplexColumnComments(commentsByField); + assertEquals("partition_datestr_comment", commentsByField.get("datestr"), + "comment of the partition column should be synced on table creation"); + // docs of nested fields are only representable in the spark schema + assertSparkSchemaPropertyContainsComments("\"comment\":\"address_comment\"", "\"comment\":\"city_comment\""); + } + + @ParameterizedTest + @MethodSource("syncMode") + public void testSyncCommentsForStandardColumnsWithMultiplePartitionColumns(String syncMode) throws Exception { + setMultiPartitionFields(); + Map commentsByField = syncTableWithCommentedSchema(syncMode, STANDARD_COLUMNS, MULTI_PARTITION_COLUMNS); + assertStandardColumnComments(commentsByField); + assertMultiPartitionColumnComments(commentsByField); + assertSparkSchemaPropertyContainsComments("\"comment\":\"name_comment\"", "\"comment\":\"partition_year_comment\""); + } + + @ParameterizedTest + @MethodSource("syncMode") + public void testSyncCommentsForComplexColumnsWithMultiplePartitionColumns(String syncMode) throws Exception { + setMultiPartitionFields(); + Map commentsByField = syncTableWithCommentedSchema(syncMode, STANDARD_COLUMNS, COMPLEX_COLUMNS, MULTI_PARTITION_COLUMNS); + assertStandardColumnComments(commentsByField); + assertComplexColumnComments(commentsByField); + assertMultiPartitionColumnComments(commentsByField); + } + + @ParameterizedTest + @MethodSource("syncMode") + public void testSyncCommentsWithSpecialCharacters(String syncMode) throws Exception { + String specialColumns = + "{\"name\": \"c_quote\", \"type\": \"string\", \"doc\": \"it's the person's 'favorite'\"}," + + "{\"name\": \"c_backslash\", \"type\": \"string\", \"doc\": \"back\\\\slash ending\\\\\"}," + + "{\"name\": \"c_double_quote\", \"type\": \"string\", \"doc\": \"he said \\\"hello\\\"\"}," + + "{\"name\": \"c_mixed\", \"type\": \"string\", \"doc\": \"semi;colon, comma=equals %percent _underscore\"}," + + "{\"name\": \"c_unicode\", \"type\": \"string\", \"doc\": \"unicode héllo 你好\"}"; + Map commentsByField = syncTableWithCommentedSchema(syncMode, specialColumns, SINGLE_PARTITION_COLUMN); + assertEquals("it's the person's 'favorite'", commentsByField.get("c_quote"), + "comment with single quotes should be synced unchanged"); + assertEquals("back\\slash ending\\", commentsByField.get("c_backslash"), + "comment with backslashes should be synced unchanged"); + assertEquals("he said \"hello\"", commentsByField.get("c_double_quote"), + "comment with double quotes should be synced unchanged"); + assertEquals("semi;colon, comma=equals %percent _underscore", commentsByField.get("c_mixed"), + "comment with SQL separator characters should be synced unchanged"); + assertEquals("unicode héllo 你好", commentsByField.get("c_unicode"), + "comment with non-ascii characters should be synced unchanged"); + assertEquals("partition_datestr_comment", commentsByField.get("datestr"), + "comment of the partition column should be synced on table creation"); + } + + @ParameterizedTest + @MethodSource("syncMode") + public void testUpdateCommentsForPartitionColumns(String syncMode) throws Exception { + hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), syncMode); + hiveSyncProps.setProperty(HIVE_SYNC_COMMENT.key(), "false"); + setMultiPartitionFields(); + String commitTime = "100"; + HiveTestUtil.createCOWTableWithSchema(commitTime, commentedSchema(STANDARD_COLUMNS, MULTI_PARTITION_COLUMNS)); + + // table is created without comments, they are applied by the second sync + reInitHiveSyncClient(); + reSyncHiveTable(); + hiveSyncProps.setProperty(HIVE_SYNC_COMMENT.key(), "true"); + hiveSyncProps.setProperty(META_SYNC_INCREMENTAL.key(), "false"); + reInitHiveSyncClient(); + reSyncHiveTable(); + + Map commentsByField = metastoreFieldComments(HiveTestUtil.TABLE_NAME); + assertStandardColumnComments(commentsByField); + if (syncMode.equals(HiveSyncMode.HMS.name().toLowerCase())) { + assertMultiPartitionColumnComments(commentsByField); + } else { + assertEquals("", commentsByField.get("year"), + "comment of a partition column cannot be updated in query based sync modes"); + assertEquals("", commentsByField.get("day"), + "comment of a partition column cannot be updated in query based sync modes"); + } + } + + private Map syncTableWithCommentedSchema(String syncMode, String... fieldJsons) throws Exception { + hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), syncMode); + hiveSyncProps.setProperty(HIVE_SYNC_COMMENT.key(), "true"); + HiveTestUtil.createCOWTableWithSchema("100", commentedSchema(fieldJsons)); + reInitHiveSyncClient(); + reSyncHiveTable(); + return metastoreFieldComments(HiveTestUtil.TABLE_NAME); + } + + private static HoodieSchema commentedSchema(String... fieldJsons) { + return HoodieSchema.parse("{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [" + + String.join(",", fieldJsons) + "]}"); + } + + private void setMultiPartitionFields() { + hiveSyncProps.setProperty(META_SYNC_PARTITION_EXTRACTOR_CLASS.key(), MultiPartKeysValueExtractor.class.getCanonicalName()); + hiveSyncProps.setProperty(META_SYNC_PARTITION_FIELDS.key(), "year,month,day"); + } + + private void assertStandardColumnComments(Map commentsByField) { + assertEquals("name_comment", commentsByField.get("name"), + "comment of a regular column should be synced"); + assertEquals("favorite_number_comment", commentsByField.get("favorite_number"), + "comment of a regular column should be synced"); + assertEquals("the person's favorite color\\", commentsByField.get("favorite_color"), + "comment with a single quote and trailing backslash should be synced unchanged"); + } + + private void assertComplexColumnComments(Map commentsByField) { + assertEquals("address_comment", commentsByField.get("address"), + "comment of a struct column should be synced"); + assertEquals("tags_comment", commentsByField.get("tags"), + "comment of an array column should be synced"); + assertEquals("", commentsByField.get("scores"), + "map column without a doc should have no comment"); + } + + private void assertMultiPartitionColumnComments(Map commentsByField) { + assertEquals("partition_year_comment", commentsByField.get("year"), + "comment of the first partition column should be synced"); + assertEquals("", commentsByField.get("month"), + "partition column without a doc should have no comment"); + assertEquals("partition_day_comment", commentsByField.get("day"), + "comment of the last partition column should be synced"); + } + + private void assertSparkSchemaPropertyContainsComments(String... expectedComments) throws Exception { + SessionState.start(HiveTestUtil.getHiveConf()); + Driver hiveDriver = new Driver(HiveTestUtil.getHiveConf()); + hiveDriver.run(String.format("SHOW TBLPROPERTIES %s.%s", HiveTestUtil.DB_NAME, HiveTestUtil.TABLE_NAME)); + List results = new ArrayList<>(); + hiveDriver.getResults(results); + String tableProperties = String.join("\n", results); + for (String expectedComment : expectedComments) { + assertTrue(tableProperties.contains(expectedComment), + "spark schema table property should contain " + expectedComment); + } + } + + private Map metastoreFieldComments(String tableName) { + return hiveClient.getMetastoreFieldSchemas(tableName) + .stream() + .collect(Collectors.toMap(FieldSchema::getName, FieldSchema::getCommentOrEmpty)); + } + @ParameterizedTest @MethodSource("syncModeAndSchemaFromCommitMetadata") public void testSyncMergeOnRead(boolean useSchemaFromCommitMetadata, String syncMode, String enablePushDown) throws Exception { diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestSparkSchemaUtils.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestSparkSchemaUtils.java index d5bf7c666e4ce..9ce19238f0939 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestSparkSchemaUtils.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestSparkSchemaUtils.java @@ -120,6 +120,31 @@ public void testConvertBasicTypes() { assertEquals(sparkSchema.json(), convertedSparkSchema.json()); } + @Test + public void testConvertWithFieldDocs() { + HoodieSchema nestedSchema = HoodieSchema.createRecord("profile", null, null, false, Arrays.asList( + HoodieSchemaField.of("city", HoodieSchema.create(HoodieSchemaType.STRING), "City of the person", null), + HoodieSchemaField.of("zip", HoodieSchema.createNullable(HoodieSchemaType.STRING), null, null))); + HoodieSchema schema = HoodieSchema.createRecord("root", null, null, false, Arrays.asList( + HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.STRING), "Unique \"id\"\nof the record", null), + HoodieSchemaField.of("name", HoodieSchema.createNullable(HoodieSchemaType.STRING), "Name of the person", null), + HoodieSchemaField.of("age", HoodieSchema.createNullable(HoodieSchemaType.INT), null, null), + HoodieSchemaField.of("profile", nestedSchema, "Profile of the person", null))); + + StructType withoutDocs = (StructType) StructType.fromJson(SparkSchemaUtils.convertToSparkSchemaJson(schema)); + assertFalse(withoutDocs.fields()[0].getComment().isDefined()); + assertFalse(((StructType) withoutDocs.fields()[3].dataType()).fields()[0].getComment().isDefined()); + + StructType withDocs = (StructType) StructType.fromJson(SparkSchemaUtils.convertToSparkSchemaJson(schema, true)); + assertEquals("Unique \"id\"\nof the record", withDocs.fields()[0].getComment().get()); + assertEquals("Name of the person", withDocs.fields()[1].getComment().get()); + assertFalse(withDocs.fields()[2].getComment().isDefined()); + assertEquals("Profile of the person", withDocs.fields()[3].getComment().get()); + StructType nestedStruct = (StructType) withDocs.fields()[3].dataType(); + assertEquals("City of the person", nestedStruct.fields()[0].getComment().get()); + assertFalse(nestedStruct.fields()[1].getComment().isDefined()); + } + @Test public void testConvertComplexType() { StructType sparkSchema = parser.parseTableSchema( diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 0e603857f26dd..b3b1e2d88d875 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -395,6 +395,11 @@ public static void addRollbackInstantToTable(String instantTime, String commitTo public static void createCOWTableWithSchema(String instantTime, String schemaFileName) throws IOException, URISyntaxException { + createCOWTableWithSchema(instantTime, SchemaTestUtil.getSchemaFromResource(HiveTestUtil.class, schemaFileName)); + } + + public static void createCOWTableWithSchema(String instantTime, HoodieSchema schema) + throws IOException, URISyntaxException { Path path = new Path(basePath); FileIOUtils.deleteDirectory(new File(basePath)); HoodieTableMetaClient.newTableBuilder() @@ -417,7 +422,6 @@ public static void createCOWTableWithSchema(String instantTime, String schemaFil String fileId = UUID.randomUUID().toString(); Path filePath = new Path(partPath.toString() + "/" + FSUtils.makeBaseFileName(instantTime, "1-0-1", fileId, HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension())); - HoodieSchema schema = SchemaTestUtil.getSchemaFromResource(HiveTestUtil.class, schemaFileName); generateParquetDataWithSchema(filePath, schema); HoodieWriteStat writeStat = new HoodieWriteStat(); writeStat.setFileId(fileId); diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkDataSourceTableUtils.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkDataSourceTableUtils.java index d747a169aa411..73b7d6c6e8990 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkDataSourceTableUtils.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkDataSourceTableUtils.java @@ -38,6 +38,17 @@ public class SparkDataSourceTableUtils { */ public static Map getSparkTableProperties(List partitionNames, String sparkVersion, int schemaLengthThreshold, HoodieSchema schema) { + return getSparkTableProperties(partitionNames, sparkVersion, schemaLengthThreshold, schema, false); + } + + /** + * Get Spark Sql related table properties. This is used for spark datasource table. + * @param schema The schema to write to the table. + * @param includeFieldDocs Whether to include the field docs as column comments in the serialized spark schema. + * @return A new parameters added the spark's table properties. + */ + public static Map getSparkTableProperties(List partitionNames, String sparkVersion, + int schemaLengthThreshold, HoodieSchema schema, boolean includeFieldDocs) { // Convert the schema and partition info used by spark sql to hive table properties. // The following code refers to the spark code in // https://github.com/apache/spark/blob/master/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -73,7 +84,7 @@ public static Map getSparkTableProperties(List partition sparkProperties.put("spark.sql.create.version", sparkVersion); } // Split the schema string to multi-parts according the schemaLengthThreshold size. - String schemaString = SparkSchemaUtils.convertToSparkSchemaJson(reOrderedSchema); + String schemaString = SparkSchemaUtils.convertToSparkSchemaJson(reOrderedSchema, includeFieldDocs); int numSchemaPart = (schemaString.length() + schemaLengthThreshold - 1) / schemaLengthThreshold; sparkProperties.put("spark.sql.sources.schema.numParts", String.valueOf(numSchemaPart)); // Add each part of schema string to sparkProperties diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkSchemaUtils.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkSchemaUtils.java index 91adf3c6097f2..044681123ad10 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkSchemaUtils.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SparkSchemaUtils.java @@ -19,8 +19,12 @@ package org.apache.hudi.sync.common.util; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; +import java.util.ArrayList; +import java.util.List; + /** * Convert the Hoodie schema to spark schema' json string. * This code is refer to org.apache.spark.sql.execution.datasources.parquet.ParquetToSparkSchemaConverter @@ -29,18 +33,69 @@ public class SparkSchemaUtils { public static String convertToSparkSchemaJson(HoodieSchema schema) { + return convertToSparkSchemaJson(schema, false); + } + + /** + * @param includeFieldDocs whether field docs should be included in the field metadata + * as {@code comment}, which spark displays as the column comment + */ + public static String convertToSparkSchemaJson(HoodieSchema schema, boolean includeFieldDocs) { String fieldsJsonString = schema.getFields().stream().map(field -> { - String metadata = "{}"; - if (field.getNonNullSchema().isBlobField()) { - metadata = String.format("{\"%s\":\"%s\"}", HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name()); - } - return "{\"name\":\"" + field.name() + "\",\"type\":" + convertFieldType(field.getNonNullSchema()) + String metadata = convertFieldMetadata(field, includeFieldDocs); + return "{\"name\":\"" + field.name() + "\",\"type\":" + convertFieldType(field.getNonNullSchema(), includeFieldDocs) + ",\"nullable\":" + field.isNullable() + ",\"metadata\":" + metadata + "}"; }).reduce((a, b) -> a + "," + b).orElse(""); return "{\"type\":\"struct\",\"fields\":[" + fieldsJsonString + "]}"; } - private static String convertFieldType(HoodieSchema originalFieldSchema) { + private static String convertFieldMetadata(HoodieSchemaField field, boolean includeFieldDocs) { + List entries = new ArrayList<>(2); + if (includeFieldDocs) { + field.doc().ifPresent(doc -> { + if (!doc.isEmpty()) { + entries.add("\"comment\":\"" + escapeJsonString(doc) + "\""); + } + }); + } + if (field.getNonNullSchema().isBlobField()) { + entries.add(String.format("\"%s\":\"%s\"", HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name())); + } + return "{" + String.join(",", entries) + "}"; + } + + private static String escapeJsonString(String value) { + StringBuilder sb = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + default: + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + return sb.toString(); + } + + private static String convertFieldType(HoodieSchema originalFieldSchema, boolean includeFieldDocs) { HoodieSchema fieldSchema = originalFieldSchema.getNonNullType(); switch (fieldSchema.getType()) { case BOOLEAN: return "\"boolean\""; @@ -78,24 +133,24 @@ private static String convertFieldType(HoodieSchema originalFieldSchema) { HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) fieldSchema; return "\"decimal(" + decimal.getPrecision() + "," + decimal.getScale() + ")\""; case ARRAY: - return arrayType(fieldSchema.getElementType()); + return arrayType(fieldSchema.getElementType(), includeFieldDocs); case MAP: HoodieSchema keyType = fieldSchema.getKeyType(); HoodieSchema valueType = fieldSchema.getValueType(); boolean valueOptional = valueType.isNullable(); - return "{\"type\":\"map\", \"keyType\":" + convertFieldType(keyType) - + ",\"valueType\":" + convertFieldType(valueType) + return "{\"type\":\"map\", \"keyType\":" + convertFieldType(keyType, includeFieldDocs) + + ",\"valueType\":" + convertFieldType(valueType, includeFieldDocs) + ",\"valueContainsNull\":" + valueOptional + "}"; case RECORD: case BLOB: case VARIANT: - return convertToSparkSchemaJson(fieldSchema); + return convertToSparkSchemaJson(fieldSchema, includeFieldDocs); default: throw new UnsupportedOperationException("Cannot convert " + fieldSchema.getType() + " to spark sql type"); } } - private static String arrayType(HoodieSchema elementType) { - return "{\"type\":\"array\", \"elementType\":" + convertFieldType(elementType) + ",\"containsNull\":" + elementType.isNullable() + "}"; + private static String arrayType(HoodieSchema elementType, boolean includeFieldDocs) { + return "{\"type\":\"array\", \"elementType\":" + convertFieldType(elementType, includeFieldDocs) + ",\"containsNull\":" + elementType.isNullable() + "}"; } } From 513d2736983ec524c51dcea6fd9782a478e0b0e2 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 20 Jul 2026 07:49:49 -0700 Subject: [PATCH 170/255] test(spark): add unit coverage for Spark row, schema and sort utilities (#19219) * test(spark): add unit coverage for Spark row, schema and sort utilities * test(spark): fix single-column range test and address review cleanups - Replace the single-column linear-order assertion, which relied on repartitionByRange sorting within partitions (it does not, per the Spark javadoc), with assertions on the actual contract: no Index column, non-overlapping ranges across partitions, both requested partitions populated, and full row preservation. - Reuse BinaryUtil.compareTo as the Z-curve ordering comparator instead of a local reimplementation. - Stop reaching into cross-module non-public ValueMetadata members: construct metadata through the public getValueMetadata factory and assert decimal precision and scale via getValueTypeInfo(). - Import cleanups (assertArrayEquals static import, Metadata import). --------- Co-authored-by: voon (cherry picked from commit 93f2d4c16aaeaf152a9e9f3fc2b67a9607ffb9be) --- .../stats/TestSparkValueMetadataUtils.java | 196 ++++++++++++++++++ .../hudi/execution/TestRangeSampleSort.java | 168 +++++++++++++++ 2 files changed, 364 insertions(+) create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java new file mode 100644 index 0000000000000..43c85b2e8b7a9 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.metadata.stats; + +import org.apache.hudi.metadata.HoodieIndexVersion; + +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.DecimalType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit coverage for the Spark value-metadata conversion helpers. Exercises the + * Spark-type to {@link ValueType} matrix, decimal precision/scale carry-through, + * and the value conversion round-trips using only in-process objects. + */ +class TestSparkValueMetadataUtils { + + private static ValueMetadata metadataFor(DataType dataType) { + return SparkValueMetadataUtils.getValueMetadata(dataType, HoodieIndexVersion.V2); + } + + private static Stream dataTypeToValueType() { + return Stream.of( + Arguments.of(DataTypes.BooleanType, ValueType.BOOLEAN), + Arguments.of(DataTypes.IntegerType, ValueType.INT), + Arguments.of(DataTypes.ShortType, ValueType.INT), + Arguments.of(DataTypes.ByteType, ValueType.INT), + Arguments.of(DataTypes.LongType, ValueType.LONG), + Arguments.of(DataTypes.FloatType, ValueType.FLOAT), + Arguments.of(DataTypes.DoubleType, ValueType.DOUBLE), + Arguments.of(DataTypes.StringType, ValueType.STRING), + Arguments.of(DataTypes.TimestampType, ValueType.TIMESTAMP_MICROS), + Arguments.of(DataTypes.DateType, ValueType.DATE), + Arguments.of(DataTypes.BinaryType, ValueType.BYTES), + Arguments.of(DataTypes.NullType, ValueType.NULL)); + } + + @ParameterizedTest + @MethodSource("dataTypeToValueType") + void getValueMetadataMapsSparkTypeToValueType(DataType dataType, ValueType expected) { + ValueMetadata metadata = SparkValueMetadataUtils.getValueMetadata(dataType, HoodieIndexVersion.V2); + assertEquals(expected, metadata.getValueType(), + "Spark type " + dataType.typeName() + " must map to value type " + expected); + } + + @Test + void getValueMetadataCarriesDecimalPrecisionAndScale() { + DecimalType decimalType = new DecimalType(12, 4); + ValueMetadata metadata = SparkValueMetadataUtils.getValueMetadata(decimalType, HoodieIndexVersion.V2); + + assertEquals(ValueType.DECIMAL, metadata.getValueType()); + assertEquals("12,4", metadata.getValueTypeInfo().getAdditionalInfo(), + "precision and scale must be encoded in the value type info"); + } + + @Test + void getValueMetadataReturnsV1EmptyBelowV2() { + // Any index version lower than V2 is unversioned column stats and yields the shared empty metadata. + ValueMetadata metadata = SparkValueMetadataUtils.getValueMetadata(DataTypes.IntegerType, HoodieIndexVersion.V1); + assertSame(ValueMetadata.V1EmptyMetadata.get(), metadata, + "index versions below V2 must return the shared V1 empty metadata"); + assertTrue(metadata.isV1()); + } + + @Test + void getValueMetadataReturnsNullMetadataForNullType() { + // A null Spark data type is distinct from NullType and maps to the shared NULL metadata singleton. + ValueMetadata metadata = SparkValueMetadataUtils.getValueMetadata(null, HoodieIndexVersion.V2); + assertSame(ValueMetadata.NULL_METADATA, metadata); + assertEquals(ValueType.NULL, metadata.getValueType()); + } + + @Test + void convertSparkToJavaReturnsNullForNullInput() { + ValueMetadata metadata = metadataFor(DataTypes.IntegerType); + assertNull(SparkValueMetadataUtils.convertSparkToJava(metadata, null)); + } + + @Test + void convertSparkToJavaPassesThroughPrimitives() { + assertEquals(Boolean.TRUE, + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.BooleanType), true)); + assertEquals(42, + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.IntegerType), 42)); + assertEquals(42L, + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.LongType), 42L)); + assertEquals(1.5f, + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.FloatType), 1.5f)); + assertEquals(2.5d, + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.DoubleType), 2.5d)); + assertEquals("hudi", + SparkValueMetadataUtils.convertSparkToJava(metadataFor(DataTypes.StringType), "hudi")); + } + + @Test + void convertSparkToJavaHandlesDecimal() { + ValueMetadata metadata = metadataFor(new DecimalType(10, 2)); + Decimal sparkDecimal = Decimal.apply(new BigDecimal("123.45")); + Comparable result = SparkValueMetadataUtils.convertSparkToJava(metadata, sparkDecimal); + assertEquals(new BigDecimal("123.45"), result, "decimal must convert to a java BigDecimal of equal value"); + } + + @Test + void convertSparkToJavaHandlesBytes() { + ValueMetadata metadata = metadataFor(DataTypes.BinaryType); + byte[] input = new byte[] {1, 2, 3, 4}; + Comparable result = SparkValueMetadataUtils.convertSparkToJava(metadata, input); + assertTrue(result instanceof ByteBuffer, "bytes must convert to a ByteBuffer"); + ByteBuffer buffer = (ByteBuffer) result; + byte[] roundTripped = new byte[buffer.remaining()]; + buffer.get(roundTripped); + assertArrayEquals(input, roundTripped, + "byte content must survive the conversion"); + } + + @Test + void convertSparkToJavaHandlesDateFromEpochDays() { + ValueMetadata metadata = metadataFor(DataTypes.DateType); + // Spark stores dates internally as epoch-day integers. + int epochDays = (int) LocalDate.of(2021, 3, 15).toEpochDay(); + Comparable result = SparkValueMetadataUtils.convertSparkToJava(metadata, epochDays); + assertEquals(LocalDate.of(2021, 3, 15), result, "epoch-day int must convert to the matching LocalDate"); + } + + @Test + void convertSparkToJavaHandlesTimestampMicros() { + ValueMetadata metadata = metadataFor(DataTypes.TimestampType); + Instant expected = Instant.ofEpochSecond(1_600_000_000L); + long micros = expected.getEpochSecond() * 1_000_000L; + Comparable result = SparkValueMetadataUtils.convertSparkToJava(metadata, micros); + assertEquals(expected, result, "micros-since-epoch must convert to the matching Instant"); + } + + @Test + void convertJavaTypeToSparkTypeConvertsInstantWhenLegacyApi() { + Instant instant = Instant.ofEpochSecond(1_600_000_000L); + // With the java8 API disabled Spark expects java.sql.Timestamp for timestamp values. + Object legacy = SparkValueMetadataUtils.convertJavaTypeToSparkType(instant, false); + assertEquals(Timestamp.from(instant), legacy); + + // With the java8 API enabled the Instant is passed through untouched. + assertSame(instant, SparkValueMetadataUtils.convertJavaTypeToSparkType(instant, true)); + } + + @Test + void convertJavaTypeToSparkTypeConvertsLocalDateWhenLegacyApi() { + LocalDate date = LocalDate.of(2022, 6, 1); + Object legacy = SparkValueMetadataUtils.convertJavaTypeToSparkType(date, false); + assertEquals(Date.valueOf(date), legacy); + + assertSame(date, SparkValueMetadataUtils.convertJavaTypeToSparkType(date, true)); + } + + @Test + void convertJavaTypeToSparkTypeLeavesOtherTypesUntouched() { + assertSame("plain", SparkValueMetadataUtils.convertJavaTypeToSparkType("plain", false)); + Integer value = 7; + assertSame(value, SparkValueMetadataUtils.convertJavaTypeToSparkType(value, false)); + } +} diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/spark/sql/hudi/execution/TestRangeSampleSort.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/spark/sql/hudi/execution/TestRangeSampleSort.java index 3b35900e6626c..2b361202ab719 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/spark/sql/hudi/execution/TestRangeSampleSort.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/spark/sql/hudi/execution/TestRangeSampleSort.java @@ -19,16 +19,28 @@ package org.apache.spark.sql.hudi.execution; +import org.apache.hudi.common.util.BinaryUtil; import org.apache.hudi.config.HoodieClusteringConfig; +import org.apache.hudi.sort.SpaceCurveSortingHelper; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.util.JavaScalaConverters; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.spark.sql.functions.spark_partition_id; class TestRangeSampleSort extends HoodieClientTestBase { @@ -54,4 +66,160 @@ void sortDataFrameBySample() { JavaScalaConverters.convertJavaListToScalaSeq(Arrays.asList("id", "content")), 1), "range sort shall not fail when 0 or 1 record incoming"); } } + + /** + * Builds a two-column integer frame; call sites pass an explicit output partition + * count so the resulting ordering is deterministic and can be asserted row by row. + */ + private Dataset buildTwoIntColumnFrame(List rows) { + StructType schema = new StructType(new StructField[] { + new StructField("c1", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("c2", DataTypes.IntegerType, true, Metadata.empty()) + }); + return sparkSession.createDataFrame(rows, schema); + } + + /** + * Recomputes the Z-curve ordinal for a two-int row using the same public byte mapping + * the helper relies on, so the expected ordering is derived independently. + */ + private byte[] expectedZOrdinal(Integer c1, Integer c2) { + byte[] b1 = BinaryUtil.intTo8Byte(c1 == null ? Integer.MAX_VALUE : c1); + byte[] b2 = BinaryUtil.intTo8Byte(c2 == null ? Integer.MAX_VALUE : c2); + return BinaryUtil.interleaving(new byte[][] {b1, b2}, 8); + } + + @Test + void orderByMappingValuesZOrderPreservesSchemaAndSortsByCurve() { + List rows = Arrays.asList( + RowFactory.create(5, 5), + RowFactory.create(1, 9), + RowFactory.create(9, 1), + RowFactory.create(1, 1)); + Dataset df = buildTwoIntColumnFrame(rows); + + Dataset ordered = SpaceCurveSortingHelper.orderDataFrameByMappingValues( + df, HoodieClusteringConfig.LayoutOptimizationStrategy.ZORDER, Arrays.asList("c1", "c2"), 1); + + // The helper drops the internal Index column, so the output schema must match the input exactly. + Assertions.assertArrayEquals( + new String[] {"c1", "c2"}, ordered.schema().fieldNames(), + "z-order output must retain only the original columns"); + + List result = ordered.collectAsList(); + Assertions.assertEquals(rows.size(), result.size(), "no rows should be dropped"); + + // Every emitted row must be non-decreasing under the independently recomputed Z-curve ordinal. + for (int i = 1; i < result.size(); i++) { + byte[] prev = expectedZOrdinal(result.get(i - 1).getInt(0), result.get(i - 1).getInt(1)); + byte[] curr = expectedZOrdinal(result.get(i).getInt(0), result.get(i).getInt(1)); + Assertions.assertTrue(BinaryUtil.compareTo(prev, 0, prev.length, curr, 0, curr.length) <= 0, + "row " + i + " violates ascending Z-curve order"); + } + + // (1,1) has the smallest interleaved ordinal, so it must sort first. + Row first = result.get(0); + Assertions.assertEquals(1, first.getInt(0)); + Assertions.assertEquals(1, first.getInt(1)); + } + + @Test + void orderByMappingValuesZOrderPlacesNullLast() { + List rows = new ArrayList<>(); + rows.add(RowFactory.create(2, 2)); + rows.add(RowFactory.create(null, null)); + rows.add(RowFactory.create(1, 1)); + Dataset df = buildTwoIntColumnFrame(rows); + + Dataset ordered = SpaceCurveSortingHelper.orderDataFrameByMappingValues( + df, HoodieClusteringConfig.LayoutOptimizationStrategy.ZORDER, Arrays.asList("c1", "c2"), 1); + + List result = ordered.collectAsList(); + Assertions.assertEquals(3, result.size()); + // Nulls map to Integer.MAX_VALUE in the byte mapping, so the all-null row sorts last. + Row last = result.get(result.size() - 1); + Assertions.assertTrue(last.isNullAt(0) && last.isNullAt(1), + "the all-null row must sort last under Z-curve mapping"); + } + + @Test + void orderByMappingValuesHilbertPreservesRowsAndSchema() { + List rows = Arrays.asList( + RowFactory.create(7, 3), + RowFactory.create(0, 0), + RowFactory.create(4, 4)); + Dataset df = buildTwoIntColumnFrame(rows); + + Dataset ordered = SpaceCurveSortingHelper.orderDataFrameByMappingValues( + df, HoodieClusteringConfig.LayoutOptimizationStrategy.HILBERT, Arrays.asList("c1", "c2"), 1); + + Assertions.assertArrayEquals(new String[] {"c1", "c2"}, ordered.schema().fieldNames(), + "hilbert output must retain only the original columns"); + List result = ordered.collectAsList(); + Assertions.assertEquals(rows.size(), result.size(), "hilbert ordering must not drop rows"); + // Origin (0,0) maps to the smallest Hilbert index, so it must sort first. + Assertions.assertEquals(0, result.get(0).getInt(0)); + Assertions.assertEquals(0, result.get(0).getInt(1)); + } + + @Test + void orderByMappingValuesSingleColumnRangePartitionsRows() { + List rows = Arrays.asList( + RowFactory.create(3, 30), + RowFactory.create(1, 10), + RowFactory.create(2, 20), + RowFactory.create(4, 40)); + Dataset df = buildTwoIntColumnFrame(rows); + + // A single ordering column short-circuits space-curve mapping into a plain range + // repartition. Spark's repartitionByRange does not sort rows within a partition, + // so only the cross-partition range contract can be asserted. + Dataset ordered = SpaceCurveSortingHelper.orderDataFrameByMappingValues( + df, HoodieClusteringConfig.LayoutOptimizationStrategy.ZORDER, Arrays.asList("c1"), 2); + + Assertions.assertArrayEquals(new String[] {"c1", "c2"}, ordered.schema().fieldNames(), + "single-column ordering must not add an Index column"); + + List result = ordered.withColumn("pid", spark_partition_id()).collectAsList(); + Assertions.assertEquals(rows.size(), result.size(), "no rows should be dropped"); + + // Range partitioning must not overlap ranges across partitions: every c1 in a + // lower-numbered partition is <= every c1 in a higher-numbered partition. + for (Row left : result) { + for (Row right : result) { + if (left.getInt(2) < right.getInt(2)) { + Assertions.assertTrue(left.getInt(0) <= right.getInt(0), + "partition ranges must not overlap"); + } + } + } + + // With four distinct keys and two target partitions the deterministic range + // bounds split the rows across both partitions, so the check above is not vacuous. + Assertions.assertEquals(2, + result.stream().map(r -> r.getInt(2)).collect(Collectors.toSet()).size(), + "rows must be spread across both requested partitions"); + + // The row multiset must be preserved: each c1 appears once with its matching c2. + List c1Values = new ArrayList<>(); + for (Row r : result) { + Assertions.assertEquals(r.getInt(0) * 10, r.getInt(1), "row content must be unchanged"); + c1Values.add(r.getInt(0)); + } + c1Values.sort(Integer::compareTo); + Assertions.assertEquals(Arrays.asList(1, 2, 3, 4), c1Values, + "range repartition must preserve all rows"); + } + + @Test + void orderByMappingValuesUnknownColumnReturnsInputUnchanged() { + List rows = Arrays.asList(RowFactory.create(2, 2), RowFactory.create(1, 1)); + Dataset df = buildTwoIntColumnFrame(rows); + + // Ordering by a column absent from the schema must be a no-op that returns the same frame. + Dataset ordered = SpaceCurveSortingHelper.orderDataFrameByMappingValues( + df, HoodieClusteringConfig.LayoutOptimizationStrategy.ZORDER, Arrays.asList("missing"), 1); + + Assertions.assertSame(df, ordered, "missing order column must return the untouched input frame"); + } } From eaa2af653ab7d8193f0ac02e2b43148e1be27271 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 20 Jul 2026 08:22:31 -0700 Subject: [PATCH 171/255] test(common): add unit coverage for config and utility classes (#19220) * test(common): add unit coverage for config and utility classes * fix(common): add missing READER group description and address review findings - Fix checkstyle ImportOrder violation in TestDateTimeUtils (java.time.temporal.ChronoUnit must follow java.time.format imports). - Add the missing READER case to ConfigGroups.getDescription, surfaced by the new placeholder test: the Reader Configs group fell through to the 'Please fill in the description' default, which leaks into the generated config docs. - Strengthen testFormatUnixTimestamp to re-parse the formatted output instead of asserting only its length, matching its comment. - Import cleanup in TestCollectionUtils and an intent comment on the emptyProps singleton assertion. - Rename copy-paste-leftover locals in generateChecksum (database/table to indexType/indexName) and correct the checksum format comment; the produced checksum is byte-identical. * test(common): make new test classes package-private and align index names with types - Drop the public modifier from TestConfigGroups and TestHoodieIndexingConfig and their test methods, matching the JUnit 5 package-private idiom used by TestCollectionUtils. TestDateTimeUtils keeps public to match its pre-existing file style. - Use index names that match the configured index type (idx_bloom, idx_record, idx_original) instead of reusing column_stats for non-column-stats types, which read confusingly. --------- Co-authored-by: voon (cherry picked from commit bd2888c6370800b2386cad7daf1e8dd0788519bb) --- .../hudi/common/config/ConfigGroups.java | 4 + .../common/config/HoodieIndexingConfig.java | 8 +- .../hudi/common/config/TestConfigGroups.java | 68 +++++++ .../config/TestHoodieIndexingConfig.java | 177 ++++++++++++++++++ .../hudi/common/util/TestCollectionUtils.java | 170 +++++++++++++++++ .../hudi/common/util/TestDateTimeUtils.java | 108 +++++++++++ 6 files changed, 531 insertions(+), 4 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigGroups.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieIndexingConfig.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java index 2048b21c7e261..3ebc61a2de79d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -130,6 +130,10 @@ public static String getDescription(Names names) { + "write schema, cleaning etc. Although Hudi provides sane defaults, from time-time " + "these configs may need to be tweaked to optimize for specific workloads."; break; + case READER: + description = "These set of configs control the behavior of reading Hudi tables, " + + "such as file group reading."; + break; case META_SYNC: description = "Configurations used by the Hudi to sync metadata to external metastores and catalogs."; break; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieIndexingConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieIndexingConfig.java index 0ab9158ad6e8c..6582d5ad91535 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieIndexingConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieIndexingConfig.java @@ -95,7 +95,7 @@ public class HoodieIndexingConfig extends HoodieConfig { .withDocumentation("Index definition checksum is used to guard against partial writes in HDFS. " + "It is added as the last entry in index.properties and then used to validate while reading table config."); - private static final String INDEX_DEFINITION_CHECKSUM_FORMAT = "%s.%s"; // . + private static final String INDEX_DEFINITION_CHECKSUM_FORMAT = "%s.%s"; // . public HoodieIndexingConfig() { super(); @@ -208,9 +208,9 @@ public static long generateChecksum(Properties props) { if (!props.containsKey(INDEX_NAME.key())) { throw new IllegalArgumentException(INDEX_NAME.key() + " property needs to be specified"); } - String table = props.getProperty(INDEX_NAME.key()); - String database = props.getProperty(INDEX_TYPE.key(), ""); - return BinaryUtil.generateChecksum(getUTF8Bytes(String.format(INDEX_DEFINITION_CHECKSUM_FORMAT, database, table))); + String indexName = props.getProperty(INDEX_NAME.key()); + String indexType = props.getProperty(INDEX_TYPE.key(), ""); + return BinaryUtil.generateChecksum(getUTF8Bytes(String.format(INDEX_DEFINITION_CHECKSUM_FORMAT, indexType, indexName))); } public static boolean validateChecksum(Properties props) { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigGroups.java b/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigGroups.java new file mode 100644 index 0000000000000..acbd318421ad5 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigGroups.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.config; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link ConfigGroups}. + */ +class TestConfigGroups { + + @ParameterizedTest + @EnumSource(ConfigGroups.Names.class) + void testGetDescriptionReturnsNonPlaceholderForEveryName(ConfigGroups.Names name) { + String description = ConfigGroups.getDescription(name); + assertNotNull(description); + assertFalse(description.isEmpty(), "Description should not be empty for " + name); + assertFalse(description.startsWith("Please fill in the description"), + "Every enum constant should have a real description branch, missing for " + name); + } + + @Test + void testGetDescriptionSpecificValues() { + assertEquals("Basic Hudi Table configuration parameters.", + ConfigGroups.getDescription(ConfigGroups.Names.TABLE_CONFIG)); + assertEquals("Configurations specific to Amazon Web Services.", + ConfigGroups.getDescription(ConfigGroups.Names.AWS)); + assertTrue(ConfigGroups.getDescription(ConfigGroups.Names.ENVIRONMENT_CONFIG) + .contains("hudi-defaults.conf")); + } + + @Test + void testNamesCarryHumanReadableName() { + assertEquals("Hudi Table Config", ConfigGroups.Names.TABLE_CONFIG.name); + assertEquals("Metrics Configs", ConfigGroups.Names.METRICS.name); + } + + @Test + void testSubGroupNamesCarryNameAndDescription() { + assertEquals("Index Configs", ConfigGroups.SubGroupNames.INDEX.name); + assertTrue(ConfigGroups.SubGroupNames.INDEX.getDescription().contains("indexing behavior")); + assertEquals("None", ConfigGroups.SubGroupNames.NONE.name); + assertNotNull(ConfigGroups.SubGroupNames.LOCK.getDescription()); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieIndexingConfig.java b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieIndexingConfig.java new file mode 100644 index 0000000000000..2dd94773cd7bf --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieIndexingConfig.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.config; + +import org.apache.hudi.common.model.HoodieIndexDefinition; +import org.apache.hudi.metadata.MetadataPartitionType; + +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link HoodieIndexingConfig}. + */ +class TestHoodieIndexingConfig { + + @Test + void testBuilderSetsProvidedValues() { + HoodieIndexingConfig config = HoodieIndexingConfig.newBuilder() + .withIndexName("idx_bloom") + .withIndexType(MetadataPartitionType.BLOOM_FILTERS.name()) + .withIndexFunction("lower") + .build(); + + assertEquals("idx_bloom", config.getIndexName()); + assertEquals(MetadataPartitionType.BLOOM_FILTERS.name(), config.getIndexType()); + assertEquals("lower", config.getIndexFunction()); + } + + @Test + void testBuildAppliesDefaultIndexType() { + HoodieIndexingConfig config = HoodieIndexingConfig.newBuilder() + .withIndexName("column_stats") + .build(); + + // INDEX_TYPE defaults to COLUMN_STATS, INDEX_NAME and INDEX_FUNCTION have no defaults. + assertEquals(MetadataPartitionType.COLUMN_STATS.name(), config.getIndexType()); + assertEquals("column_stats", config.getIndexName()); + assertNull(config.getIndexFunction()); + } + + @Test + void testIsIndexUsingHelpers() { + HoodieIndexingConfig bloom = HoodieIndexingConfig.newBuilder() + .withIndexName("idx_bloom") + .withIndexType(MetadataPartitionType.BLOOM_FILTERS.name()) + .build(); + assertTrue(bloom.isIndexUsingBloomFilter()); + assertFalse(bloom.isIndexUsingColumnStats()); + assertFalse(bloom.isIndexUsingRecordIndex()); + + HoodieIndexingConfig columnStats = HoodieIndexingConfig.newBuilder() + .withIndexName("column_stats") + .withIndexType(MetadataPartitionType.COLUMN_STATS.name()) + .build(); + assertTrue(columnStats.isIndexUsingColumnStats()); + + HoodieIndexingConfig recordIndex = HoodieIndexingConfig.newBuilder() + .withIndexName("idx_record") + .withIndexType(MetadataPartitionType.RECORD_INDEX.name()) + .build(); + assertTrue(recordIndex.isIndexUsingRecordIndex()); + } + + @Test + void testFromPropertiesCarriesOverValues() { + Properties props = new Properties(); + props.setProperty(HoodieIndexingConfig.INDEX_NAME.key(), "column_stats"); + props.setProperty(HoodieIndexingConfig.INDEX_FUNCTION.key(), "identity"); + + HoodieIndexingConfig config = HoodieIndexingConfig.newBuilder() + .fromProperties(props) + .build(); + + assertEquals("column_stats", config.getIndexName()); + assertEquals("identity", config.getIndexFunction()); + } + + @Test + void testCopyProducesEqualConfig() { + HoodieIndexingConfig source = HoodieIndexingConfig.newBuilder() + .withIndexName("idx_bloom") + .withIndexType(MetadataPartitionType.BLOOM_FILTERS.name()) + .withIndexFunction("lower") + .build(); + + HoodieIndexingConfig copy = HoodieIndexingConfig.copy(source); + assertEquals(source.getIndexName(), copy.getIndexName()); + assertEquals(source.getIndexType(), copy.getIndexType()); + assertEquals(source.getIndexFunction(), copy.getIndexFunction()); + } + + @Test + void testMergeLetsSecondConfigOverride() { + HoodieIndexingConfig first = HoodieIndexingConfig.newBuilder() + .withIndexName("idx_original") + .withIndexType(MetadataPartitionType.COLUMN_STATS.name()) + .build(); + HoodieIndexingConfig second = HoodieIndexingConfig.newBuilder() + .withIndexType(MetadataPartitionType.BLOOM_FILTERS.name()) + .withIndexFunction("upper") + .build(); + + HoodieIndexingConfig merged = HoodieIndexingConfig.merge(first, second); + assertEquals("idx_original", merged.getIndexName()); + assertEquals(MetadataPartitionType.BLOOM_FILTERS.name(), merged.getIndexType()); + assertEquals("upper", merged.getIndexFunction()); + } + + @Test + void testFromIndexDefinition() { + HoodieIndexDefinition definition = HoodieIndexDefinition.newBuilder() + .withIndexName("column_stats") + .withIndexType(MetadataPartitionType.COLUMN_STATS.name()) + .withIndexFunction("identity") + .build(); + + HoodieIndexingConfig config = HoodieIndexingConfig.fromIndexDefinition(definition); + assertEquals("column_stats", config.getIndexName()); + assertEquals(MetadataPartitionType.COLUMN_STATS.name(), config.getIndexType()); + assertEquals("identity", config.getIndexFunction()); + } + + @Test + void testGenerateAndValidateChecksum() { + Properties props = new Properties(); + props.setProperty(HoodieIndexingConfig.INDEX_NAME.key(), "column_stats"); + props.setProperty(HoodieIndexingConfig.INDEX_TYPE.key(), MetadataPartitionType.COLUMN_STATS.name()); + + long checksum = HoodieIndexingConfig.generateChecksum(props); + // Checksum must be deterministic for the same inputs. + assertEquals(checksum, HoodieIndexingConfig.generateChecksum(props)); + + props.setProperty(HoodieIndexingConfig.INDEX_DEFINITION_CHECKSUM.key(), String.valueOf(checksum)); + assertTrue(HoodieIndexingConfig.validateChecksum(props)); + + props.setProperty(HoodieIndexingConfig.INDEX_DEFINITION_CHECKSUM.key(), String.valueOf(checksum + 1)); + assertFalse(HoodieIndexingConfig.validateChecksum(props)); + } + + @Test + void testGenerateChecksumRequiresIndexName() { + Properties props = new Properties(); + props.setProperty(HoodieIndexingConfig.INDEX_TYPE.key(), MetadataPartitionType.COLUMN_STATS.name()); + assertThrows(IllegalArgumentException.class, () -> HoodieIndexingConfig.generateChecksum(props)); + } + + @Test + void testDefaultExpressionIndexRangeMetadataStorageLevel() { + HoodieIndexingConfig config = HoodieIndexingConfig.newBuilder() + .withIndexName("column_stats") + .build(); + assertEquals("MEMORY_AND_DISK_SER", config.getExpressionIndexRangeMetadataStorageLevel()); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCollectionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCollectionUtils.java index 75829f112a38c..1b9849078c85c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCollectionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCollectionUtils.java @@ -19,6 +19,8 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.util.collection.Pair; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -28,14 +30,39 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.hudi.common.util.CollectionUtils.append; import static org.apache.hudi.common.util.CollectionUtils.batches; +import static org.apache.hudi.common.util.CollectionUtils.combine; import static org.apache.hudi.common.util.CollectionUtils.containsAll; +import static org.apache.hudi.common.util.CollectionUtils.copy; +import static org.apache.hudi.common.util.CollectionUtils.createImmutableList; +import static org.apache.hudi.common.util.CollectionUtils.createImmutableMap; +import static org.apache.hudi.common.util.CollectionUtils.createImmutableSet; +import static org.apache.hudi.common.util.CollectionUtils.createSet; +import static org.apache.hudi.common.util.CollectionUtils.diff; +import static org.apache.hudi.common.util.CollectionUtils.diffSet; +import static org.apache.hudi.common.util.CollectionUtils.elementsEqual; +import static org.apache.hudi.common.util.CollectionUtils.isNullOrEmpty; +import static org.apache.hudi.common.util.CollectionUtils.nonEmpty; +import static org.apache.hudi.common.util.CollectionUtils.reduce; +import static org.apache.hudi.common.util.CollectionUtils.reverseMap; +import static org.apache.hudi.common.util.CollectionUtils.tail; +import static org.apache.hudi.common.util.CollectionUtils.toList; +import static org.apache.hudi.common.util.CollectionUtils.toStream; +import static org.apache.hudi.common.util.CollectionUtils.zipToMap; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class TestCollectionUtils { @@ -95,4 +122,147 @@ void getBatchesFromList() { assertEquals(Arrays.asList(1, 2, 3, 4, 5), intsBatches2.get(0)); assertEquals(Collections.singletonList(6), intsBatches2.get(1)); } + + @Test + void isNullOrEmptyAndNonEmptyForCollections() { + assertTrue(isNullOrEmpty((List) null)); + assertTrue(isNullOrEmpty(Collections.emptyList())); + assertFalse(isNullOrEmpty(Collections.singletonList("a"))); + assertFalse(nonEmpty((List) null)); + assertFalse(nonEmpty(Collections.emptyList())); + assertTrue(nonEmpty(Collections.singletonList("a"))); + } + + @Test + void isNullOrEmptyAndNonEmptyForMaps() { + assertTrue(isNullOrEmpty((Map) null)); + assertTrue(isNullOrEmpty(Collections.emptyMap())); + assertFalse(isNullOrEmpty(Collections.singletonMap("k", "v"))); + assertFalse(nonEmpty((Map) null)); + assertTrue(nonEmpty(Collections.singletonMap("k", "v"))); + } + + @Test + void reduceAppliesReducerSequentially() { + int sum = reduce(Arrays.asList(1, 2, 3, 4), 0, Integer::sum); + assertEquals(10, sum); + assertEquals(100, reduce(Collections.emptyList(), 100, Integer::sum)); + } + + @Test + void copyReturnsIndependentProperties() { + Properties original = new Properties(); + original.setProperty("k", "v"); + Properties copied = copy(original); + assertEquals("v", copied.getProperty("k")); + copied.setProperty("k2", "v2"); + assertFalse(original.containsKey("k2"), "Copy must not share state with the original"); + } + + @Test + void tailReturnsLastElementAndRejectsEmpty() { + assertEquals("c", tail(new String[] {"a", "b", "c"})); + assertThrows(IllegalArgumentException.class, () -> tail(new String[0])); + } + + @Test + void toStreamAndToListDrainIterator() { + List source = Arrays.asList(1, 2, 3); + assertEquals(source, toList(source.iterator())); + assertEquals(source, toStream(source.iterator()).collect(Collectors.toList())); + } + + @Test + void combineArraysAndAppendElement() { + Integer[] combined = combine(new Integer[] {1, 2}, new Integer[] {3, 4}); + assertEquals(Arrays.asList(1, 2, 3, 4), Arrays.asList(combined)); + Integer[] appended = append(new Integer[] {1, 2}, 3); + assertEquals(Arrays.asList(1, 2, 3), Arrays.asList(appended)); + } + + @Test + void combineListsAndMaps() { + assertEquals(Arrays.asList(1, 2, 3, 4), combine(Arrays.asList(1, 2), Arrays.asList(3, 4))); + + Map one = new HashMap<>(); + one.put("a", 1); + one.put("b", 2); + Map another = new HashMap<>(); + another.put("b", 20); + another.put("c", 3); + + Map overridden = combine(one, another); + assertEquals(20, overridden.get("b"), "Second map should override on key conflict"); + assertEquals(3, overridden.size()); + + Map merged = combine(one, another, Integer::sum); + assertEquals(22, merged.get("b"), "Merge function should combine overlapping values"); + assertEquals(1, merged.get("a")); + assertEquals(3, merged.get("c")); + } + + @Test + void zipToMapPairsKeysAndValues() { + Map zipped = zipToMap(Arrays.asList("a", "b"), Arrays.asList(1, 2)); + assertEquals(1, zipped.get("a")); + assertEquals(2, zipped.get("b")); + assertThrows(IllegalArgumentException.class, + () -> zipToMap(Arrays.asList("a", "b"), Collections.singletonList(1))); + } + + @Test + void diffAndDiffSetRemoveCommonElements() { + Set setDiff = diffSet(Arrays.asList(1, 2, 3), new HashSet<>(Arrays.asList(2, 3))); + assertEquals(Collections.singleton(1), setDiff); + + List listDiff = diff(Arrays.asList(1, 2, 2, 3), Collections.singletonList(3)); + assertEquals(Arrays.asList(1, 2, 2), listDiff); + } + + @Test + void elementsEqualComparesInOrder() { + assertTrue(elementsEqual(Arrays.asList(1, 2, 3).iterator(), Arrays.asList(1, 2, 3).iterator())); + assertFalse(elementsEqual(Arrays.asList(1, 2).iterator(), Arrays.asList(1, 2, 3).iterator())); + assertFalse(elementsEqual(Arrays.asList(1, 2, 3).iterator(), Arrays.asList(1, 2).iterator())); + assertFalse(elementsEqual(Arrays.asList(1, 9).iterator(), Arrays.asList(1, 2).iterator())); + } + + @Test + void createImmutableCollectionsRejectMutation() { + List list = createImmutableList("a", "b"); + assertEquals(Arrays.asList("a", "b"), list); + assertThrows(UnsupportedOperationException.class, () -> list.add("c")); + + Set set = createImmutableSet("a", "b"); + assertEquals(new HashSet<>(Arrays.asList("a", "b")), set); + assertThrows(UnsupportedOperationException.class, () -> set.add("c")); + + Map map = createImmutableMap(Pair.of("a", 1), Pair.of("b", 2)); + assertEquals(1, map.get("a")); + assertEquals(2, map.get("b")); + assertThrows(UnsupportedOperationException.class, () -> map.put("c", 3)); + } + + @Test + void createSetCollectsDistinctElements() { + assertEquals(new HashSet<>(Arrays.asList("a", "b")), createSet("a", "b", "a")); + } + + @Test + void reverseMapSwapsKeysAndValues() { + Map source = new HashMap<>(); + source.put("a", 1); + source.put("b", 2); + Map reversed = reverseMap(source); + assertEquals("a", reversed.get(1)); + assertEquals("b", reversed.get(2)); + assertThrows(UnsupportedOperationException.class, () -> reversed.put(3, "c")); + } + + @Test + void emptyPropsReturnsSharedSingleton() { + // Emptiness also guards against illegal mutation of the shared singleton elsewhere. + assertTrue(CollectionUtils.emptyProps().isEmpty()); + assertSame(CollectionUtils.emptyProps(), CollectionUtils.emptyProps()); + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java index 0b886ffcca19d..d9de5b98be7b8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java @@ -21,11 +21,19 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -55,4 +63,104 @@ public void testParseDateTimeWithNull() { DateTimeUtils.parseDateTime(null); }); } + + @Test + public void testParseDateTimeParsesEpochMillisAsMillis() { + // A numeric string is treated as epoch millis, not ISO-8601. + assertEquals(Instant.ofEpochMilli(1612542030000L), DateTimeUtils.parseDateTime("1612542030000")); + } + + @Test + public void testMicrosInstantRoundTripForPositiveEpoch() { + Instant instant = Instant.ofEpochSecond(1, 2000); + assertEquals(1_000_002L, DateTimeUtils.instantToMicros(instant)); + assertEquals(instant, DateTimeUtils.microsToInstant(1_000_002L)); + } + + @Test + public void testMicrosInstantForNegativeEpochWithNanos() { + // Before the epoch with a sub-second nano component exercises the negative-seconds branch. + Instant instant = Instant.ofEpochSecond(-2, 500_000_000); + long micros = DateTimeUtils.instantToMicros(instant); + assertEquals(-1_500_000L, micros); + assertEquals(instant, DateTimeUtils.microsToInstant(micros)); + } + + @Test + public void testNanosInstantRoundTripForPositiveEpoch() { + Instant instant = Instant.ofEpochSecond(3, 456); + assertEquals(3_000_000_456L, DateTimeUtils.instantToNanos(instant)); + assertEquals(instant, DateTimeUtils.nanosToInstant(3_000_000_456L)); + } + + @Test + public void testNanosInstantForNegativeEpochWithNanos() { + Instant instant = Instant.ofEpochSecond(-2, 500_000_000); + long nanos = DateTimeUtils.instantToNanos(instant); + assertEquals(-1_500_000_000L, nanos); + assertEquals(instant, DateTimeUtils.nanosToInstant(nanos)); + } + + @Test + public void testMicrosMillisConversion() { + assertEquals(1234L, DateTimeUtils.microsToMillis(1_234_567L)); + // floorDiv rounds towards negative infinity for negative micros. + assertEquals(-158L, DateTimeUtils.microsToMillis(-157_500L)); + assertEquals(1_234_000L, DateTimeUtils.millisToMicros(1234L)); + } + + @ParameterizedTest + @CsvSource({ + "123, 123", + "'123ms', 123", + "'321 s', 321000", + "'2 min', 120000", + "'1 day', 86400000" + }) + public void testParseDurationValidLabels(String text, long expectedMillis) { + assertEquals(Duration.of(expectedMillis, ChronoUnit.MILLIS), DateTimeUtils.parseDuration(text)); + } + + @Test + public void testParseDurationDefaultsToMillisWhenUnitOmitted() { + assertEquals(Duration.of(500, ChronoUnit.MILLIS), DateTimeUtils.parseDuration("500")); + } + + @Test + public void testParseDurationRejectsUnknownUnit() { + assertThrows(IllegalArgumentException.class, () -> DateTimeUtils.parseDuration("10 fortnights")); + } + + @Test + public void testParseDurationRejectsMissingNumber() { + assertThrows(NumberFormatException.class, () -> DateTimeUtils.parseDuration("ms")); + } + + @ParameterizedTest + @ValueSource(strings = {"", " "}) + public void testParseDurationRejectsBlank(String text) { + assertThrows(IllegalArgumentException.class, () -> DateTimeUtils.parseDuration(text)); + } + + @Test + public void testParseDurationRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> DateTimeUtils.parseDuration(null)); + } + + @Test + public void testFormatUnixTimestamp() { + // Uses the system default zone, so validate by re-parsing the formatted output rather + // than hardcoding a zone-dependent string. + long unixTimestamp = 1_612_542_030L; + String pattern = "yyyy-MM-dd HH:mm:ss"; + String formatted = DateTimeUtils.formatUnixTimestamp(unixTimestamp, pattern); + LocalDateTime parsed = LocalDateTime.parse(formatted, DateTimeFormatter.ofPattern(pattern)); + assertEquals(unixTimestamp, parsed.atZone(ZoneId.systemDefault()).toEpochSecond()); + assertDoesNotThrow(() -> DateTimeUtils.formatUnixTimestamp(unixTimestamp, "yyyy")); + } + + @Test + public void testFormatUnixTimestampRejectsEmptyFormat() { + assertThrows(IllegalArgumentException.class, () -> DateTimeUtils.formatUnixTimestamp(0L, "")); + } } From 7cdc03e8bd4923b61c8ffe0b69bff58546fa6ba1 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 20 Jul 2026 08:22:45 -0700 Subject: [PATCH 172/255] test(client): add unit coverage for client utilities and services (#19222) * test(client): add unit coverage for client utilities and services * fix(client): fix TSM client default config and lock owner info; address review findings - Fix two compile errors in TestHoodieTableServiceManagerClient: import HoodieTableType from common.model, and declare the checked UnsupportedEncodingException from URLDecoder.decode via throws IOException. - TABLE_SERVICE_MANAGER_DEPLOY_EXTRA_PARAMS now defaults to the empty string: its getter uses getStringOrDefault, which throws for no-default properties, so every compact/clean/cluster call through HoodieTableServiceManagerClient failed with HoodieException unless hoodie.table.service.manager.deploy.extra.params was explicitly set. - FileSystemBasedLockProvider.acquireLock writes the owner lock info unconditionally after the atomic create: the previous 'if (!storage.exists(lockFile))' guard ran right after create() and was always false, so lock files were empty and lock-conflict messages from LockManager and TimeGeneratorBase carried no owner info. - Consolidate filesystem lock provider tests in hudi-client-common: move the default-lock-path and non-reentrancy cases from hudi-spark-client's TestFileBasedLockProvider (which used no Spark) into TestFileSystemBasedLockProvider and delete the old file. - Test nits: drop an inert retry-wait property, import TypedProperties instead of inline qualification, use assertEquals over assertTrue(equals). --------- Co-authored-by: voon (cherry picked from commit f558c0039457de30bcf4fd6d8ecf1972dd3bc4af) --- .../lock/FileSystemBasedLockProvider.java | 6 +- .../TestHoodieTableServiceManagerClient.java | 215 +++++++++++++++++ .../lock/TestFileSystemBasedLockProvider.java | 228 ++++++++++++++++++ .../client/TestFileBasedLockProvider.java | 114 --------- .../HoodieTableServiceManagerConfig.java | 2 +- 5 files changed, 446 insertions(+), 119 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestHoodieTableServiceManagerClient.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java delete mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestFileBasedLockProvider.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java index 235f254cd914e..7c9362d34e404 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java @@ -176,10 +176,8 @@ private boolean checkIfExpired() { private void acquireLock() { try (OutputStream os = storage.create(this.lockFile, false)) { - if (!storage.exists(this.lockFile)) { - initLockInfo(); - os.write(StringUtils.getUTF8Bytes(lockInfo.toString())); - } + initLockInfo(); + os.write(StringUtils.getUTF8Bytes(lockInfo.toString())); } catch (IOException e) { throw new HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestHoodieTableServiceManagerClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestHoodieTableServiceManagerClient.java new file mode 100644 index 0000000000000..6fb8b29ed782a --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestHoodieTableServiceManagerClient.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.common.config.HoodieTableServiceManagerConfig; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieRemoteException; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link HoodieTableServiceManagerClient} with the HTTP transport mocked by a + * local in-process {@link HttpServer}. Assertions cover the request path, the query + * parameters sent to the service, and retry/error handling. + */ +public class TestHoodieTableServiceManagerClient { + + private static final String DB_NAME = "test_db"; + private static final String TABLE_NAME = "test_table"; + + @TempDir + Path tempDir; + + private HttpServer server; + + @AfterEach + public void tearDown() { + if (server != null) { + server.stop(0); + server = null; + } + } + + private HoodieTableMetaClient initMetaClient() throws IOException { + Properties props = new Properties(); + props.setProperty(HoodieTableConfig.NAME.key(), TABLE_NAME); + props.setProperty(HoodieTableConfig.DATABASE_NAME.key(), DB_NAME); + return HoodieTestUtils.init( + HoodieTestUtils.getDefaultStorageConf(), + tempDir.resolve("table").toString(), + HoodieTableType.COPY_ON_WRITE, + props); + } + + private HoodieTableServiceManagerConfig configFor(String uri) { + Properties props = new Properties(); + // Keep retry cheap so the error-path test stays fast. + props.setProperty(HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_RETRIES.key(), "2"); + props.setProperty(HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_RETRY_DELAY_SEC.key(), "1"); + props.setProperty(HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_TIMEOUT_SEC.key(), "5"); + return HoodieTableServiceManagerConfig.newBuilder().fromProperties(props).setURIs(uri).build(); + } + + /** + * Parses a raw query string of the form {@code a=1&b=2} into a decoded map. + */ + private static Map parseQuery(String rawQuery) throws IOException { + Map params = new HashMap<>(); + if (rawQuery == null || rawQuery.isEmpty()) { + return params; + } + for (String pair : rawQuery.split("&")) { + int idx = pair.indexOf('='); + String key = idx >= 0 ? pair.substring(0, idx) : pair; + String value = idx >= 0 ? pair.substring(idx + 1) : ""; + params.put( + URLDecoder.decode(key, StandardCharsets.UTF_8.name()), + URLDecoder.decode(value, StandardCharsets.UTF_8.name())); + } + return params; + } + + /** + * Starts a local HTTP server that records the request path and query params of the last + * request and replies 200. Returns the base URI (scheme + host + port). + */ + private String startRecordingServer(Map captured) throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + captured.put("path", exchange.getRequestURI().getPath()); + captured.put("query", parseQuery(exchange.getRequestURI().getRawQuery())); + byte[] body = "ok".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + } + }); + server.start(); + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + @Test + public void testExecuteCompactionSendsExpectedRequest() throws IOException { + Map captured = new HashMap<>(); + String uri = startRecordingServer(captured); + HoodieTableServiceManagerClient client = + new HoodieTableServiceManagerClient(initMetaClient(), configFor(uri)); + + Option result = client.executeCompaction(); + + // With no pending compaction the instant range is empty, but the request is still sent. + assertTrue(result.isPresent()); + assertEquals("", result.get()); + assertEquals(HoodieTableServiceManagerClient.EXECUTE_COMPACTION, captured.get("path")); + + @SuppressWarnings("unchecked") + Map query = (Map) captured.get("query"); + assertEquals(HoodieTableServiceManagerClient.Action.REQUEST.name(), + query.get(HoodieTableServiceManagerClient.ACTION)); + assertEquals(DB_NAME, query.get(HoodieTableServiceManagerClient.DATABASE_NAME_PARAM)); + assertEquals(TABLE_NAME, query.get(HoodieTableServiceManagerClient.TABLE_NAME_PARAM)); + assertTrue(query.containsKey(HoodieTableServiceManagerClient.BASEPATH_PARAM)); + assertTrue(query.containsKey(HoodieTableServiceManagerClient.INSTANT_PARAM)); + assertTrue(query.containsKey(HoodieTableServiceManagerClient.EXECUTION_ENGINE)); + assertTrue(query.containsKey(HoodieTableServiceManagerClient.PARALLELISM)); + } + + @Test + public void testExecuteCleanTargetsCleanEndpoint() throws IOException { + Map captured = new HashMap<>(); + String uri = startRecordingServer(captured); + HoodieTableServiceManagerClient client = + new HoodieTableServiceManagerClient(initMetaClient(), configFor(uri)); + + client.executeClean(); + assertEquals(HoodieTableServiceManagerClient.EXECUTE_CLEAN, captured.get("path")); + } + + @Test + public void testExecuteClusteringTargetsClusterEndpoint() throws IOException { + Map captured = new HashMap<>(); + String uri = startRecordingServer(captured); + HoodieTableServiceManagerClient client = + new HoodieTableServiceManagerClient(initMetaClient(), configFor(uri)); + + client.executeClustering(); + assertEquals(HoodieTableServiceManagerClient.EXECUTE_CLUSTERING, captured.get("path")); + } + + @Test + public void testServerErrorRetriesAndSurfacesRemoteException() throws IOException { + AtomicInteger requestCount = new AtomicInteger(0); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + requestCount.incrementAndGet(); + // Always fail so the retry limit is exhausted and an exception propagates. + byte[] body = "boom".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(500, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + }); + server.start(); + String uri = "http://127.0.0.1:" + server.getAddress().getPort(); + + HoodieTableServiceManagerClient client = + new HoodieTableServiceManagerClient(initMetaClient(), configFor(uri)); + + assertThrows(HoodieRemoteException.class, client::executeCompaction); + // Two retries configured means the request is attempted more than once. + assertTrue(requestCount.get() >= 2, + "expected at least 2 attempts, got " + requestCount.get()); + } + + @Test + public void testUnreachableServerSurfacesRemoteException() throws IOException { + // Port 1 is a privileged, unbound port: the connection will be refused. + HoodieTableServiceManagerClient client = + new HoodieTableServiceManagerClient(initMetaClient(), configFor("http://127.0.0.1:1")); + assertThrows(HoodieRemoteException.class, client::executeCompaction); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java new file mode 100644 index 0000000000000..fccbae2ecfa08 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.transaction.lock; + +import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.HoodieStorageUtils; +import org.apache.hudi.config.HoodieLockConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link FileSystemBasedLockProvider} against a local temp directory. + */ +public class TestFileSystemBasedLockProvider { + + @TempDir + Path tempDir; + + private LockConfiguration lockConfiguration(String lockPath, int expireMinutes) { + Properties props = new Properties(); + props.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, lockPath); + props.setProperty(FILESYSTEM_LOCK_EXPIRE_PROP_KEY, String.valueOf(expireMinutes)); + return new LockConfiguration(props); + } + + private String lockDir(String name) { + return tempDir.resolve(name).toString(); + } + + @Test + public void testAcquireAndReleaseLock() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + FileSystemBasedLockProvider provider = + new FileSystemBasedLockProvider(lockConfiguration(lockDir("acquire"), 0), storageConf); + try { + assertTrue(provider.tryLock(1, TimeUnit.SECONDS), "first acquisition should succeed"); + // getLock exposes the fully-qualified lock file path used on the backing storage. + assertTrue(provider.getLock().endsWith("/lock")); + provider.unlock(); + // After unlock the file is gone, so the lock is acquirable again. + assertTrue(provider.tryLock(1, TimeUnit.SECONDS), "re-acquisition after unlock should succeed"); + } finally { + provider.unlock(); + provider.close(); + } + } + + @Test + public void testConcurrentProvidersCannotBothHoldLock() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + LockConfiguration config = lockConfiguration(lockDir("contended"), 0); + FileSystemBasedLockProvider holder = new FileSystemBasedLockProvider(config, storageConf); + FileSystemBasedLockProvider contender = new FileSystemBasedLockProvider(config, storageConf); + try { + assertTrue(holder.tryLock(1, TimeUnit.SECONDS)); + // A non-expired lock owned by another provider blocks acquisition. + assertFalse(contender.tryLock(1, TimeUnit.SECONDS), "second provider must not acquire a held lock"); + // The contender is able to read the current owner's lock info. + assertTrue(contender.getCurrentOwnerLockInfo() != null && !contender.getCurrentOwnerLockInfo().isEmpty()); + holder.unlock(); + assertTrue(contender.tryLock(1, TimeUnit.SECONDS), "contender acquires after holder releases"); + } finally { + holder.unlock(); + contender.unlock(); + holder.close(); + } + } + + @Test + public void testExpiredLockIsReclaimed() throws Exception { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + String path = lockDir("expiry"); + // Expiry of 1 minute; we age the lock file deterministically rather than sleeping. + LockConfiguration config = lockConfiguration(path, 1); + FileSystemBasedLockProvider holder = new FileSystemBasedLockProvider(config, storageConf); + FileSystemBasedLockProvider contender = new FileSystemBasedLockProvider(config, storageConf); + try { + assertTrue(holder.tryLock(1, TimeUnit.SECONDS)); + // Not yet expired: contender is blocked. + assertFalse(contender.tryLock(1, TimeUnit.SECONDS)); + + // Age the lock file well beyond the 1-minute expiry window. + StoragePath lockFile = new StoragePath(path + StoragePath.SEPARATOR + "lock"); + HoodieStorage storage = HoodieStorageUtils.getStorage(lockFile.toString(), storageConf); + storage.setModificationTime(lockFile, System.currentTimeMillis() - (5 * 60 * 1000L)); + + // The expired lock file is deleted and the contender acquires it. + assertTrue(contender.tryLock(1, TimeUnit.SECONDS), "expired lock should be reclaimable"); + } finally { + holder.unlock(); + contender.unlock(); + contender.close(); + } + } + + @Test + public void testZeroExpiryNeverReclaims() throws Exception { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + String path = lockDir("noexpiry"); + // Expiry of 0 disables reclamation entirely. + LockConfiguration config = lockConfiguration(path, 0); + FileSystemBasedLockProvider holder = new FileSystemBasedLockProvider(config, storageConf); + FileSystemBasedLockProvider contender = new FileSystemBasedLockProvider(config, storageConf); + try { + assertTrue(holder.tryLock(1, TimeUnit.SECONDS)); + + // Even a very old lock file must not be reclaimed when expiry is disabled. + StoragePath lockFile = new StoragePath(path + StoragePath.SEPARATOR + "lock"); + HoodieStorage storage = HoodieStorageUtils.getStorage(lockFile.toString(), storageConf); + storage.setModificationTime(lockFile, System.currentTimeMillis() - (60 * 60 * 1000L)); + + assertFalse(contender.tryLock(1, TimeUnit.SECONDS), "zero expiry must never reclaim a lock"); + } finally { + holder.unlock(); + contender.unlock(); + holder.close(); + } + } + + @Test + public void testUnlockWithoutLockIsNoOp() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + FileSystemBasedLockProvider provider = + new FileSystemBasedLockProvider(lockConfiguration(lockDir("idempotent"), 0), storageConf); + // Releasing when no lock is held must not raise. + assertDoesNotThrow(provider::unlock); + provider.close(); + } + + @Test + public void testConstructorRejectsNegativeExpiry() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + LockConfiguration config = lockConfiguration(lockDir("bad"), -1); + assertThrows(IllegalArgumentException.class, + () -> new FileSystemBasedLockProvider(config, storageConf)); + } + + @Test + public void testGetLockConfigProducesUsableProperties() { + String tablePath = tempDir.resolve("table").toString(); + TypedProperties props = FileSystemBasedLockProvider.getLockConfig(tablePath); + // The generated config points the lock provider at the table's auxiliary folder. + assertTrue(props.getString(HoodieLockConfig.FILESYSTEM_LOCK_PATH.key()).startsWith(tablePath)); + assertEquals(FileSystemBasedLockProvider.class.getName(), + props.getString(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key())); + + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + FileSystemBasedLockProvider provider = + new FileSystemBasedLockProvider(new LockConfiguration(props), storageConf); + try { + assertTrue(provider.tryLock(1, TimeUnit.SECONDS)); + } finally { + provider.unlock(); + provider.close(); + } + } + + @Test + public void testLockPathDefaultsToMetafolderFromBasePath() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + Properties props = new Properties(); + props.setProperty(HoodieWriteConfig.BASE_PATH.key(), lockDir("defaultpath")); + props.setProperty(FILESYSTEM_LOCK_EXPIRE_PROP_KEY, "0"); + FileSystemBasedLockProvider provider = + new FileSystemBasedLockProvider(new LockConfiguration(props), storageConf); + try { + assertTrue(provider.tryLock(1, TimeUnit.SECONDS), + "lock acquisition must work without an explicit lock path"); + // Without an explicit lock path the provider locks under the table metafolder. + assertTrue(provider.getLock().endsWith( + HoodieTableMetaClient.METAFOLDER_NAME + StoragePath.SEPARATOR + "lock")); + } finally { + provider.unlock(); + provider.close(); + } + } + + @Test + public void testSameProviderSecondTryLockFails() { + StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf(); + FileSystemBasedLockProvider provider = + new FileSystemBasedLockProvider(lockConfiguration(lockDir("nonreentrant"), 0), storageConf); + try { + assertTrue(provider.tryLock(1, TimeUnit.SECONDS)); + // The file lock is not reentrant: a second tryLock by the same provider fails. + assertFalse(provider.tryLock(1, TimeUnit.SECONDS)); + } finally { + provider.unlock(); + provider.close(); + } + } +} diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestFileBasedLockProvider.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestFileBasedLockProvider.java deleted file mode 100644 index 0fcc9dadea18d..0000000000000 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestFileBasedLockProvider.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hudi.client; - -import org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider; -import org.apache.hudi.common.config.LockConfiguration; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.storage.StorageConfiguration; - -import org.apache.hadoop.conf.Configuration; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY; -import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY; -import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY; -import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY; -import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY; -import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestFileBasedLockProvider { - - @TempDir - Path tempDir; - String basePath; - LockConfiguration lockConfiguration; - StorageConfiguration storageConf; - - @BeforeEach - public void setUp() throws IOException { - basePath = tempDir.toUri().getPath(); - Properties properties = new Properties(); - properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath); - properties.setProperty(FILESYSTEM_LOCK_EXPIRE_PROP_KEY, "1"); - properties.setProperty(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "1000"); - properties.setProperty(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "1000"); - properties.setProperty(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY, "3"); - lockConfiguration = new LockConfiguration(properties); - storageConf = getDefaultStorageConf(); - } - - @Test - public void testAcquireLock() { - FileSystemBasedLockProvider fileBasedLockProvider = new FileSystemBasedLockProvider(lockConfiguration, storageConf); - assertTrue(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - fileBasedLockProvider.unlock(); - } - - @Test - public void testAcquireLockWithDefaultPath() { - lockConfiguration.getConfig().remove(FILESYSTEM_LOCK_PATH_PROP_KEY); - lockConfiguration.getConfig().setProperty(HoodieWriteConfig.BASE_PATH.key(), basePath); - FileSystemBasedLockProvider fileBasedLockProvider = new FileSystemBasedLockProvider(lockConfiguration, storageConf); - assertTrue(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - fileBasedLockProvider.unlock(); - lockConfiguration.getConfig().setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath); - } - - @Test - public void testUnLock() { - FileSystemBasedLockProvider fileBasedLockProvider = new FileSystemBasedLockProvider(lockConfiguration, storageConf); - assertTrue(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - fileBasedLockProvider.unlock(); - assertTrue(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - } - - @Test - public void testReentrantLock() { - FileSystemBasedLockProvider fileBasedLockProvider = new FileSystemBasedLockProvider(lockConfiguration, storageConf); - assertTrue(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - assertFalse(fileBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - fileBasedLockProvider.unlock(); - } - - @Test - public void testUnlockWithoutLock() { - assertDoesNotThrow(() -> { - FileSystemBasedLockProvider fileBasedLockProvider = new FileSystemBasedLockProvider(lockConfiguration, storageConf); - fileBasedLockProvider.unlock(); - }); - } -} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java index a2cef4558e1b6..7ffa5ef6099fa 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java @@ -95,7 +95,7 @@ public class HoodieTableServiceManagerConfig extends HoodieConfig { public static final ConfigProperty TABLE_SERVICE_MANAGER_DEPLOY_EXTRA_PARAMS = ConfigProperty .key(TABLE_SERVICE_MANAGER_PREFIX + ".deploy.extra.params") - .noDefaultValue() + .defaultValue("") .markAdvanced() .sinceVersion("0.13.0") .withDocumentation("The extra params to deploy for table service of this table, split by ';'"); From 128e962abc57e6d06ad26aa5bc4cea26c1db7dec Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Mon, 20 Jul 2026 18:16:21 -0700 Subject: [PATCH 173/255] fix(hive-sync): close proxied IMetaStoreClient in HoodieHiveSyncClient.close() to prevent HMS connection leak (#19331) (cherry picked from commit afec0c9adcce1e67dea0a9fb475834bb6a1bdec9) --- .../hudi/hive/HoodieHiveSyncClient.java | 12 +++ .../hive/TestHoodieHiveSyncClientClose.java | 78 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHoodieHiveSyncClientClose.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java index 9ad9070a465ef..3de4077fd8dab 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java @@ -598,6 +598,18 @@ public void close() { try { ddlExecutor.close(); if (client != null) { + // Close the proxied IMetaStoreClient directly before Hive.closeCurrent(). + // When RetryingMetaStoreClient rebuilds the underlying client on a transient + // TException, the fresh MSC is reachable only through this proxy, while the + // thread-local Hive singleton still references the older instance. So + // Hive.closeCurrent() alone closes the stale MSC and orphans the retry-created + // one, leaking a connection per sync cycle. client.close() releases the live + // MSC by identity; Hive.closeCurrent() remains a fallback for the singleton path. + try { + client.close(); + } catch (Exception e) { + log.warn("Failed to close IMetaStoreClient directly; Hive.closeCurrent() will run anyway", e); + } Hive.closeCurrent(); client = null; } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHoodieHiveSyncClientClose.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHoodieHiveSyncClientClose.java new file mode 100644 index 0000000000000..53c725dc6a3b5 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHoodieHiveSyncClientClose.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive; + +import org.apache.hudi.hive.ddl.DDLExecutor; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * Unit tests for {@link HoodieHiveSyncClient#close()} connection cleanup. + * + *

    Regression coverage for the metastore connection leak: when + * {@code RetryingMetaStoreClient} rebuilds the underlying client on a transient + * error, {@code Hive.closeCurrent()} alone closes the stale singleton-bound + * client and orphans the retry-created one. {@code close()} must therefore + * release the client held on the proxy field directly. + */ +class TestHoodieHiveSyncClientClose { + + @Test + void closeReleasesProxiedMetastoreClientDirectly() throws Exception { + HoodieHiveSyncClient syncClient = mock(HoodieHiveSyncClient.class, CALLS_REAL_METHODS); + IMetaStoreClient metaStoreClient = mock(IMetaStoreClient.class); + DDLExecutor ddlExecutor = mock(DDLExecutor.class); + setField(syncClient, "client", metaStoreClient); + setField(syncClient, "ddlExecutor", ddlExecutor); + + syncClient.close(); + + verify(metaStoreClient).close(); + verify(ddlExecutor).close(); + } + + @Test + void closeSwallowsProxyCloseFailure() throws Exception { + HoodieHiveSyncClient syncClient = mock(HoodieHiveSyncClient.class, CALLS_REAL_METHODS); + IMetaStoreClient metaStoreClient = mock(IMetaStoreClient.class); + DDLExecutor ddlExecutor = mock(DDLExecutor.class); + doThrow(new RuntimeException("transient close failure")).when(metaStoreClient).close(); + setField(syncClient, "client", metaStoreClient); + setField(syncClient, "ddlExecutor", ddlExecutor); + + // A transient failure closing the proxied client must not propagate. + assertDoesNotThrow(syncClient::close); + verify(metaStoreClient).close(); + } + + private static void setField(Object target, String name, Object value) throws Exception { + Field field = HoodieHiveSyncClient.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} From a3bdfcc52a746f2fed83bb272aded96beed3107a Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Tue, 21 Jul 2026 09:20:16 +0800 Subject: [PATCH 174/255] fix(flink): avoid reusing split reader functions across fetchers (#19315) (cherry picked from commit d78a7b5baa42eb5cc4d493ed2101e0af35a441e2) --- .../org/apache/hudi/source/HoodieSource.java | 12 +-- .../source/reader/HoodieSourceReader.java | 5 +- .../reader/HoodieSourceSplitReader.java | 7 +- .../apache/hudi/table/HoodieTableSource.java | 27 +++---- .../apache/hudi/source/TestHoodieSource.java | 18 ++--- .../reader/TestHoodieSourceSplitReader.java | 80 ++++++++++++------- 6 files changed, 87 insertions(+), 62 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/HoodieSource.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/HoodieSource.java index 24357fbe21e7a..45dec15844e7f 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/HoodieSource.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/HoodieSource.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; +import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; @@ -75,7 +76,7 @@ public class HoodieSource extends FileIndexReader implements Source readerFunction; + private final SerializableSupplier> readerFunctionSupplier; private final SerializableComparator splitComparator; private final HoodieTableMetaClient metaClient; private final HoodieRecordEmitter recordEmitter; @@ -83,18 +84,18 @@ public class HoodieSource extends FileIndexReader implements Source readerFunction, + SerializableSupplier> readerFunctionSupplier, SerializableComparator splitComparator, HoodieTableMetaClient metaClient, HoodieRecordEmitter recordEmitter) { ValidationUtils.checkArgument(scanContext != null, "scanContext can't be null."); - ValidationUtils.checkArgument(readerFunction != null, "readerFunction can't be null."); + ValidationUtils.checkArgument(readerFunctionSupplier != null, "readerFunctionSupplier can't be null."); ValidationUtils.checkArgument(splitComparator != null, "splitComparator can't be null."); ValidationUtils.checkArgument(metaClient != null, "metaClient can't be null."); ValidationUtils.checkArgument(recordEmitter != null, "recordEmitter can't be null."); this.scanContext = scanContext; - this.readerFunction = readerFunction; + this.readerFunctionSupplier = readerFunctionSupplier; this.splitComparator = splitComparator; this.metaClient = metaClient; this.recordEmitter = recordEmitter; @@ -129,7 +130,8 @@ public SimpleVersionedSerializer getEnumeratorCheckp @Override public SourceReader createReader(SourceReaderContext readerContext) throws Exception { - return new HoodieSourceReader(tableName, recordEmitter, scanContext, readerContext, readerFunction, splitComparator); + return new HoodieSourceReader( + tableName, recordEmitter, scanContext, readerContext, readerFunctionSupplier, splitComparator); } private SplitEnumerator createEnumerator( diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java index d1252823aeda7..70e232295c1d1 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java @@ -22,6 +22,7 @@ import org.apache.flink.connector.base.source.reader.RecordEmitter; import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; +import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.util.Option; import org.apache.hudi.source.HoodieScanContext; import org.apache.hudi.source.reader.function.SplitReaderFunction; @@ -45,9 +46,9 @@ public HoodieSourceReader( RecordEmitter, T, HoodieSourceSplit> recordEmitter, HoodieScanContext scanContext, SourceReaderContext context, - SplitReaderFunction readerFunction, + SerializableSupplier> readerFunctionSupplier, SerializableComparator splitComparator) { - super(() -> new HoodieSourceSplitReader<>(tableName, context, readerFunction, splitComparator, + super(() -> new HoodieSourceSplitReader<>(tableName, context, readerFunctionSupplier, splitComparator, scanContext.getLimit() == RecordLimiter.NO_LIMIT ? Option.empty() : Option.of(new RecordLimiter(scanContext.getLimit()))), recordEmitter, scanContext.getConf(), context); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java index 52cc54db017af..47379789af813 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java @@ -18,6 +18,7 @@ package org.apache.hudi.source.reader; +import org.apache.hudi.common.function.SerializableSupplier; import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.connector.base.source.reader.RecordsBySplits; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; @@ -76,12 +77,14 @@ public class HoodieSourceSplitReader implements SplitReader readerFunction, + SerializableSupplier> readerFunctionSupplier, SerializableComparator splitComparator, Option recordLimiter) { this.splitComparator = splitComparator; this.splits = new ArrayDeque<>(); - this.readerFunction = readerFunction; + // Flink can start a new fetcher before the previous idle fetcher finishes closing. Supply a + // fresh stateful cursor for each split reader so the old fetcher's close cannot affect the new one. + this.readerFunction = readerFunctionSupplier.get(); this.recordLimiter = recordLimiter; this.readerMetrics = new FlinkStreamReadMetrics(context.metricGroup(), tableName); this.readerMetrics.registerMetrics(); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java index 5167c1f623421..30b41e34f8ac3 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java @@ -20,6 +20,7 @@ import org.apache.hudi.adapter.DataStreamScanProviderAdapter; import org.apache.hudi.adapter.InputFormatSourceFunctionAdapter; +import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieTableType; @@ -303,7 +304,7 @@ private HoodieSource createHoodieSource() { HoodieScanContext context = createHoodieScanContext(rowType); final HoodieTableType tableType = HoodieTableType.valueOf(this.conf.get(FlinkOptions.TABLE_TYPE)); - final SplitReaderFunction splitReaderFunction; + final SerializableSupplier> splitReaderFunctionSupplier; final MergeOnReadTableState hoodieTableState = new MergeOnReadTableState<>( rowType, requiredRowType, @@ -313,24 +314,16 @@ private HoodieSource createHoodieSource() { boolean emitDelete = tableType == HoodieTableType.MERGE_ON_READ && context.isStreaming(); if (conf.get(FlinkOptions.CDC_ENABLED)) { List fieldTypes = rowDataType.getChildren(); - splitReaderFunction = new HoodieCdcSplitReaderFunction( - conf, - hoodieTableState, - internalSchemaManager, - fieldTypes, - predicates, - emitDelete); + splitReaderFunctionSupplier = () -> new HoodieCdcSplitReaderFunction( + conf, hoodieTableState, internalSchemaManager, fieldTypes, predicates, emitDelete); } else { - splitReaderFunction = new HoodieSplitReaderFunction( - conf, - tableSchema, - HoodieSchemaConverter.convertToSchema(requiredRowType), - internalSchemaManager, - conf.get(FlinkOptions.MERGE_TYPE), - predicates, - emitDelete); + splitReaderFunctionSupplier = () -> new HoodieSplitReaderFunction( + conf, tableSchema, HoodieSchemaConverter.convertToSchema(requiredRowType), internalSchemaManager, + conf.get(FlinkOptions.MERGE_TYPE), predicates, emitDelete); } - return new HoodieSource<>(context, splitReaderFunction, new HoodieSourceSplitComparator(), metaClient, new HoodieRecordEmitter<>()); + return new HoodieSource<>( + context, splitReaderFunctionSupplier, new HoodieSourceSplitComparator(), metaClient, + new HoodieRecordEmitter<>()); } /** diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestHoodieSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestHoodieSource.java index 2a8956e20f7e8..516d5e7c2da55 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestHoodieSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestHoodieSource.java @@ -413,18 +413,18 @@ private HoodieSource createHoodieSourceWithPruner( .build(); HoodieSchema schema = HoodieSchemaConverter.convertToSchema(rowType); HadoopStorageConfiguration hadoopConf = new HadoopStorageConfiguration(HadoopConfigurations.getHadoopConf(conf)); - HoodieSplitReaderFunction splitReaderFunction = new HoodieSplitReaderFunction( - conf, - schema, // schema will be resolved from table - schema, // required schema - InternalSchemaManager.get(hadoopConf, this.metaClient), - conf.get(FlinkOptions.MERGE_TYPE), - Collections.emptyList(), - false); + InternalSchemaManager internalSchemaManager = InternalSchemaManager.get(hadoopConf, this.metaClient); return new HoodieSource<>( scanContext, - splitReaderFunction, + () -> new HoodieSplitReaderFunction( + conf, + schema, // schema will be resolved from table + schema, // required schema + internalSchemaManager, + conf.get(FlinkOptions.MERGE_TYPE), + Collections.emptyList(), + false), new HoodieSourceSplitComparator(), metaClient, new HoodieRecordEmitter<>()); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java index 2ea14b0ac0060..285d924b94521 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java @@ -20,6 +20,7 @@ import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; +import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.util.Option; import org.apache.hudi.source.reader.function.SplitReaderFunction; import org.apache.hudi.source.split.HoodieSourceSplit; @@ -45,6 +46,7 @@ import java.util.stream.IntStream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -68,7 +70,7 @@ public void setUp() { public void testFetchWithNoSplits() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); RecordsWithSplitIds> result = reader.fetch(); @@ -82,7 +84,7 @@ public void testFetchWithSingleSplit() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); SplitsAddition splitsChange = new SplitsAddition<>(Collections.singletonList(split)); @@ -100,7 +102,7 @@ public void testFetchWithMultipleSplits() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -128,7 +130,7 @@ public void testHandleSplitsChangesWithComparator() throws IOException { (s1, s2) -> s2.getFileId().compareTo(s1.getFileId()); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, comparator, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, comparator, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -151,7 +153,7 @@ public void testAddingSplitsInMultipleBatches() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); // First batch HoodieSourceSplit split1 = createTestSplit(1, "file1"); @@ -173,7 +175,7 @@ public void testAddingSplitsInMultipleBatches() throws IOException { public void testClose() throws Exception { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -192,7 +194,7 @@ public void testClose() throws Exception { public void testWakeUp() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); // wakeUp() now sets a flag but must not throw, and the flag is reset at the top of each fetch() // so a wakeUp with no in-flight drain leaves the next fetch() unaffected. @@ -214,7 +216,7 @@ public void testWakeUpMidDrainReturnsPartialBatchAndResumes() throws IOException List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -261,7 +263,7 @@ public void testWakeUpBeforeAnyRecordReturnsEmptyNonFinishingBatch() throws IOEx List testData = Arrays.asList("r1", "r2", "r3"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { // Wake at the start of the drain (count 0) while data is still available. @@ -293,7 +295,7 @@ public void testWakeUpCoincidingWithEofDefersFinishByOneFetch() throws IOExcepti List testData = Arrays.asList("r1", "r2"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { // Wake only at the start of a drain that finds the cursor already exhausted. @@ -341,7 +343,7 @@ public boolean isLimitReached() { } }; HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(wakingLimiter)); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(wakingLimiter)); holder.set(reader); HoodieSourceSplit split = createTestSplit(1, "file1"); @@ -365,7 +367,7 @@ public void testWakeUpPartialBatchRespectsLimit() throws IOException { List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(3L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -398,7 +400,7 @@ public void testCloseReleasesWokenStillOpenSplit() throws Exception { List testData = Arrays.asList("r1", "r2", "r3"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -424,7 +426,7 @@ public void testCloseReleasesWokenStillOpenSplit() throws Exception { public void testPauseOrResumeSplits() { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -442,7 +444,7 @@ public void testReaderFunctionCalledCorrectly() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -457,18 +459,42 @@ public void testReaderFunctionCalledCorrectly() throws IOException { public void testReaderFunctionClosedOnReaderClose() throws Exception { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); reader.close(); assertTrue(readerFunction.isClosed(), "Reader function should be closed"); } + @Test + public void testEachSplitReaderUsesIndependentReaderFunction() throws Exception { + List readerFunctions = new ArrayList<>(); + SerializableSupplier> readerFunctionSupplier = () -> { + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); + readerFunctions.add(readerFunction); + return readerFunction; + }; + + HoodieSourceSplitReader firstReader = new HoodieSourceSplitReader<>( + TABLE_NAME, readerContext, readerFunctionSupplier, null, Option.empty()); + HoodieSourceSplitReader secondReader = new HoodieSourceSplitReader<>( + TABLE_NAME, readerContext, readerFunctionSupplier, null, Option.empty()); + + assertEquals(2, readerFunctions.size()); + firstReader.close(); + assertTrue(readerFunctions.get(0).isClosed()); + assertFalse(readerFunctions.get(1).isClosed(), + "Closing an idle fetcher's split reader must not close the next fetcher's reader function"); + + secondReader.close(); + assertTrue(readerFunctions.get(1).isClosed()); + } + @Test public void testFetchEmptyResultWhenNoSplitsAdded() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); RecordsWithSplitIds> result = reader.fetch(); @@ -484,7 +510,7 @@ public void testSplitOrderPreservedWithoutComparator() throws IOException { // No comparator - should preserve insertion order HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split3 = createTestSplit(3, "file3"); HoodieSourceSplit split1 = createTestSplit(1, "file1"); @@ -506,7 +532,7 @@ public void testReaderIteratorClosedOnSplitFinish() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -528,7 +554,7 @@ public void testLimitCapsRecordsFromSingleSplit() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(2L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -556,7 +582,7 @@ public void testLimitDrainsRemainingSplitsWhenLimitReached() throws IOException TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(2L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -584,7 +610,7 @@ public void testLimitZeroDrainsAllSplitsImmediately() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(Arrays.asList("r1", "r2")); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(0L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(0L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -607,7 +633,7 @@ public void testLimitExactlyMatchingTotalRecords() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(3L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -633,7 +659,7 @@ public void testLimitSpanningMultipleSplits() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.of(new RecordLimiter(5L))); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(5L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -666,7 +692,7 @@ public void testNoLimitSentinelReturnsAllRecords() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); @@ -690,7 +716,7 @@ public void testSplitSpanningMultipleMinibatches() throws IOException { List testData = IntStream.range(0, n).mapToObj(i -> "r" + i).collect(Collectors.toList()); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -733,7 +759,7 @@ public void testResumeSkipsConsumedRecords() throws IOException { List testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); split.updatePosition(0, 2L); // 2 records already consumed before recovery From 6ff2463de8de789dc2e9b76eb9fb0ad410ca4ccb Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 13:35:25 +0800 Subject: [PATCH 175/255] fix(build): import HoodieStorageUtils from its release-branch package TestFileSystemBasedLockProvider, added by 688c5c24448d (#19222), imports org.apache.hudi.common.util.HoodieStorageUtils. That package location comes from d291efccaad2 (#19195), the hudi-common core/common package reorganization, which is not on release-1.2.1. Here the class is still at org.apache.hudi.storage.HoodieStorageUtils, so hudi-client-common failed to test-compile with "cannot find symbol: class HoodieStorageUtils". Point the import at the release-branch package and group it with the other org.apache.hudi.storage imports. --- .../transaction/lock/TestFileSystemBasedLockProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java index fccbae2ecfa08..f7ea8a3cac70c 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestFileSystemBasedLockProvider.java @@ -22,10 +22,10 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieTestUtils; -import org.apache.hudi.common.util.HoodieStorageUtils; import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.HoodieStorageUtils; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; From 0d279d70322c389068cb48484f627887860ec57e Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Tue, 21 Jul 2026 01:16:28 -0700 Subject: [PATCH 176/255] test(common): add unit coverage for metrics reporters and schema utilities (#19221) Adds unit tests for three under-covered hudi-common areas: - TestValueType: per-type conversion, cast, and round-trip coverage across numeric, string, bytes, decimal, UUID, date, time, and timestamp helpers, plus fromParquetPrimitiveType, fromSchema, and fromOrdinal. - TestHoodieSchemaTypePromotion: promotion matrix (widening, narrowing, string<->bytes, unrelated types) and decimal widening rules. - TestM3ScopeReporterAdaptor: registry-to-scope mapping for counters, gauges, histograms, meters, and timers using a mocked Scope. Co-authored-by: sivabalan (cherry picked from commit b90627ae393dd3d9b4b846ee85beb96c3aa8d5a8) --- .../schema/TestHoodieSchemaTypePromotion.java | 138 +++++++++++ .../m3/TestM3ScopeReporterAdaptor.java | 163 +++++++++++++ .../org/apache/hudi/stats/TestValueType.java | 229 ++++++++++++++++++ 3 files changed, 530 insertions(+) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaTypePromotion.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/metrics/m3/TestM3ScopeReporterAdaptor.java diff --git a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaTypePromotion.java b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaTypePromotion.java new file mode 100644 index 0000000000000..e6215b9a19aa2 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaTypePromotion.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.schema; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for the internal promotion matrix used by schema compatibility checks. + */ +public class TestHoodieSchemaTypePromotion { + + @Test + public void testSameTypeAlwaysPromotable() { + for (HoodieSchemaType type : HoodieSchemaType.values()) { + assertTrue(HoodieSchemaTypePromotion.canPromote(type, type), + "same type should promote to itself: " + type); + } + } + + @Test + public void testIntWidening() { + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.LONG, HoodieSchemaType.INT)); + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.FLOAT, HoodieSchemaType.INT)); + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.DOUBLE, HoodieSchemaType.INT)); + } + + @Test + public void testLongWidening() { + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.FLOAT, HoodieSchemaType.LONG)); + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.DOUBLE, HoodieSchemaType.LONG)); + // LONG cannot read a wider reader-narrows case + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.LONG, HoodieSchemaType.FLOAT)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.LONG, HoodieSchemaType.DOUBLE)); + } + + @Test + public void testFloatWidening() { + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.DOUBLE, HoodieSchemaType.FLOAT)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.FLOAT, HoodieSchemaType.DOUBLE)); + } + + @Test + public void testNarrowingNotAllowed() { + // reader cannot narrow: long data cannot be read by an int reader, etc. + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.INT, HoodieSchemaType.LONG)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.INT, HoodieSchemaType.FLOAT)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.INT, HoodieSchemaType.DOUBLE)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.LONG, HoodieSchemaType.DOUBLE)); + // FLOAT reader CAN read LONG writer (Avro promotion allows this despite the mantissa + // precision loss). Asserted here as canonical to guard against regressions. + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.FLOAT, HoodieSchemaType.LONG)); + } + + @Test + public void testStringBytesBidirectional() { + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.STRING, HoodieSchemaType.BYTES)); + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.BYTES, HoodieSchemaType.STRING)); + // STRING can also read numeric writer types + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.STRING, HoodieSchemaType.INT)); + assertTrue(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.STRING, HoodieSchemaType.DOUBLE)); + // BYTES cannot read numeric types + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.BYTES, HoodieSchemaType.INT)); + } + + @Test + public void testUnrelatedTypesNotPromotable() { + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.BOOLEAN, HoodieSchemaType.INT)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.INT, HoodieSchemaType.BOOLEAN)); + assertFalse(HoodieSchemaTypePromotion.canPromote(HoodieSchemaType.LONG, HoodieSchemaType.STRING)); + } + + @Test + public void testDecimalWideningSameSizeIncreasedPrecision() { + HoodieSchema writer = fixedDecimal(8, 10, 2); + HoodieSchema reader = fixedDecimal(8, 15, 2); + assertTrue(HoodieSchemaTypePromotion.isDecimalWidening(reader, writer)); + } + + @Test + public void testDecimalWideningIdenticalIsAllowed() { + HoodieSchema writer = fixedDecimal(8, 10, 2); + HoodieSchema reader = fixedDecimal(8, 10, 2); + assertTrue(HoodieSchemaTypePromotion.isDecimalWidening(reader, writer)); + } + + @Test + public void testDecimalWideningRejectsDecreasedPrecision() { + HoodieSchema writer = fixedDecimal(8, 15, 2); + HoodieSchema reader = fixedDecimal(8, 10, 2); + assertFalse(HoodieSchemaTypePromotion.isDecimalWidening(reader, writer)); + } + + @Test + public void testDecimalWideningRejectsIncreasedScaleWithoutRoom() { + // integer digits shrink from 8 to 5, so widening is invalid + HoodieSchema writer = fixedDecimal(8, 10, 2); + HoodieSchema reader = fixedDecimal(8, 10, 5); + assertFalse(HoodieSchemaTypePromotion.isDecimalWidening(reader, writer)); + } + + @Test + public void testDecimalWideningRejectsDifferentFixedSize() { + HoodieSchema writer = fixedDecimal(8, 10, 2); + HoodieSchema reader = fixedDecimal(16, 10, 2); + assertFalse(HoodieSchemaTypePromotion.isDecimalWidening(reader, writer)); + } + + @Test + public void testDecimalWideningRejectsNonDecimal() { + HoodieSchema decimal = fixedDecimal(8, 10, 2); + HoodieSchema plainInt = HoodieSchema.create(HoodieSchemaType.INT); + assertFalse(HoodieSchemaTypePromotion.isDecimalWidening(decimal, plainInt)); + assertFalse(HoodieSchemaTypePromotion.isDecimalWidening(plainInt, decimal)); + } + + private static HoodieSchema fixedDecimal(int size, int precision, int scale) { + return HoodieSchema.createDecimal("FixedDecimal", null, null, precision, scale, size); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/metrics/m3/TestM3ScopeReporterAdaptor.java b/hudi-common/src/test/java/org/apache/hudi/metrics/m3/TestM3ScopeReporterAdaptor.java new file mode 100644 index 0000000000000..a445e82a7f756 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/metrics/m3/TestM3ScopeReporterAdaptor.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics.m3; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.codahale.metrics.UniformReservoir; +import com.uber.m3.tally.Gauge; +import com.uber.m3.tally.Scope; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Verifies that {@link M3ScopeReporterAdaptor} maps codahale registry metrics + * onto the target m3 tally {@link Scope}. The scope is a mock so that the exact + * counters and gauges reaching it can be asserted without any network I/O. + */ +public class TestM3ScopeReporterAdaptor { + + private MetricRegistry registry; + private Scope scope; + private com.uber.m3.tally.Counter scopeCounter; + private Gauge scopeGauge; + private M3ScopeReporterAdaptor reporter; + + @BeforeEach + public void setUp() { + registry = new MetricRegistry(); + scope = mock(Scope.class); + scopeCounter = mock(com.uber.m3.tally.Counter.class); + scopeGauge = mock(Gauge.class); + when(scope.counter(org.mockito.ArgumentMatchers.anyString())).thenReturn(scopeCounter); + when(scope.gauge(org.mockito.ArgumentMatchers.anyString())).thenReturn(scopeGauge); + reporter = new M3ScopeReporterAdaptor(registry, scope); + } + + @Test + public void testCounterIsForwardedToScope() { + Counter counter = registry.counter("requests"); + counter.inc(7); + + reporter.report(); + + verify(scope).counter("requests"); + verify(scopeCounter).inc(7L); + } + + @Test + public void testGaugeValueIsForwardedToScope() { + registry.register("in_flight", (com.codahale.metrics.Gauge) () -> 42); + + reporter.report(); + + verify(scope).gauge("in_flight"); + verify(scopeGauge).update(42.0d); + } + + @Test + public void testHistogramEmitsCountAndSnapshotGauges() { + Histogram histogram = new Histogram(new UniformReservoir()); + registry.register("latency", histogram); + histogram.update(10); + histogram.update(20); + + reporter.report(); + + // count is emitted as its own gauge suffix + verify(scope).gauge("latency.count"); + // the snapshot expands into ten percentile/stat gauges + verify(scope).gauge("latency.max"); + verify(scope).gauge("latency.mean"); + verify(scope).gauge("latency.min"); + verify(scope).gauge("latency.stddev"); + verify(scope).gauge("latency.p50"); + verify(scope).gauge("latency.p75"); + verify(scope).gauge("latency.p95"); + verify(scope).gauge("latency.p98"); + verify(scope).gauge("latency.p99"); + verify(scope).gauge("latency.p999"); + // no bare "latency" gauge, only the suffixed ones + verify(scope, never()).gauge("latency"); + } + + @Test + public void testMeterEmitsCountCounterAndRateGauges() { + Meter meter = registry.meter("throughput"); + meter.mark(5); + + reporter.report(); + + verify(scope).counter("throughput.count"); + verify(scopeCounter).inc(5L); + verify(scope).gauge("throughput.m1_rate"); + verify(scope).gauge("throughput.m5_rate"); + verify(scope).gauge("throughput.m15_rate"); + verify(scope).gauge("throughput.mean_rate"); + } + + @Test + public void testTimerEmitsBothMeteredAndSnapshotMetrics() { + Timer timer = registry.timer("op_time"); + timer.update(java.time.Duration.ofMillis(3)); + timer.update(java.time.Duration.ofMillis(9)); + + reporter.report(); + + // timer reports the metered count as a counter + verify(scope).counter("op_time.count"); + // and the four rate gauges from the metered portion + verify(scope).gauge("op_time.m1_rate"); + verify(scope).gauge("op_time.mean_rate"); + // plus the full snapshot set from the timer's histogram + verify(scope).gauge("op_time.max"); + verify(scope).gauge("op_time.p99"); + } + + @Test + public void testEmptyRegistryTouchesNothing() { + reporter.report(); + + verify(scope, never()).counter(org.mockito.ArgumentMatchers.anyString()); + verify(scope, never()).gauge(org.mockito.ArgumentMatchers.anyString()); + } + + @Test + public void testMultipleGaugesEachMapped() { + registry.register("g1", (com.codahale.metrics.Gauge) () -> 1L); + registry.register("g2", (com.codahale.metrics.Gauge) () -> 2L); + + reporter.report(); + + verify(scope).gauge("g1"); + verify(scope).gauge("g2"); + verify(scopeGauge).update(1.0d); + verify(scopeGauge).update(2.0d); + verify(scope, times(2)).gauge(org.mockito.ArgumentMatchers.startsWith("g")); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/stats/TestValueType.java b/hudi-common/src/test/java/org/apache/hudi/stats/TestValueType.java index d49664525583a..8c2ce71f657ba 100644 --- a/hudi-common/src/test/java/org/apache/hudi/stats/TestValueType.java +++ b/hudi-common/src/test/java/org/apache/hudi/stats/TestValueType.java @@ -19,9 +19,31 @@ package org.apache.hudi.stats; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaType; + +import org.apache.avro.generic.GenericData; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.UUID; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestValueType { @@ -54,4 +76,211 @@ public void testValueTypeNumbering() { // IN THE FUTURE assertEquals(21, ValueType.values().length); } + + @Test + public void testFromOrdinalRoundTrips() { + for (ValueType type : ValueType.values()) { + assertSame(type, ValueType.fromOrdinal(type.ordinal())); + } + } + + private static Comparable standardize(ValueType type, Object val) { + return type.standardizeJavaTypeAndPromote(val, ValueMetadata.NULL_METADATA); + } + + @Test + public void testCastToInteger() { + assertNull(standardize(ValueType.INT, null)); + assertEquals(7, standardize(ValueType.INT, 7)); + assertEquals(1, standardize(ValueType.INT, Boolean.TRUE)); + assertEquals(0, standardize(ValueType.INT, Boolean.FALSE)); + // best effort parse from a string representation + assertEquals(42, standardize(ValueType.INT, "42")); + } + + @Test + public void testCastToLong() { + assertEquals(5L, standardize(ValueType.LONG, 5)); + assertEquals(9L, standardize(ValueType.LONG, 9L)); + assertEquals(1L, standardize(ValueType.LONG, Boolean.TRUE)); + assertEquals(123L, standardize(ValueType.LONG, "123")); + } + + @Test + public void testCastToFloatAndDouble() { + assertEquals(3.0f, standardize(ValueType.FLOAT, 3)); + assertEquals(4.0f, standardize(ValueType.FLOAT, 4L)); + assertEquals(2.5f, standardize(ValueType.FLOAT, 2.5f)); + assertEquals(1.0f, standardize(ValueType.FLOAT, Boolean.TRUE)); + assertEquals(6.0d, standardize(ValueType.DOUBLE, 6)); + assertEquals(7.0d, standardize(ValueType.DOUBLE, 7L)); + assertEquals(0.0d, standardize(ValueType.DOUBLE, Boolean.FALSE)); + assertEquals(8.25d, standardize(ValueType.DOUBLE, 8.25d)); + } + + @Test + public void testCastToBoolean() { + assertEquals(Boolean.TRUE, ValueType.BOOLEAN.standardizeJavaTypeAndPromote(true, ValueMetadata.NULL_METADATA)); + assertThrows(UnsupportedOperationException.class, + () -> ValueType.BOOLEAN.standardizeJavaTypeAndPromote("nope", ValueMetadata.NULL_METADATA)); + } + + @Test + public void testCastToString() { + assertEquals("abc", ValueType.castToString("abc")); + assertEquals("11", ValueType.castToString(11)); + assertEquals("true", ValueType.castToString(Boolean.TRUE)); + assertEquals("bin", ValueType.castToString(Binary.fromString("bin"))); + assertThrows(UnsupportedOperationException.class, () -> ValueType.castToString(new Object())); + } + + @Test + public void testCastToBytesFromVariousSources() { + byte[] raw = "hello".getBytes(StandardCharsets.UTF_8); + assertEquals(ByteBuffer.wrap(raw), ValueType.castToBytes(ByteBuffer.wrap(raw))); + assertEquals(ByteBuffer.wrap(raw), ValueType.castToBytes(raw)); + assertEquals(ByteBuffer.wrap(raw), ValueType.castToBytes(Binary.fromConstantByteArray(raw))); + assertEquals(ByteBuffer.wrap(raw), ValueType.castToBytes("hello")); + assertThrows(UnsupportedOperationException.class, () -> ValueType.castToBytes(new Object())); + } + + @Test + public void testCastToFixedRejectsString() { + byte[] raw = {1, 2, 3}; + assertEquals(ByteBuffer.wrap(raw), ValueType.castToFixed(raw)); + // castToFixed, unlike castToBytes, does not accept String + assertThrows(UnsupportedOperationException.class, () -> ValueType.castToFixed("abc")); + } + + @Test + public void testDecimalRoundTrip() { + ValueMetadata.DecimalMetadata meta = ValueMetadata.DecimalMetadata.create(10, 2); + BigDecimal value = new BigDecimal("12.34"); + // fromDecimal produces the primitive representation, toDecimal reverses it + ByteBuffer primitive = ValueType.fromDecimal(value, meta); + BigDecimal roundTripped = ValueType.toDecimal(primitive, meta); + assertEquals(value, roundTripped); + // castToDecimal accepts an already-typed BigDecimal unchanged + assertEquals(value, ValueType.castToDecimal(value, meta)); + // integer input scaled by the metadata scale + assertEquals(new BigDecimal("1.00"), ValueType.castToDecimal(100, meta)); + assertThrows(UnsupportedOperationException.class, () -> ValueType.castToDecimal(new Object(), meta)); + } + + @Test + public void testUuidRoundTrip() { + UUID uuid = UUID.fromString("12345678-1234-1234-1234-1234567890ab"); + assertEquals(uuid, ValueType.castToUUID(uuid, ValueMetadata.NULL_METADATA)); + assertEquals(uuid, ValueType.castToUUID(uuid.toString(), ValueMetadata.NULL_METADATA)); + String primitive = ValueType.fromUUID(uuid, ValueMetadata.NULL_METADATA); + assertEquals(uuid, ValueType.toUUID(primitive, ValueMetadata.NULL_METADATA)); + assertThrows(UnsupportedOperationException.class, + () -> ValueType.castToUUID(1, ValueMetadata.NULL_METADATA)); + } + + @Test + public void testDateRoundTrip() { + LocalDate date = LocalDate.of(2020, 1, 2); + int epochDay = (int) date.toEpochDay(); + assertEquals(date, ValueType.castToDate(date, ValueMetadata.NULL_METADATA)); + assertEquals(date, ValueType.castToDate(epochDay, ValueMetadata.NULL_METADATA)); + assertEquals(date, ValueType.castToDate(java.sql.Date.valueOf(date), ValueMetadata.NULL_METADATA)); + assertEquals(epochDay, ValueType.fromDate(date, ValueMetadata.NULL_METADATA)); + assertEquals(date, ValueType.toDate(epochDay, ValueMetadata.NULL_METADATA)); + assertThrows(UnsupportedOperationException.class, + () -> ValueType.castToDate("2020-01-02", ValueMetadata.NULL_METADATA)); + } + + @Test + public void testTimeMillisAndMicrosRoundTrip() { + LocalTime time = LocalTime.of(1, 2, 3, 4_000_000); + int millisOfDay = time.toSecondOfDay() * 1000 + time.getNano() / 1_000_000; + assertEquals(time, ValueType.castToTimeMillis(time, ValueMetadata.NULL_METADATA)); + assertEquals(time, ValueType.castToTimeMillis(millisOfDay, ValueMetadata.NULL_METADATA)); + assertEquals(millisOfDay, ValueType.fromTimeMillis(time, ValueMetadata.NULL_METADATA)); + assertEquals(time, ValueType.toTimeMillis(millisOfDay, ValueMetadata.NULL_METADATA)); + + LocalTime microTime = LocalTime.of(4, 5, 6, 7_000); + long microsOfDay = microTime.toSecondOfDay() * 1_000_000L + microTime.getNano() / 1_000; + assertEquals(microTime, ValueType.castToTimeMicros(microsOfDay, ValueMetadata.NULL_METADATA)); + assertEquals(microsOfDay, ValueType.fromTimeMicros(microTime, ValueMetadata.NULL_METADATA)); + } + + @Test + public void testTimestampMillisMicrosNanosRoundTrip() { + Instant instant = Instant.ofEpochMilli(1_600_000_000_123L); + assertEquals(instant, ValueType.castToTimestampMillis(instant, ValueMetadata.NULL_METADATA)); + assertEquals(instant, ValueType.castToTimestampMillis(Timestamp.from(instant), ValueMetadata.NULL_METADATA)); + assertEquals(instant, ValueType.castToTimestampMillis(instant.toEpochMilli(), ValueMetadata.NULL_METADATA)); + assertEquals(instant.toEpochMilli(), ValueType.fromTimestampMillis(instant, ValueMetadata.NULL_METADATA)); + assertEquals(instant, ValueType.toTimestampMillis(instant.toEpochMilli(), ValueMetadata.NULL_METADATA)); + + Instant micros = Instant.ofEpochSecond(1_600_000_000L, 123_000L); + long microVal = ValueType.fromTimestampMicros(micros, ValueMetadata.NULL_METADATA); + assertEquals(micros, ValueType.toTimestampMicros(microVal, ValueMetadata.NULL_METADATA)); + + Instant nanos = Instant.ofEpochSecond(1_600_000_000L, 123_456L); + long nanoVal = ValueType.fromTimestampNanos(nanos, ValueMetadata.NULL_METADATA); + assertEquals(nanos, ValueType.toTimestampNanos(nanoVal, ValueMetadata.NULL_METADATA)); + } + + @Test + public void testLocalTimestampRoundTrip() { + LocalDateTime local = LocalDateTime.of(2021, 5, 6, 7, 8, 9, 10_000_000); + long millis = ValueType.fromLocalTimestampMillis(local, ValueMetadata.NULL_METADATA); + assertEquals(local, ValueType.toLocalTimestampMillis(millis, ValueMetadata.NULL_METADATA)); + assertEquals(local, ValueType.castToLocalTimestampMillis(local, ValueMetadata.NULL_METADATA)); + assertEquals(local, + ValueType.castToLocalTimestampMillis(local.toInstant(ZoneOffset.UTC).toEpochMilli(), ValueMetadata.NULL_METADATA)); + + LocalDateTime micros = LocalDateTime.of(2021, 5, 6, 7, 8, 9, 11_000); + long microVal = ValueType.fromLocalTimestampMicros(micros, ValueMetadata.NULL_METADATA); + assertEquals(micros, ValueType.toLocalTimestampMicros(microVal, ValueMetadata.NULL_METADATA)); + + LocalDateTime nanos = LocalDateTime.of(2021, 5, 6, 7, 8, 9, 12_345); + long nanoVal = ValueType.fromLocalTimestampNanos(nanos, ValueMetadata.NULL_METADATA); + assertEquals(nanos, ValueType.toLocalTimestampNanos(nanoVal, ValueMetadata.NULL_METADATA)); + } + + @Test + public void testFromParquetPrimitiveType() { + assertEquals(ValueType.LONG, valueTypeFor(PrimitiveType.PrimitiveTypeName.INT64)); + assertEquals(ValueType.INT, valueTypeFor(PrimitiveType.PrimitiveTypeName.INT32)); + assertEquals(ValueType.BOOLEAN, valueTypeFor(PrimitiveType.PrimitiveTypeName.BOOLEAN)); + assertEquals(ValueType.BYTES, valueTypeFor(PrimitiveType.PrimitiveTypeName.BINARY)); + assertEquals(ValueType.FLOAT, valueTypeFor(PrimitiveType.PrimitiveTypeName.FLOAT)); + assertEquals(ValueType.DOUBLE, valueTypeFor(PrimitiveType.PrimitiveTypeName.DOUBLE)); + } + + private static ValueType valueTypeFor(PrimitiveType.PrimitiveTypeName name) { + PrimitiveType type = name == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY + ? Types.required(name).length(4).named("f") + : Types.required(name).named("f"); + return ValueType.fromParquetPrimitiveType(type); + } + + @Test + public void testFromSchemaPrimitives() { + assertEquals(ValueType.INT, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.INT))); + assertEquals(ValueType.LONG, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.LONG))); + assertEquals(ValueType.STRING, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.STRING))); + assertEquals(ValueType.BOOLEAN, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.BOOLEAN))); + assertEquals(ValueType.DATE, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.DATE))); + assertEquals(ValueType.UUID, ValueType.fromSchema(HoodieSchema.create(HoodieSchemaType.UUID))); + } + + @Test + public void testFromSchemaUnwrapsUnion() { + HoodieSchema nullableInt = HoodieSchema.createNullable(HoodieSchemaType.INT); + assertEquals(ValueType.INT, ValueType.fromSchema(nullableInt)); + } + + @Test + public void testCastToBytesFromFixed() { + byte[] raw = {9, 8, 7}; + GenericData.Fixed fixed = new GenericData.Fixed( + org.apache.avro.Schema.createFixed("f", null, null, raw.length), raw); + assertEquals(ByteBuffer.wrap(raw), ValueType.castToBytes(fixed)); + assertTrue(ValueType.castToBytes(fixed).hasArray()); + } } From 6e8d46d27634719956fdebcd2f4dfd9ae5ac7a58 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 21 Jul 2026 23:45:52 +0700 Subject: [PATCH 177/255] fix(hive-sync): fix partition-value parsing on '=' and pushdown comparator overflow (#19336) Two edge-case defects in the partition path of hudi-hive-sync: - PartitionFilterGenerator.ValueComparator sorted int/bigint partition values with subtraction (i1 - i2, Long.signum(l1 - l2)) to derive the pushdown min/max bounds. The subtraction overflows when the difference exceeds the type range (e.g. Integer.MAX_VALUE and a negative value), giving a wrong ordering and wrong bounds that can exclude valid partitions or trip the Comparator contract check. Use Integer.compare/Long.compare. - MultiPartKeysValueExtractor and HiveStylePartitionValueExtractor split a hive-style key=value segment with split("=") (no limit), so a value that itself contains '=' (base64 padding or an embedded '=') was rejected (aborting the sync) or silently truncated (trailing '=' dropped), causing drift from the metastore. Split on the first '=' only with split("=", 2). Extends TestPartitionFilterGenerator (extreme int/bigint min/max bounds), TestMultiPartKeysValueExtractor and TestPartitionValueExtractor (values containing '='). Co-authored-by: Vova Kolmakov (cherry picked from commit cc75051fc4efa5449f1e0d9e6f6bffe29e2b14f6) --- .../HiveStylePartitionValueExtractor.java | 4 +++- .../hive/MultiPartKeysValueExtractor.java | 4 +++- .../hive/util/PartitionFilterGenerator.java | 11 ++++----- .../hive/TestMultiPartKeysValueExtractor.java | 15 ++++++++++++ .../hive/TestPartitionValueExtractor.java | 8 +++++++ .../util/TestPartitionFilterGenerator.java | 24 +++++++++++++++++++ 6 files changed, 58 insertions(+), 8 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java index 11098698e8aeb..41ccaa59d486d 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java @@ -34,7 +34,9 @@ public class HiveStylePartitionValueExtractor implements PartitionValueExtractor @Override public List extractPartitionValuesInPath(String partitionPath) { // partition path is expected to be in this format partition_key=partition_value. - String[] splits = partitionPath.split("="); + // Split on the first '=' only so a value that itself contains '=' (e.g. base64 padding) + // is kept intact rather than making the split produce more than two parts. + String[] splits = partitionPath.split("=", 2); if (splits.length != 2) { throw new IllegalArgumentException( "Partition path " + partitionPath + " is not in the form partition_key=partition_value."); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java index dd356638a47e6..5404649a5fd6f 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java @@ -42,7 +42,9 @@ public List extractPartitionValuesInPath(String partitionPath) { String[] splits = partitionPath.split("/"); return Arrays.stream(splits).map(s -> { if (s.contains("=")) { - String[] moreSplit = s.split("="); + // Split on the first '=' only so partition values that themselves contain '=' + // (e.g. base64 padding like "col=YWJj==") are preserved instead of being rejected or truncated. + String[] moreSplit = s.split("=", 2); ValidationUtils.checkArgument(moreSplit.length == 2, "Partition Field (" + s + ") not in expected format"); return moreSplit[1]; } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java index d1b934988d2f3..b3ec1b35afc27 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java @@ -140,13 +140,12 @@ public ValueComparator(String type) { public int compare(String s1, String s2) { switch (valueType.toLowerCase(Locale.ROOT)) { case HiveSchemaUtil.INT_TYPE_NAME: - int i1 = Integer.parseInt(s1); - int i2 = Integer.parseInt(s2); - return i1 - i2; + // Use Integer.compare rather than subtraction, which overflows for values whose + // difference exceeds the int range and yields a wrong ordering. + return Integer.compare(Integer.parseInt(s1), Integer.parseInt(s2)); case HiveSchemaUtil.BIGINT_TYPE_NAME: - long l1 = Long.parseLong(s1); - long l2 = Long.parseLong(s2); - return Long.signum(l1 - l2); + // Use Long.compare rather than subtraction, which overflows the long arithmetic. + return Long.compare(Long.parseLong(s1), Long.parseLong(s2)); case HiveSchemaUtil.DATE_TYPE_NAME: case HiveSchemaUtil.STRING_TYPE_NAME: return s1.compareTo(s2); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java index d8b9100309e58..74610a0ea8e66 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java @@ -21,6 +21,8 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,4 +42,17 @@ public void testMultiPartExtractor() { // Test extract hive style partition path assertEquals(expected, valueExtractor.extractPartitionValuesInPath("ds=2021-04-25/hh=04")); } + + @Test + public void testValuesContainingEquals() { + MultiPartKeysValueExtractor valueExtractor = new MultiPartKeysValueExtractor(); + // Only the first '=' separates key from value, so a value containing '=' is kept intact. + assertEquals(Collections.singletonList("a=b"), valueExtractor.extractPartitionValuesInPath("k=a=b")); + // base64-encoded value with '=' padding must not be truncated + assertEquals(Collections.singletonList("YWJjZA=="), valueExtractor.extractPartitionValuesInPath("col=YWJjZA==")); + // empty value + assertEquals(Collections.singletonList(""), valueExtractor.extractPartitionValuesInPath("dt=")); + // multiple hive-style parts whose values contain '=' + assertEquals(Arrays.asList("a=b", "YWJj=="), valueExtractor.extractPartitionValuesInPath("k1=a=b/k2=YWJj==")); + } } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java index 075542d596717..0cce6ee794236 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java @@ -49,6 +49,14 @@ public void testHiveStylePartition() { assertThrows( IllegalArgumentException.class, () -> hiveStylePartition.extractPartitionValuesInPath("2021/04/02")); + // Only the first '=' is the separator, so a value containing '=' is preserved. + assertEquals( + Collections.singletonList("a=b=c"), + hiveStylePartition.extractPartitionValuesInPath("k=a=b=c")); + // base64-encoded value with '=' padding must not be truncated + assertEquals( + Collections.singletonList("YWJjZA=="), + hiveStylePartition.extractPartitionValuesInPath("col=YWJjZA==")); } @Test diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java index b607e7f6948c6..a010261a21bdf 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java @@ -79,6 +79,30 @@ public void testPushDownFilters() { partitionFilterGenerator.generatePushDownFilter(writtenPartitions, partitionFieldSchemas, config)); } + @Test + public void testMinMaxFilterNoNumericOverflow() { + Properties props = new Properties(); + // force the min/max branch so the values are sorted by ValueComparator + props.put(HIVE_SYNC_FILTER_PUSHDOWN_MAX_SIZE.key(), "0"); + HiveSyncConfig config = new HiveSyncConfig(props); + List partitionFieldSchemas = new ArrayList<>(2); + partitionFieldSchemas.add(new FieldSchema("intcol", "int")); + partitionFieldSchemas.add(new FieldSchema("bigcol", "bigint")); + + List writtenPartitions = new ArrayList<>(); + // extreme values whose pairwise difference overflows int/long subtraction + writtenPartitions.add("2147483647/9223372036854775807"); + writtenPartitions.add("-2147483648/-9223372036854775808"); + writtenPartitions.add("0/0"); + + // bounds must reflect the true numeric order; a subtraction-based comparator overflows and + // would pick wrong min/max (or throw a comparator-contract violation). + assertEquals( + "((intcol >= -2147483648 AND intcol <= 2147483647) " + + "AND (bigcol >= -9223372036854775808 AND bigcol <= 9223372036854775807))", + partitionFilterGenerator.generatePushDownFilter(writtenPartitions, partitionFieldSchemas, config)); + } + @Test public void testPushDownFilterIfExceedLimit() { Properties props = new Properties(); From 46468ab15ff940ca50f9147e6a4eeb77bf1c5939 Mon Sep 17 00:00:00 2001 From: vamsikarnika Date: Wed, 22 Jul 2026 07:59:16 +0530 Subject: [PATCH 178/255] fix: relax existing column to nullable in reconcileSchema when source made it nullable (#19337) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In reconcileSchema, detect existing columns whose incoming schema is nullable but the table is required (nullabilityRelaxColumns), exclude them from the early-return short-circuit, and relax them to nullable in the result via updateColumnNullability(col, true). This only ever widens, never tightens (a nullable→required incoming leaves the table nullable), consistent with reconcileSchemaRequirements, and still null-fills genuinely missing columns. (cherry picked from commit d8ebba4974e74d990af9f0c71ebd6073205e603e) --- .../utils/AvroSchemaEvolutionUtils.java | 17 +++++- .../utils/TestAvroSchemaEvolutionUtils.java | 60 +++++++++++++++++++ .../apache/hudi/TestHoodieSchemaUtils.java | 19 ++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java index c31b5c057181a..6ed1722589fb3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java @@ -78,7 +78,19 @@ public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, Intern .stream() .filter(f -> colNamesFromOldSchema.contains(f) && !inComingInternalSchema.findType(f).equals(oldTableSchema.findType(f))) .collect(Collectors.toList()); - if (colNamesFromIncoming.size() == colNamesFromOldSchema.size() && diffFromOldSchema.size() == 0 && typeChangeColumns.isEmpty()) { + // check columns the incoming schema relaxed from required to nullable. Since the result is built from + // oldTableSchema (to preserve column order/ids and to null-fill missing columns), an existing column + // whose incoming counterpart became nullable would otherwise silently keep the table's REQUIRED + // nullability, blocking a valid required -> nullable evolution. We only ever relax (never tighten). + List nullabilityRelaxColumns = colNamesFromIncoming + .stream() + .filter(f -> colNamesFromOldSchema.contains(f) + && !META_FIELD_NAMES.contains(f) + && inComingInternalSchema.findField(f).isOptional() + && !oldTableSchema.findField(f).isOptional()) + .collect(Collectors.toList()); + if (colNamesFromIncoming.size() == colNamesFromOldSchema.size() && diffFromOldSchema.size() == 0 + && typeChangeColumns.isEmpty() && nullabilityRelaxColumns.isEmpty()) { return oldTableSchema; } @@ -122,6 +134,9 @@ public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, Intern typeChange.updateColumnType(col, inComingInternalSchema.findType(col)); }); + // relax existing columns to nullable when the incoming schema made them nullable (valid widening) + nullabilityRelaxColumns.forEach(col -> typeChange.updateColumnNullability(col, true)); + if (makeMissingFieldsNullable) { // mark columns missing from incoming schema as nullable Set visited = new HashSet<>(); diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java index 19d4205d35561..08e425093974f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java @@ -567,4 +567,64 @@ public void testNotEvolveSchemaIfReconciledSchemaUnchanged() { // the evolved schema should be the old table schema, since there is no type change at all. Assertions.assertEquals(oldInternalSchema, evolvedSchema); } + + /** + * When the incoming schema relaxes an existing required column to nullable, reconcileSchema must evolve + * that column to nullable in the result, even when makeMissingFieldsNullable is true. Previously the + * result was rebuilt from the required table and the relaxation was silently dropped, so records with + * null in that column failed the write / were quarantined. + */ + @Test + public void testReconcileSchemaRelaxesExistingColumnToNullable() { + // table: id (required int), flag (required boolean) -- same column set as the incoming schema + Types.RecordType oldRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, false, "flag", Types.BooleanType.get()) + ); + InternalSchema oldSchema = new InternalSchema(oldRecord); + // incoming: identical columns, but the source relaxed "flag" to nullable + Types.RecordType incomingRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, true, "flag", Types.BooleanType.get()) + ); + incomingRecord = (Types.RecordType) InternalSchemaBuilder.getBuilder().refreshNewId(incomingRecord, new AtomicInteger(0)); + HoodieSchema incomingSchema = InternalSchemaConverter.convert(incomingRecord, "test1"); + + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true); + + Types.RecordType checkedRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, true, "flag", Types.BooleanType.get()) + ); + Assertions.assertEquals(checkedRecord, result.getRecord()); + } + + /** + * reconcileSchema must only ever relax (widen) an existing column's nullability, never tighten it: if the + * incoming schema marks a column required but the table has it nullable, the table stays nullable. + */ + @Test + public void testReconcileSchemaDoesNotTightenNullableToRequired() { + // table: id (required int), flag (nullable boolean) + Types.RecordType oldRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, true, "flag", Types.BooleanType.get()) + ); + InternalSchema oldSchema = new InternalSchema(oldRecord); + // incoming: source tightened "flag" to required -- must NOT tighten the table + Types.RecordType incomingRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, false, "flag", Types.BooleanType.get()) + ); + incomingRecord = (Types.RecordType) InternalSchemaBuilder.getBuilder().refreshNewId(incomingRecord, new AtomicInteger(0)); + HoodieSchema incomingSchema = InternalSchemaConverter.convert(incomingRecord, "test1"); + + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true); + + Types.RecordType checkedRecord = Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()), + Types.Field.get(1, true, "flag", Types.BooleanType.get()) + ); + Assertions.assertEquals(checkedRecord, result.getRecord()); + } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieSchemaUtils.java b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieSchemaUtils.java index 18d488098c8f8..d4e1e3aeba7df 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieSchemaUtils.java +++ b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieSchemaUtils.java @@ -321,6 +321,25 @@ void testFieldReordering() { assertEquals(expected, deduceWriterSchema(end, start, true)); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testExistingColumnRelaxedToNullableEvolves(boolean setNullForMissingColumns) { + // Table has field2 as a required boolean; the incoming (source) schema relaxed it to nullable, same + // column set otherwise. The deduced writer schema must evolve field2 to nullable regardless of the + // set.null.for.missing.columns flag -- with the flag on this used to silently stay required, so records + // with null in field2 failed the write / were quarantined. + HoodieSchema table = createRecord("relaxRec", + createPrimitiveField("field1", HoodieSchemaType.INT), + createPrimitiveField("field2", HoodieSchemaType.BOOLEAN)); + HoodieSchema incoming = createRecord("relaxRec", + createPrimitiveField("field1", HoodieSchemaType.INT), + createNullablePrimitiveField("field2", HoodieSchemaType.BOOLEAN)); + HoodieSchema expected = createRecord("relaxRec", + createPrimitiveField("field1", HoodieSchemaType.INT), + createNullablePrimitiveField("field2", HoodieSchemaType.BOOLEAN)); + assertEquals(expected, deduceWriterSchema(incoming, table, setNullForMissingColumns)); + } + private static HoodieSchema deduceWriterSchema(HoodieSchema incomingSchema, HoodieSchema latestTableSchema) { return deduceWriterSchema(incomingSchema, latestTableSchema, false); } From 55a0957e0a9543be737d572eb26df0cc46b96a5d Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 22 Jul 2026 10:41:10 +0700 Subject: [PATCH 179/255] fix(hive-sync): set HMS table createTime in seconds instead of milliseconds (#19335) (cherry picked from commit 58dd532e64a7631971c0f735b82edb87285ad326) --- .../apache/hudi/hive/ddl/HMSDDLExecutor.java | 4 +- .../ddl/TestHMSDDLExecutorCreateTable.java | 70 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestHMSDDLExecutorCreateTable.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java index fa0d31a74cfae..b97bc8010d75f 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java @@ -122,7 +122,9 @@ public void createTable(String tableName, HoodieSchema storageSchema, String inp newTb.setDbName(databaseName); newTb.setTableName(tableName); newTb.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); - newTb.setCreateTime((int) System.currentTimeMillis()); + // Hive stores createTime as seconds since the epoch in an i32 field; passing raw + // milliseconds both uses the wrong unit and overflows the int cast. + newTb.setCreateTime((int) (System.currentTimeMillis() / 1000)); StorageDescriptor storageDescriptor = new StorageDescriptor(); storageDescriptor.setCols(fieldSchema); storageDescriptor.setInputFormat(inputFormatClass); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestHMSDDLExecutorCreateTable.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestHMSDDLExecutorCreateTable.java new file mode 100644 index 0000000000000..a03a7c28042f2 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestHMSDDLExecutorCreateTable.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive.ddl; + +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.hive.HiveSyncConfig; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Properties; + +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +class TestHMSDDLExecutorCreateTable { + + @Test + void createTableSetsCreateTimeInSeconds() throws Exception { + Properties props = new Properties(); + props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb"); + props.setProperty(META_SYNC_BASE_PATH.key(), "/tmp/test_table"); + HiveSyncConfig config = new HiveSyncConfig(props, new Configuration()); + + IMetaStoreClient client = mock(IMetaStoreClient.class); + HMSDDLExecutor executor = new HMSDDLExecutor(config, client); + + HoodieSchema schema = HoodieSchema.createRecord("test_record", null, null, + Collections.singletonList(HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)))); + + long beforeSec = System.currentTimeMillis() / 1000; + executor.createTable("test_table", schema, "input.Format", "output.Format", "serde.Class", + new HashMap<>(), new HashMap<>()); + long afterSec = System.currentTimeMillis() / 1000; + + ArgumentCaptor

    captor = ArgumentCaptor.forClass(Table.class); + verify(client).createTable(captor.capture()); + int createTime = captor.getValue().getCreateTime(); + + // createTime must be epoch seconds within the call window. + assertTrue(createTime >= beforeSec && createTime <= afterSec, + "createTime should be epoch seconds within [" + beforeSec + ", " + afterSec + "] but was " + createTime); + } +} From 9d4c6036179f1f3b75ae2d26d49432510d2b9926 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Wed, 22 Jul 2026 15:02:39 +0800 Subject: [PATCH 180/255] chore: upload Flink integration-test coverage (#19343) * ci: upload Flink integration-test coverage * chore: clarify Flink integration-test coverage (cherry picked from commit 7f786d434fa8e1d60880fac9bc5a9323b99c0f0a) --- .github/workflows/bot.yml | 26 ++++++++++++++++++++++++-- pom.xml | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index e4d3238ffc3fb..4d180e94d6f24 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -941,7 +941,18 @@ jobs: if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }} run: | mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS - mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS + mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false + - name: Generate merged coverage report + if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE + - name: Upload coverage to Codecov + if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 + with: + files: ./jacoco-report.xml + disable_search: true + flags: flink-integration-tests + token: ${{ secrets.CODECOV_TOKEN }} test-flink-2: runs-on: ubuntu-latest @@ -977,7 +988,18 @@ jobs: if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }} run: | mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS - mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS + mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false + - name: Generate merged coverage report + if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE + - name: Upload coverage to Codecov + if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 + with: + files: ./jacoco-report.xml + disable_search: true + flags: flink-integration-tests + token: ${{ secrets.CODECOV_TOKEN }} docker-java17-test: runs-on: ubuntu-latest diff --git a/pom.xml b/pom.xml index dc95ae3e750d5..57e29a16f5fa6 100644 --- a/pom.xml +++ b/pom.xml @@ -2370,6 +2370,25 @@ + + + org.jacoco + jacoco-maven-plugin + + + + prepare-agent + + + ${project.build.directory}/jacoco-agent/${jacoco.agent.dest.filename} + + + + From 0311b3bb19f9555ef1c256d86ff58c3c391a18d9 Mon Sep 17 00:00:00 2001 From: vamshipasunuru1 <127545665+vamshipasunuru1@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:53:52 -0700 Subject: [PATCH 181/255] fix: do not fall back to timeline server markers on transient HDFS failures (#18887) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: do not fall back to timeline server markers on transient HDFS failures When MARKERS.type is absent, MarkerBasedRollbackUtils tries DIRECT markers and catches IOException to fall back to TIMELINE_SERVER_BASED. This was too broad: a transient "Server too busy" RetriableException is also an IOException, causing rollback to use the timeline server marker path which finds 0 markers and deletes nothing, leaving orphan data files behind. Only catch IllegalArgumentException (marker path format mismatch) for the fallback. Let IOException propagate so the rollback fails and retries rather than silently producing an incorrect result. * update log * Address review: use parameterized SLF4J logging and fix storage reference - Switch String.format to SLF4J parameterized form so the exception stack trace is preserved in the log output. - Fix compile error: getTimelineServerBasedMarkers takes HoodieStorage (storage), not the undefined fileSystem variable. * Address review: rewrite test to exercise the actual code path The prior test stubbed metaClient.getFs() and HoodieWrapperFileSystem.listStatus(), but the production code goes through table.getStorage() and HoodieStorage.listDirectEntries(). The injected IOException never reached the catch block under test. Rewrite to mock the correct seams: - table.getStorage() returns a mock HoodieStorage - storage.exists(MARKERS.type) → false to trigger the fallback branch - storage.exists(markerDir) → true so allMarkerFilePaths() lists entries - storage.listDirectEntries(markerDir) throws the failure of interest Verified by inspecting the surefire stack trace: the exception now originates in FSUtils.processFiles → DirectWriteMarkers.allMarkerFilePaths → MarkerBasedRollbackUtils.getAllMarkerPaths, confirming the fix is exercised. Also add a companion test asserting that IllegalArgumentException still falls back to TIMELINE_SERVER_BASED (preserving the original intended behavior). --------- Co-authored-by: vamshi_UBER Co-authored-by: vamshipasunuru1 (cherry picked from commit 55becc7b03f4cf8ec3e1e5303d14fd46bfc8f9b9) --- .../marker/MarkerBasedRollbackUtils.java | 15 +- .../marker/TestMarkerBasedRollbackUtils.java | 134 ++++++++++++++++++ 2 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestMarkerBasedRollbackUtils.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java index e7e539fa88990..562f6bc9221bb 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java @@ -70,9 +70,18 @@ public static List getAllMarkerPaths(HoodieTable table, HoodieEngineCont WriteMarkers writeMarkers = WriteMarkersFactory.get(DIRECT, table, instant); try { return new ArrayList<>(writeMarkers.allMarkerFilePaths()); - } catch (IOException | IllegalArgumentException e) { - log.warn("{} not present and {} marker failed with error: {}. Falling back to {} marker", - MARKER_TYPE_FILENAME, DIRECT, e.getMessage(), TIMELINE_SERVER_BASED); + } catch (IOException e) { + // Do NOT fall back to TIMELINE_SERVER_BASED on transient IO failures (e.g., HDFS throttling). + // The timeline server looks in a different location and would return 0 markers, causing the + // rollback to skip deleting data files and leaving orphan files on the table. + log.warn("{} not present and {} marker listing failed with IO error. " + + "Propagating exception, rollback will retry rather than fall back to {}.", + MARKER_TYPE_FILENAME, DIRECT, TIMELINE_SERVER_BASED, e); + throw e; + } catch (IllegalArgumentException e) { + // IllegalArgumentException indicates a marker path format mismatch, fall back to timeline server. + log.warn("{} not present and {} marker failed. Falling back to {} marker", + MARKER_TYPE_FILENAME, DIRECT, TIMELINE_SERVER_BASED, e); return getTimelineServerBasedMarkers(context, parallelism, markerDir, storage); } } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestMarkerBasedRollbackUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestMarkerBasedRollbackUtils.java new file mode 100644 index 0000000000000..cc964e4c65d3a --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestMarkerBasedRollbackUtils.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.table.marker; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.table.HoodieTable; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Collections; + +import static org.apache.hudi.common.util.MarkerUtils.MARKER_TYPE_FILENAME; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link MarkerBasedRollbackUtils}. + * + *

    These tests target the {@code MARKERS.type}-absent code path in + * {@link MarkerBasedRollbackUtils#getAllMarkerPaths}. That branch attempts to list DIRECT + * markers first and used to catch {@code IOException | IllegalArgumentException} and fall + * back to TIMELINE_SERVER_BASED. A transient HDFS failure would therefore be swallowed and + * the rollback would return zero marker paths, leaving orphan data files on the table. + * + *

    The tests mock the {@link HoodieStorage} seams the production code actually goes + * through: + *

      + *
    • {@code readMarkerType(...)} -> {@code storage.exists(MARKERS.type)} = false + *
    • {@code DirectWriteMarkers.doesMarkerDirExist()} -> {@code storage.exists(markerDir)} = true + *
    • {@code FSUtils.processFiles} -> {@code storage.listDirectEntries(markerDir)} throws + *
    + */ +public class TestMarkerBasedRollbackUtils { + + private static final String INSTANT = "20260101000000"; + private static final String BASE_PATH = "/tmp/test-table"; + private static final String MARKER_DIR = BASE_PATH + "/.hoodie/.temp/" + INSTANT; + + private HoodieTable mockTable; + private HoodieTableMetaClient mockMetaClient; + private HoodieEngineContext mockContext; + private HoodieStorage mockStorage; + + @BeforeEach + public void setUp() throws IOException { + mockTable = mock(HoodieTable.class); + mockMetaClient = mock(HoodieTableMetaClient.class); + mockContext = mock(HoodieEngineContext.class); + mockStorage = mock(HoodieStorage.class); + HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class); + + when(mockTable.getMetaClient()).thenReturn(mockMetaClient); + when(mockTable.getContext()).thenReturn(mockContext); + when(mockTable.getStorage()).thenReturn(mockStorage); + when(mockMetaClient.getBasePath()).thenReturn(new StoragePath(BASE_PATH)); + when(mockMetaClient.getMarkerFolderPath(INSTANT)).thenReturn(MARKER_DIR); + when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig); + // Table version 8+ selects DirectWriteMarkers in WriteMarkersFactory. + when(mockTableConfig.getTableVersion()).thenReturn(HoodieTableVersion.EIGHT); + + StoragePath markerDirPath = new StoragePath(MARKER_DIR); + StoragePath markerTypeFilePath = new StoragePath(markerDirPath, MARKER_TYPE_FILENAME); + // MARKERS.type is absent, this drives execution into the fallback branch under test. + when(mockStorage.exists(markerTypeFilePath)).thenReturn(false); + // The marker directory itself exists so DirectWriteMarkers.allMarkerFilePaths() + // proceeds to list entries (rather than early-returning an empty set). + when(mockStorage.exists(markerDirPath)).thenReturn(true); + } + + /** + * A transient IO failure while listing DIRECT markers must propagate as IOException + * rather than silently falling back to TIMELINE_SERVER_BASED. Falling back would return + * zero marker paths (timeline server uses a different location) and cause rollback to + * leave orphan data files on the table. + */ + @Test + public void testGetAllMarkerPathsPropagatesIOExceptionOnTransientListingFailure() throws IOException { + when(mockStorage.listDirectEntries(new StoragePath(MARKER_DIR))) + .thenThrow(new IOException("Server too busy - disconnecting")); + + IOException thrown = assertThrows(IOException.class, + () -> MarkerBasedRollbackUtils.getAllMarkerPaths(mockTable, mockContext, INSTANT, 1)); + // Verify the original transient error surfaces to the caller (not a wrapped fallback error). + assertEquals("Server too busy - disconnecting", thrown.getMessage()); + } + + /** + * IllegalArgumentException (e.g., marker path format mismatch) must retain the original + * fallback behavior, read markers via TIMELINE_SERVER_BASED path instead of failing. + * + *

    Both DirectWriteMarkers and the timeline-server-based reader end up calling + * {@code storage.listDirectEntries(markerDir)}. We stub the first invocation to throw + * (triggering the fallback under test) and subsequent invocations to return an empty + * list so the fallback path completes cleanly. + */ + @Test + public void testGetAllMarkerPathsFallsBackToTimelineServerOnIllegalArgumentException() throws IOException { + when(mockStorage.listDirectEntries(new StoragePath(MARKER_DIR))) + .thenThrow(new IllegalArgumentException("bad marker path")) + .thenReturn(Collections.emptyList()); + + // No exception should surface, the IllegalArgumentException must be handled by the + // fallback rather than propagating to the caller. + assertDoesNotThrow( + () -> MarkerBasedRollbackUtils.getAllMarkerPaths(mockTable, mockContext, INSTANT, 1)); + } +} From d08d653760cfee8494a7bcc41d12586a3a7e1a90 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 22 Jul 2026 02:10:30 -0700 Subject: [PATCH 182/255] refactor(spark): share the Spark 3.x legacy parquet file format via a common base (#19341) (cherry picked from commit 0acf20fed0cc376ed68680b67df71ddbb0db423b) --- .../Spark3LegacyHoodieParquetFileFormat.scala | 480 ++++++++++++++++++ ...Spark33LegacyHoodieParquetFileFormat.scala | 460 ++--------------- ...Spark34LegacyHoodieParquetFileFormat.scala | 438 +--------------- ...Spark35LegacyHoodieParquetFileFormat.scala | 439 +--------------- 4 files changed, 565 insertions(+), 1252 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3LegacyHoodieParquetFileFormat.scala diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3LegacyHoodieParquetFileFormat.scala new file mode 100644 index 0000000000000..d6b3ab3e3f990 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3LegacyHoodieParquetFileFormat.scala @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.hudi.client.utils.SparkInternalSchemaConverter +import org.apache.hudi.common.fs.FSUtils +import org.apache.hudi.common.table.timeline.TimelineLayout +import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion +import org.apache.hudi.common.util.InternalSchemaCache +import org.apache.hudi.common.util.StringUtils.isNullOrEmpty +import org.apache.hudi.common.util.collection.Pair +import org.apache.hudi.hadoop.fs.HadoopFSUtils +import org.apache.hudi.internal.schema.InternalSchema +import org.apache.hudi.internal.schema.action.InternalSchemaMerger +import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper} +import org.apache.hudi.storage.HoodieStorageUtils + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.hadoop.mapred.FileSplit +import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS +import org.apache.parquet.hadoop.{ParquetInputFormat, ParquetRecordReader} +import org.apache.spark.TaskContext +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, JoinedRow} +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator} +import org.apache.spark.sql.execution.datasources.parquet.Spark3LegacyHoodieParquetFileFormat._ +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType} +import org.apache.spark.util.SerializableConfiguration + +import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` + +/** + * Base [[ParquetFileFormat]] shared by the Spark 3.3, 3.4 and 3.5 legacy readers. + * + * It holds the logic common to all three versions. Every expression that relies on a + * version-specific Spark API is delegated to a protected hook that each concrete subclass + * overrides, so this class compiles unchanged against each supported Spark 3.x version. + * + * This is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior + * that's not possible to customize in any other way, with the following changes applied: + *

      + *
    1. Avoiding appending partition values to the rows read from the data file
      2. + *
    2. Schema on-read
      3. + *
    + */ +abstract class Spark3LegacyHoodieParquetFileFormat(shouldAppendPartitionValues: Boolean) extends ParquetFileFormat { + + /** + * Converts a [[StructType]] to its attributes. Spark 3.3/3.4 expose [[StructType.toAttributes]] + * while Spark 3.5 moved it to [[org.apache.spark.sql.catalyst.types.DataTypeUtils]]. + */ + protected def toAttributes(structType: StructType): Seq[Attribute] + + /** + * Extracts the [[Path]] of the file being read. Spark 3.3 keeps the path as a string while + * Spark 3.4+ wraps it in a `SparkPath`. + */ + protected def getFilePath(file: PartitionedFile): Path + + /** + * Whether the vectorized reader is enabled for the given schema. + */ + protected def isVectorizedReaderEnabled(sparkSession: SparkSession, resultSchema: StructType): Boolean + + /** + * Whether string-predicate push-down is enabled (renamed between Spark 3.3 and 3.4). + */ + protected def getPushDownStringPredicate(sqlConf: SQLConf): Boolean + + /** + * Whether the reader should return columnar batches. + */ + protected def getReturningBatch(sparkSession: SparkSession, resultSchema: StructType): Boolean + + /** + * Sets the version-specific timestamp and nanos-as-long related flags on the hadoop conf. + */ + protected def setParquetTimeConfs(hadoopConf: Configuration, sparkSession: SparkSession): Unit + + override def buildReaderWithPartitionValues(sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { + hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) + hadoopConf.set( + ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, + requiredSchema.json) + hadoopConf.set( + ParquetWriteSupport.SPARK_ROW_SCHEMA, + requiredSchema.json) + hadoopConf.set( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + sparkSession.sessionState.conf.sessionLocalTimeZone) + hadoopConf.setBoolean( + SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, + sparkSession.sessionState.conf.nestedSchemaPruningEnabled) + hadoopConf.setBoolean( + SQLConf.CASE_SENSITIVE.key, + sparkSession.sessionState.conf.caseSensitiveAnalysis) + + ParquetWriteSupport.setSchema(requiredSchema, hadoopConf) + + // Sets flags for `ParquetToSparkSchemaConverter` + hadoopConf.setBoolean( + SQLConf.PARQUET_BINARY_AS_STRING.key, + sparkSession.sessionState.conf.isParquetBinaryAsString) + hadoopConf.setBoolean( + SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, + sparkSession.sessionState.conf.isParquetINT96AsTimestamp) + // Version-specific timestamp and nanos-as-long flags. + setParquetTimeConfs(hadoopConf, sparkSession) + val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) + // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself, + // therefore it's safe to do schema projection here + if (!isNullOrEmpty(internalSchemaStr)) { + val prunedInternalSchemaStr = + pruneInternalSchema(internalSchemaStr, requiredSchema) + hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr) + } + + val broadcastedHadoopConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) + + // TODO: if you move this into the closure it reverts to the default values. + // If true, enable using the custom RecordReader for parquet. This only works for + // a subset of the types (no complex types). + val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) + val sqlConf = sparkSession.sessionState.conf + val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled + val enableVectorizedReader: Boolean = isVectorizedReaderEnabled(sparkSession, resultSchema) + val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled + val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion + val capacity = sqlConf.parquetVectorizedReaderBatchSize + val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown + val pushDownDate = sqlConf.parquetFilterPushDownDate + val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp + val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal + val pushDownStringStartWith = getPushDownStringPredicate(sqlConf) + val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold + val isCaseSensitive = sqlConf.caseSensitiveAnalysis + val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) + val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead + val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead + val timeZoneId = Option(sqlConf.sessionLocalTimeZone) + // Whole stage codegen (PhysicalRDD) is able to deal with batches directly. + val returningBatch = getReturningBatch(sparkSession, resultSchema) + + (file: PartitionedFile) => { + assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size) + + val filePath = getFilePath(file) + val split = new FileSplit(filePath, file.start, file.length, Array.empty[String]) + + val sharedConf = broadcastedHadoopConf.value.value + + // Fetch internal schema + val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) + // Internal schema has to be pruned at this point + val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) + + var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent + + val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH) + val fileSchema = if (shouldUseInternalSchema) { + val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong; + val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST) + //TODO: HARDCODED TIMELINE OBJECT + val layout = TimelineLayout.fromVersion(TimelineLayoutVersion.CURR_LAYOUT_VERSION) + val storage = HoodieStorageUtils.getStorage(tablePath, HadoopFSUtils.getStorageConf(sharedConf)) + InternalSchemaCache.getInternalSchemaByVersionId( + commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits, + layout) + } else { + null + } + + lazy val footerFileMetaData = + ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData + // Try to push down filters when filter push-down is enabled. + val pushed = if (enableParquetFilterPushDown) { + val parquetSchema = footerFileMetaData.getSchema + val datetimeRebaseSpec = + DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) + val parquetFilters = new ParquetFilters( + parquetSchema, + pushDownDate, + pushDownTimestamp, + pushDownDecimal, + pushDownStringStartWith, + pushDownInFilterThreshold, + isCaseSensitive, + datetimeRebaseSpec) + filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null))) + // Collects all converted Parquet filter predicates. Notice that not all predicates can be + // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` + // is used here. + .flatMap(parquetFilters.createFilter) + .reduceOption(FilterApi.and) + } else { + None + } + + // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' + // *only* if the file was created by something other than "parquet-mr", so check the actual + // writer here for this file. We have to do this per-file, as each file in the table may + // have different writers. + // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. + def isCreatedByParquetMr: Boolean = + footerFileMetaData.getCreatedBy().startsWith("parquet-mr") + + val convertTz = + if (timestampConversion && !isCreatedByParquetMr) { + Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) + } else { + None + } + + val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) + + // Clone new conf + val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value) + val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) { + val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema() + val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema) + + hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json) + + SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema) + } else { + val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema) + if (!implicitTypeChangeInfo.isEmpty) { + shouldUseInternalSchema = true + hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json) + } + implicitTypeChangeInfo + } + + if (enableVectorizedReader && shouldUseInternalSchema && + !typeChangeInfos.values().forall(_.getLeft.isInstanceOf[AtomicType])) { + throw new IllegalArgumentException( + "Nested types with type changes(implicit or explicit) cannot be read in vectorized mode. " + + "To workaround this issue, set spark.sql.parquet.enableVectorizedReader=false.") + } + + val hadoopAttemptContext = + new TaskAttemptContextImpl(hadoopAttemptConf, attemptId) + + // Try to push down filters when filter push-down is enabled. + // Notice: This push-down is RowGroups level, not individual records. + if (pushed.isDefined) { + ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) + } + val taskContext = Option(TaskContext.get()) + if (enableVectorizedReader) { + val vectorizedReader = + if (shouldUseInternalSchema) { + val int96RebaseSpec = + DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) + val datetimeRebaseSpec = + DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) + new HoodieVectorizedParquetRecordReader( + convertTz.orNull, + datetimeRebaseSpec.mode.toString, + datetimeRebaseSpec.timeZone, + int96RebaseSpec.mode.toString, + int96RebaseSpec.timeZone, + enableOffHeapColumnVector && taskContext.isDefined, + capacity, + typeChangeInfos) + } else { + val int96RebaseSpec = + DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) + val datetimeRebaseSpec = + DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) + new VectorizedParquetRecordReader( + convertTz.orNull, + datetimeRebaseSpec.mode.toString, + datetimeRebaseSpec.timeZone, + int96RebaseSpec.mode.toString, + int96RebaseSpec.timeZone, + enableOffHeapColumnVector && taskContext.isDefined, + capacity) + } + + // SPARK-37089: We cannot register a task completion listener to close this iterator here + // because downstream exec nodes have already registered their listeners. Since listeners + // are executed in reverse order of registration, a listener registered here would close the + // iterator while downstream exec nodes are still running. When off-heap column vectors are + // enabled, this can cause a use-after-free bug leading to a segfault. + // + // Instead, we use FileScanRDD's task completion listener to close this iterator. + val iter = new RecordReaderIterator(vectorizedReader) + try { + vectorizedReader.initialize(split, hadoopAttemptContext) + + // NOTE: We're making appending of the partitioned values to the rows read from the + // data file configurable + if (shouldAppendPartitionValues) { + logDebug(s"Appending $partitionSchema ${file.partitionValues}") + vectorizedReader.initBatch(partitionSchema, file.partitionValues) + } else { + vectorizedReader.initBatch(StructType(Nil), InternalRow.empty) + } + + if (returningBatch) { + vectorizedReader.enableReturningBatches() + } + + // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. + iter.asInstanceOf[Iterator[InternalRow]] + } catch { + case e: Throwable => + // SPARK-23457: In case there is an exception in initialization, close the iterator to + // avoid leaking resources. + iter.close() + throw e + } + } else { + logDebug(s"Falling back to parquet-mr") + val int96RebaseSpec = + DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) + val datetimeRebaseSpec = + DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) + val readSupport = new HoodieParquetReadSupport( + convertTz, + enableVectorizedReader = false, + enableTimestampFieldRepair = true, + datetimeRebaseSpec, + int96RebaseSpec) + + val reader = if (pushed.isDefined && enableRecordFilter) { + val parquetFilter = FilterCompat.get(pushed.get, null) + new ParquetRecordReader[InternalRow](readSupport, parquetFilter) + } else { + new ParquetRecordReader[InternalRow](readSupport) + } + val iter = new RecordReaderIterator[InternalRow](reader) + try { + reader.initialize(split, hadoopAttemptContext) + + val fullSchema = toAttributes(requiredSchema) ++ toAttributes(partitionSchema) + val unsafeProjection = if (typeChangeInfos.isEmpty) { + GenerateUnsafeProjection.generate(fullSchema, fullSchema) + } else { + // find type changed. + val newSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) => + if (typeChangeInfos.containsKey(i)) { + StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata) + } else f + }) + val newFullSchema = toAttributes(newSchema) ++ toAttributes(partitionSchema) + val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) => + if (typeChangeInfos.containsKey(i)) { + val srcType = typeChangeInfos.get(i).getRight + val dstType = typeChangeInfos.get(i).getLeft + val needTimeZone = Cast.needsTimeZone(srcType, dstType) + Cast(attr, dstType, if (needTimeZone) timeZoneId else None) + } else attr + } + GenerateUnsafeProjection.generate(castSchema, newFullSchema) + } + + // NOTE: We're making appending of the partitioned values to the rows read from the + // data file configurable + if (!shouldAppendPartitionValues || partitionSchema.length == 0) { + // There is no partition columns + iter.map(unsafeProjection) + } else { + val joinedRow = new JoinedRow() + iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) + } + } catch { + case e: Throwable => + // SPARK-23457: In case there is an exception in initialization, close the iterator to + // avoid leaking resources. + iter.close() + throw e + } + } + } + } +} + +object Spark3LegacyHoodieParquetFileFormat { + + def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = { + val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) + if (querySchemaOption.isPresent && requiredSchema.nonEmpty) { + val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get()) + SerDeHelper.toJson(prunedSchema) + } else { + internalSchemaStr + } + } + + private def rebuildFilterFromParquet(oldFilter: Filter, fileSchema: InternalSchema, querySchema: InternalSchema): Filter = { + if (fileSchema == null || querySchema == null) { + oldFilter + } else { + oldFilter match { + case eq: EqualTo => + val newAttribute = InternalSchemaUtils.reBuildFilterName(eq.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else eq.copy(attribute = newAttribute) + case eqs: EqualNullSafe => + val newAttribute = InternalSchemaUtils.reBuildFilterName(eqs.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else eqs.copy(attribute = newAttribute) + case gt: GreaterThan => + val newAttribute = InternalSchemaUtils.reBuildFilterName(gt.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else gt.copy(attribute = newAttribute) + case gtr: GreaterThanOrEqual => + val newAttribute = InternalSchemaUtils.reBuildFilterName(gtr.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else gtr.copy(attribute = newAttribute) + case lt: LessThan => + val newAttribute = InternalSchemaUtils.reBuildFilterName(lt.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else lt.copy(attribute = newAttribute) + case lte: LessThanOrEqual => + val newAttribute = InternalSchemaUtils.reBuildFilterName(lte.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else lte.copy(attribute = newAttribute) + case i: In => + val newAttribute = InternalSchemaUtils.reBuildFilterName(i.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else i.copy(attribute = newAttribute) + case isn: IsNull => + val newAttribute = InternalSchemaUtils.reBuildFilterName(isn.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else isn.copy(attribute = newAttribute) + case isnn: IsNotNull => + val newAttribute = InternalSchemaUtils.reBuildFilterName(isnn.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else isnn.copy(attribute = newAttribute) + case And(left, right) => + And(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) + case Or(left, right) => + Or(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) + case Not(child) => + Not(rebuildFilterFromParquet(child, fileSchema, querySchema)) + case ssw: StringStartsWith => + val newAttribute = InternalSchemaUtils.reBuildFilterName(ssw.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else ssw.copy(attribute = newAttribute) + case ses: StringEndsWith => + val newAttribute = InternalSchemaUtils.reBuildFilterName(ses.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else ses.copy(attribute = newAttribute) + case sc: StringContains => + val newAttribute = InternalSchemaUtils.reBuildFilterName(sc.attribute, fileSchema, querySchema) + if (newAttribute.isEmpty) AlwaysTrue else sc.copy(attribute = newAttribute) + case AlwaysTrue => + AlwaysTrue + case AlwaysFalse => + AlwaysFalse + case _ => + AlwaysTrue + } + } + } +} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala index c19d3a5126d63..fc1df0b16f8e6 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala @@ -1,451 +1,61 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.spark.sql.execution.datasources.parquet -import org.apache.hudi.client.utils.SparkInternalSchemaConverter -import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.common.table.timeline.TimelineLayout -import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion -import org.apache.hudi.common.util.InternalSchemaCache -import org.apache.hudi.common.util.StringUtils.isNullOrEmpty -import org.apache.hudi.common.util.collection.Pair -import org.apache.hudi.hadoop.fs.HadoopFSUtils -import org.apache.hudi.internal.schema.InternalSchema -import org.apache.hudi.internal.schema.action.InternalSchemaMerger -import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper} -import org.apache.hudi.storage.HoodieStorageUtils - import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.hadoop.mapred.FileSplit -import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType} -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS -import org.apache.parquet.hadoop.{ParquetInputFormat, ParquetRecordReader} -import org.apache.spark.TaskContext import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Cast, JoinedRow} -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator} -import org.apache.spark.sql.execution.datasources.parquet.Spark33LegacyHoodieParquetFileFormat._ +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType} -import org.apache.spark.util.SerializableConfiguration +import org.apache.spark.sql.types.StructType import java.net.URI -import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` - /** - * This class is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior - * that's not possible to customize in any other way - * - * NOTE: This is a version of [[AvroDeserializer]] impl from Spark 3.2.1 w/ w/ the following changes applied to it: - *
      - *
    1. Avoiding appending partition values to the rows read from the data file
      2. - *
    2. Schema on-read
      3. - *
    + * Spark 3.3 concrete implementation of [[Spark3LegacyHoodieParquetFileFormat]]. It only overrides + * the version-specific hooks; the shared reader logic lives in the base class. */ -class Spark33LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValues: Boolean) extends ParquetFileFormat { +class Spark33LegacyHoodieParquetFileFormat(appendPartitionValues: Boolean) + extends Spark3LegacyHoodieParquetFileFormat(appendPartitionValues) { - override def buildReaderWithPartitionValues(sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) - hadoopConf.set( - ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, - requiredSchema.json) - hadoopConf.set( - ParquetWriteSupport.SPARK_ROW_SCHEMA, - requiredSchema.json) - hadoopConf.set( - SQLConf.SESSION_LOCAL_TIMEZONE.key, - sparkSession.sessionState.conf.sessionLocalTimeZone) - hadoopConf.setBoolean( - SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, - sparkSession.sessionState.conf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean( - SQLConf.CASE_SENSITIVE.key, - sparkSession.sessionState.conf.caseSensitiveAnalysis) + override protected def toAttributes(structType: StructType): Seq[Attribute] = + structType.toAttributes - ParquetWriteSupport.setSchema(requiredSchema, hadoopConf) + override protected def getFilePath(file: PartitionedFile): Path = + new Path(new URI(file.filePath)) - // Sets flags for `ParquetToSparkSchemaConverter` - hadoopConf.setBoolean( - SQLConf.PARQUET_BINARY_AS_STRING.key, - sparkSession.sessionState.conf.isParquetBinaryAsString) - hadoopConf.setBoolean( - SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, - sparkSession.sessionState.conf.isParquetINT96AsTimestamp) + override protected def isVectorizedReaderEnabled(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + ParquetUtils.isBatchReadSupportedForSchema(sparkSession.sessionState.conf, resultSchema) + + override protected def getPushDownStringPredicate(sqlConf: SQLConf): Boolean = + sqlConf.parquetFilterPushDownStringStartWith + + override protected def getReturningBatch(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + supportBatch(sparkSession, resultSchema) + + override protected def setParquetTimeConfs(hadoopConf: Configuration, sparkSession: SparkSession): Unit = { // Using string value of this conf to preserve compatibility across spark versions. hadoopConf.setBoolean( "spark.sql.legacy.parquet.nanosAsLong", sparkSession.sessionState.conf.getConfString("spark.sql.legacy.parquet.nanosAsLong", "false").toBoolean ) - val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself, - // therefore it's safe to do schema projection here - if (!isNullOrEmpty(internalSchemaStr)) { - val prunedInternalSchemaStr = - pruneInternalSchema(internalSchemaStr, requiredSchema) - hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr) - } - - val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - - // TODO: if you move this into the closure it reverts to the default values. - // If true, enable using the custom RecordReader for parquet. This only works for - // a subset of the types (no complex types). - val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) - val sqlConf = sparkSession.sessionState.conf - val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled - val enableVectorizedReader: Boolean = - ParquetUtils.isBatchReadSupportedForSchema(sqlConf, resultSchema) - val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled - val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion - val capacity = sqlConf.parquetVectorizedReaderBatchSize - val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown - // Whole stage codegen (PhysicalRDD) is able to deal with batches directly - val returningBatch = supportBatch(sparkSession, resultSchema) - val pushDownDate = sqlConf.parquetFilterPushDownDate - val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp - val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal - val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith - val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold - val isCaseSensitive = sqlConf.caseSensitiveAnalysis - val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) - val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead - val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead - val timeZoneId = Option(sqlConf.sessionLocalTimeZone) - - (file: PartitionedFile) => { - assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size) - - val filePath = new Path(new URI(file.filePath)) - val split = new FileSplit(filePath, file.start, file.length, Array.empty[String]) - - val sharedConf = broadcastedHadoopConf.value.value - - // Fetch internal schema - val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // Internal schema has to be pruned at this point - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - - var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent - - val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH) - val fileSchema = if (shouldUseInternalSchema) { - val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong; - val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST) - //TODO: HARDCODED TIMELINE OBJECT - val layout = TimelineLayout.fromVersion(TimelineLayoutVersion.CURR_LAYOUT_VERSION) - val storage = HoodieStorageUtils.getStorage(tablePath, HadoopFSUtils.getStorageConf(sharedConf)) - InternalSchemaCache.getInternalSchemaByVersionId( - commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits, - layout) - } else { - null - } - - lazy val footerFileMetaData = - ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData - // Try to push down filters when filter push-down is enabled. - val pushed = if (enableParquetFilterPushDown) { - val parquetSchema = footerFileMetaData.getSchema - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val parquetFilters = new ParquetFilters( - parquetSchema, - pushDownDate, - pushDownTimestamp, - pushDownDecimal, - pushDownStringStartWith, - pushDownInFilterThreshold, - isCaseSensitive, - datetimeRebaseSpec) - filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null))) - // Collects all converted Parquet filter predicates. Notice that not all predicates can be - // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` - // is used here. - .flatMap(parquetFilters.createFilter) - .reduceOption(FilterApi.and) - } else { - None - } - - // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' - // *only* if the file was created by something other than "parquet-mr", so check the actual - // writer here for this file. We have to do this per-file, as each file in the table may - // have different writers. - // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. - def isCreatedByParquetMr: Boolean = - footerFileMetaData.getCreatedBy().startsWith("parquet-mr") - - val convertTz = - if (timestampConversion && !isCreatedByParquetMr) { - Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) - } else { - None - } - - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - - // Clone new conf - val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value) - val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) { - val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema() - val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema) - - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json) - - SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema) - } else { - val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema) - if (!implicitTypeChangeInfo.isEmpty) { - shouldUseInternalSchema = true - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json) - } - implicitTypeChangeInfo - } - - if (enableVectorizedReader && shouldUseInternalSchema && - !typeChangeInfos.values().forall(_.getLeft.isInstanceOf[AtomicType])) { - throw new IllegalArgumentException( - "Nested types with type changes(implicit or explicit) cannot be read in vectorized mode. " + - "To workaround this issue, set spark.sql.parquet.enableVectorizedReader=false.") - } - - val hadoopAttemptContext = - new TaskAttemptContextImpl(hadoopAttemptConf, attemptId) - - // Try to push down filters when filter push-down is enabled. - // Notice: This push-down is RowGroups level, not individual records. - if (pushed.isDefined) { - ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) - } - val taskContext = Option(TaskContext.get()) - if (enableVectorizedReader) { - val vectorizedReader = - if (shouldUseInternalSchema) { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new HoodieVectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity, - typeChangeInfos) - } else { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new VectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity) - } - - // SPARK-37089: We cannot register a task completion listener to close this iterator here - // because downstream exec nodes have already registered their listeners. Since listeners - // are executed in reverse order of registration, a listener registered here would close the - // iterator while downstream exec nodes are still running. When off-heap column vectors are - // enabled, this can cause a use-after-free bug leading to a segfault. - // - // Instead, we use FileScanRDD's task completion listener to close this iterator. - val iter = new RecordReaderIterator(vectorizedReader) - try { - vectorizedReader.initialize(split, hadoopAttemptContext) - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (shouldAppendPartitionValues) { - logDebug(s"Appending $partitionSchema ${file.partitionValues}") - vectorizedReader.initBatch(partitionSchema, file.partitionValues) - } else { - vectorizedReader.initBatch(StructType(Nil), InternalRow.empty) - } - - if (returningBatch) { - vectorizedReader.enableReturningBatches() - } - - // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. - iter.asInstanceOf[Iterator[InternalRow]] - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } else { - logDebug(s"Falling back to parquet-mr") - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val readSupport = new HoodieParquetReadSupport( - convertTz, - enableVectorizedReader = false, - enableTimestampFieldRepair = true, - datetimeRebaseSpec, - int96RebaseSpec) - - val reader = if (pushed.isDefined && enableRecordFilter) { - val parquetFilter = FilterCompat.get(pushed.get, null) - new ParquetRecordReader[InternalRow](readSupport, parquetFilter) - } else { - new ParquetRecordReader[InternalRow](readSupport) - } - val iter = new RecordReaderIterator[InternalRow](reader) - try { - reader.initialize(split, hadoopAttemptContext) - - val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes - val unsafeProjection = if (typeChangeInfos.isEmpty) { - GenerateUnsafeProjection.generate(fullSchema, fullSchema) - } else { - // find type changed. - val newFullSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) => - if (typeChangeInfos.containsKey(i)) { - StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata) - } else f - }).toAttributes ++ partitionSchema.toAttributes - val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) => - if (typeChangeInfos.containsKey(i)) { - val srcType = typeChangeInfos.get(i).getRight - val dstType = typeChangeInfos.get(i).getLeft - val needTimeZone = Cast.needsTimeZone(srcType, dstType) - Cast(attr, dstType, if (needTimeZone) timeZoneId else None) - } else attr - } - GenerateUnsafeProjection.generate(castSchema, newFullSchema) - } - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (!shouldAppendPartitionValues || partitionSchema.length == 0) { - // There is no partition columns - iter.map(unsafeProjection) - } else { - val joinedRow = new JoinedRow() - iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) - } - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } - } - } -} - -object Spark33LegacyHoodieParquetFileFormat { - - def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = { - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - if (querySchemaOption.isPresent && requiredSchema.nonEmpty) { - val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get()) - SerDeHelper.toJson(prunedSchema) - } else { - internalSchemaStr - } - } - - private def rebuildFilterFromParquet(oldFilter: Filter, fileSchema: InternalSchema, querySchema: InternalSchema): Filter = { - if (fileSchema == null || querySchema == null) { - oldFilter - } else { - oldFilter match { - case eq: EqualTo => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eq.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eq.copy(attribute = newAttribute) - case eqs: EqualNullSafe => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eqs.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eqs.copy(attribute = newAttribute) - case gt: GreaterThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gt.copy(attribute = newAttribute) - case gtr: GreaterThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gtr.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gtr.copy(attribute = newAttribute) - case lt: LessThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lt.copy(attribute = newAttribute) - case lte: LessThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lte.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lte.copy(attribute = newAttribute) - case i: In => - val newAttribute = InternalSchemaUtils.reBuildFilterName(i.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else i.copy(attribute = newAttribute) - case isn: IsNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isn.copy(attribute = newAttribute) - case isnn: IsNotNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isnn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isnn.copy(attribute = newAttribute) - case And(left, right) => - And(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Or(left, right) => - Or(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Not(child) => - Not(rebuildFilterFromParquet(child, fileSchema, querySchema)) - case ssw: StringStartsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ssw.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ssw.copy(attribute = newAttribute) - case ses: StringEndsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ses.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ses.copy(attribute = newAttribute) - case sc: StringContains => - val newAttribute = InternalSchemaUtils.reBuildFilterName(sc.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else sc.copy(attribute = newAttribute) - case AlwaysTrue => - AlwaysTrue - case AlwaysFalse => - AlwaysFalse - case _ => - AlwaysTrue - } - } } } diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala index adca01edf06bf..abcb6d5c8b90a 100644 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala @@ -17,54 +17,21 @@ package org.apache.spark.sql.execution.datasources.parquet -import org.apache.hudi.client.utils.SparkInternalSchemaConverter -import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.common.table.timeline.TimelineLayout -import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion -import org.apache.hudi.common.util.InternalSchemaCache -import org.apache.hudi.common.util.StringUtils.isNullOrEmpty -import org.apache.hudi.common.util.collection.Pair -import org.apache.hudi.hadoop.fs.HadoopFSUtils -import org.apache.hudi.internal.schema.InternalSchema -import org.apache.hudi.internal.schema.action.InternalSchemaMerger -import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper} -import org.apache.hudi.storage.HoodieStorageUtils - import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.mapred.FileSplit -import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType} -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS -import org.apache.parquet.hadoop.{ParquetInputFormat, ParquetRecordReader} -import org.apache.spark.TaskContext +import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Cast, JoinedRow} -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator} -import org.apache.spark.sql.execution.datasources.parquet.Spark34LegacyHoodieParquetFileFormat._ +import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType} -import org.apache.spark.util.SerializableConfiguration - -import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` +import org.apache.spark.sql.types.StructType /** - * This class is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior - * that's not possible to customize in any other way - * - * NOTE: This is a version of [[AvroDeserializer]] impl from Spark 3.2.1 w/ w/ the following changes applied to it: - *
      - *
    1. Avoiding appending partition values to the rows read from the data file
      2. - *
    2. Schema on-read
      3. - *
    + * Spark 3.4 concrete implementation of [[Spark3LegacyHoodieParquetFileFormat]]. It only overrides + * the version-specific hooks; the shared reader logic lives in the base class. */ -class Spark34LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValues: Boolean) extends ParquetFileFormat { +class Spark34LegacyHoodieParquetFileFormat(appendPartitionValues: Boolean) + extends Spark3LegacyHoodieParquetFileFormat(appendPartitionValues) { def supportsColumnar(sparkSession: SparkSession, schema: StructType): Boolean = { val conf = sparkSession.sessionState.conf @@ -75,39 +42,25 @@ class Spark34LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu supportBatch(sparkSession, schema) } - override def buildReaderWithPartitionValues(sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) - hadoopConf.set( - ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, - requiredSchema.json) - hadoopConf.set( - ParquetWriteSupport.SPARK_ROW_SCHEMA, - requiredSchema.json) - hadoopConf.set( - SQLConf.SESSION_LOCAL_TIMEZONE.key, - sparkSession.sessionState.conf.sessionLocalTimeZone) - hadoopConf.setBoolean( - SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, - sparkSession.sessionState.conf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean( - SQLConf.CASE_SENSITIVE.key, - sparkSession.sessionState.conf.caseSensitiveAnalysis) + override protected def toAttributes(structType: StructType): Seq[Attribute] = + structType.toAttributes - ParquetWriteSupport.setSchema(requiredSchema, hadoopConf) + override protected def getFilePath(file: PartitionedFile): Path = + file.filePath.toPath - // Sets flags for `ParquetToSparkSchemaConverter` - hadoopConf.setBoolean( - SQLConf.PARQUET_BINARY_AS_STRING.key, - sparkSession.sessionState.conf.isParquetBinaryAsString) - hadoopConf.setBoolean( - SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, - sparkSession.sessionState.conf.isParquetINT96AsTimestamp) + override protected def isVectorizedReaderEnabled(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + supportBatch(sparkSession, resultSchema) + + override protected def getPushDownStringPredicate(sqlConf: SQLConf): Boolean = + sqlConf.parquetFilterPushDownStringPredicate + + override protected def getReturningBatch(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + sparkSession.sessionState.conf.parquetVectorizedReaderEnabled && + supportsColumnar(sparkSession, resultSchema).toString.equals("true") + + override protected def setParquetTimeConfs(hadoopConf: Configuration, sparkSession: SparkSession): Unit = { // Using string value of this conf to preserve compatibility across spark versions. hadoopConf.setBoolean( SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key, @@ -117,346 +70,5 @@ class Spark34LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu ) hadoopConf.setBoolean(SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED.key, sparkSession.sessionState.conf.parquetInferTimestampNTZEnabled) hadoopConf.setBoolean(SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key, sparkSession.sessionState.conf.legacyParquetNanosAsLong) - val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself, - // therefore it's safe to do schema projection here - if (!isNullOrEmpty(internalSchemaStr)) { - val prunedInternalSchemaStr = - pruneInternalSchema(internalSchemaStr, requiredSchema) - hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr) - } - - val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - - // TODO: if you move this into the closure it reverts to the default values. - // If true, enable using the custom RecordReader for parquet. This only works for - // a subset of the types (no complex types). - val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) - val sqlConf = sparkSession.sessionState.conf - val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled - val enableVectorizedReader: Boolean = supportBatch(sparkSession, resultSchema) - val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled - val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion - val capacity = sqlConf.parquetVectorizedReaderBatchSize - val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown - val pushDownDate = sqlConf.parquetFilterPushDownDate - val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp - val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal - val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringPredicate - val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold - val isCaseSensitive = sqlConf.caseSensitiveAnalysis - val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) - val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead - val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead - val timeZoneId = Option(sqlConf.sessionLocalTimeZone) - // Should always be set by FileSourceScanExec creating this. - // Check conf before checking option, to allow working around an issue by changing conf. - val returningBatch = sparkSession.sessionState.conf.parquetVectorizedReaderEnabled && - supportsColumnar(sparkSession, resultSchema).toString.equals("true") - - - (file: PartitionedFile) => { - assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size) - - val filePath = file.filePath.toPath - val split = new FileSplit(filePath, file.start, file.length, Array.empty[String]) - - val sharedConf = broadcastedHadoopConf.value.value - - // Fetch internal schema - val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // Internal schema has to be pruned at this point - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - - var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent - - val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH) - val fileSchema = if (shouldUseInternalSchema) { - val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong; - val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST) - //TODO: HARDCODED TIMELINE OBJECT - val layout = TimelineLayout.fromVersion(TimelineLayoutVersion.CURR_LAYOUT_VERSION) - val storage = HoodieStorageUtils.getStorage(tablePath, HadoopFSUtils.getStorageConf(sharedConf)) - InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, tablePath, storage, - if (validCommits == null) "" else validCommits, - layout) - } else { - null - } - - lazy val footerFileMetaData = - ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData - // Try to push down filters when filter push-down is enabled. - val pushed = if (enableParquetFilterPushDown) { - val parquetSchema = footerFileMetaData.getSchema - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val parquetFilters = new ParquetFilters( - parquetSchema, - pushDownDate, - pushDownTimestamp, - pushDownDecimal, - pushDownStringStartWith, - pushDownInFilterThreshold, - isCaseSensitive, - datetimeRebaseSpec) - filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null))) - // Collects all converted Parquet filter predicates. Notice that not all predicates can be - // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` - // is used here. - .flatMap(parquetFilters.createFilter) - .reduceOption(FilterApi.and) - } else { - None - } - - // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' - // *only* if the file was created by something other than "parquet-mr", so check the actual - // writer here for this file. We have to do this per-file, as each file in the table may - // have different writers. - // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. - def isCreatedByParquetMr: Boolean = - footerFileMetaData.getCreatedBy().startsWith("parquet-mr") - - val convertTz = - if (timestampConversion && !isCreatedByParquetMr) { - Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) - } else { - None - } - - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - - // Clone new conf - val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value) - val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) { - val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema() - val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema) - - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json) - - SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema) - } else { - val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema) - if (!implicitTypeChangeInfo.isEmpty) { - shouldUseInternalSchema = true - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json) - } - implicitTypeChangeInfo - } - - if (enableVectorizedReader && shouldUseInternalSchema && - !typeChangeInfos.values().forall(_.getLeft.isInstanceOf[AtomicType])) { - throw new IllegalArgumentException( - "Nested types with type changes(implicit or explicit) cannot be read in vectorized mode. " + - "To workaround this issue, set spark.sql.parquet.enableVectorizedReader=false.") - } - - val hadoopAttemptContext = - new TaskAttemptContextImpl(hadoopAttemptConf, attemptId) - - // Try to push down filters when filter push-down is enabled. - // Notice: This push-down is RowGroups level, not individual records. - if (pushed.isDefined) { - ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) - } - val taskContext = Option(TaskContext.get()) - if (enableVectorizedReader) { - val vectorizedReader = - if (shouldUseInternalSchema) { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new HoodieVectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity, - typeChangeInfos) - } else { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new VectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity) - } - - // SPARK-37089: We cannot register a task completion listener to close this iterator here - // because downstream exec nodes have already registered their listeners. Since listeners - // are executed in reverse order of registration, a listener registered here would close the - // iterator while downstream exec nodes are still running. When off-heap column vectors are - // enabled, this can cause a use-after-free bug leading to a segfault. - // - // Instead, we use FileScanRDD's task completion listener to close this iterator. - val iter = new RecordReaderIterator(vectorizedReader) - try { - vectorizedReader.initialize(split, hadoopAttemptContext) - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (shouldAppendPartitionValues) { - logDebug(s"Appending $partitionSchema ${file.partitionValues}") - vectorizedReader.initBatch(partitionSchema, file.partitionValues) - } else { - vectorizedReader.initBatch(StructType(Nil), InternalRow.empty) - } - - if (returningBatch) { - vectorizedReader.enableReturningBatches() - } - - // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. - iter.asInstanceOf[Iterator[InternalRow]] - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } else { - logDebug(s"Falling back to parquet-mr") - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val readSupport = new HoodieParquetReadSupport( - convertTz, - enableVectorizedReader = false, - enableTimestampFieldRepair = true, - datetimeRebaseSpec, - int96RebaseSpec) - - val reader = if (pushed.isDefined && enableRecordFilter) { - val parquetFilter = FilterCompat.get(pushed.get, null) - new ParquetRecordReader[InternalRow](readSupport, parquetFilter) - } else { - new ParquetRecordReader[InternalRow](readSupport) - } - val iter = new RecordReaderIterator[InternalRow](reader) - try { - reader.initialize(split, hadoopAttemptContext) - - val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes - val unsafeProjection = if (typeChangeInfos.isEmpty) { - GenerateUnsafeProjection.generate(fullSchema, fullSchema) - } else { - // find type changed. - val newFullSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) => - if (typeChangeInfos.containsKey(i)) { - StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata) - } else f - }).toAttributes ++ partitionSchema.toAttributes - val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) => - if (typeChangeInfos.containsKey(i)) { - val srcType = typeChangeInfos.get(i).getRight - val dstType = typeChangeInfos.get(i).getLeft - val needTimeZone = Cast.needsTimeZone(srcType, dstType) - Cast(attr, dstType, if (needTimeZone) timeZoneId else None) - } else attr - } - GenerateUnsafeProjection.generate(castSchema, newFullSchema) - } - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (!shouldAppendPartitionValues || partitionSchema.length == 0) { - // There is no partition columns - iter.map(unsafeProjection) - } else { - val joinedRow = new JoinedRow() - iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) - } - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } - } - } -} - -object Spark34LegacyHoodieParquetFileFormat { - - def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = { - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - if (querySchemaOption.isPresent && requiredSchema.nonEmpty) { - val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get()) - SerDeHelper.toJson(prunedSchema) - } else { - internalSchemaStr - } - } - - private def rebuildFilterFromParquet(oldFilter: Filter, fileSchema: InternalSchema, querySchema: InternalSchema): Filter = { - if (fileSchema == null || querySchema == null) { - oldFilter - } else { - oldFilter match { - case eq: EqualTo => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eq.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eq.copy(attribute = newAttribute) - case eqs: EqualNullSafe => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eqs.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eqs.copy(attribute = newAttribute) - case gt: GreaterThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gt.copy(attribute = newAttribute) - case gtr: GreaterThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gtr.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gtr.copy(attribute = newAttribute) - case lt: LessThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lt.copy(attribute = newAttribute) - case lte: LessThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lte.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lte.copy(attribute = newAttribute) - case i: In => - val newAttribute = InternalSchemaUtils.reBuildFilterName(i.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else i.copy(attribute = newAttribute) - case isn: IsNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isn.copy(attribute = newAttribute) - case isnn: IsNotNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isnn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isnn.copy(attribute = newAttribute) - case And(left, right) => - And(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Or(left, right) => - Or(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Not(child) => - Not(rebuildFilterFromParquet(child, fileSchema, querySchema)) - case ssw: StringStartsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ssw.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ssw.copy(attribute = newAttribute) - case ses: StringEndsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ses.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ses.copy(attribute = newAttribute) - case sc: StringContains => - val newAttribute = InternalSchemaUtils.reBuildFilterName(sc.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else sc.copy(attribute = newAttribute) - case AlwaysTrue => - AlwaysTrue - case AlwaysFalse => - AlwaysFalse - case _ => - AlwaysTrue - } - } } } diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala index 6e062b1319bfb..b6887c8814960 100644 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala @@ -17,55 +17,22 @@ package org.apache.spark.sql.execution.datasources.parquet -import org.apache.hudi.client.utils.SparkInternalSchemaConverter -import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.common.table.timeline.TimelineLayout -import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion -import org.apache.hudi.common.util.InternalSchemaCache -import org.apache.hudi.common.util.StringUtils.isNullOrEmpty -import org.apache.hudi.common.util.collection.Pair -import org.apache.hudi.hadoop.fs.HadoopFSUtils -import org.apache.hudi.internal.schema.InternalSchema -import org.apache.hudi.internal.schema.action.InternalSchemaMerger -import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper} -import org.apache.hudi.storage.HoodieStorageUtils - import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.mapred.FileSplit -import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType} -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS -import org.apache.parquet.hadoop.{ParquetInputFormat, ParquetRecordReader} -import org.apache.spark.TaskContext +import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Cast, JoinedRow} -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator} -import org.apache.spark.sql.execution.datasources.parquet.Spark35LegacyHoodieParquetFileFormat._ +import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType} -import org.apache.spark.util.SerializableConfiguration - -import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` +import org.apache.spark.sql.types.StructType /** - * This class is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior - * that's not possible to customize in any other way - * - * NOTE: This is a version of [[AvroDeserializer]] impl from Spark 3.2.1 w/ w/ the following changes applied to it: - *
      - *
    1. Avoiding appending partition values to the rows read from the data file
      2. - *
    2. Schema on-read
      3. - *
    + * Spark 3.5 concrete implementation of [[Spark3LegacyHoodieParquetFileFormat]]. It only overrides + * the version-specific hooks; the shared reader logic lives in the base class. */ -class Spark35LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValues: Boolean) extends ParquetFileFormat { +class Spark35LegacyHoodieParquetFileFormat(appendPartitionValues: Boolean) + extends Spark3LegacyHoodieParquetFileFormat(appendPartitionValues) { def supportsColumnar(sparkSession: SparkSession, schema: StructType): Boolean = { val conf = sparkSession.sessionState.conf @@ -76,39 +43,25 @@ class Spark35LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu supportBatch(sparkSession, schema) } - override def buildReaderWithPartitionValues(sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) - hadoopConf.set( - ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, - requiredSchema.json) - hadoopConf.set( - ParquetWriteSupport.SPARK_ROW_SCHEMA, - requiredSchema.json) - hadoopConf.set( - SQLConf.SESSION_LOCAL_TIMEZONE.key, - sparkSession.sessionState.conf.sessionLocalTimeZone) - hadoopConf.setBoolean( - SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, - sparkSession.sessionState.conf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean( - SQLConf.CASE_SENSITIVE.key, - sparkSession.sessionState.conf.caseSensitiveAnalysis) + override protected def toAttributes(structType: StructType): Seq[Attribute] = + DataTypeUtils.toAttributes(structType) - ParquetWriteSupport.setSchema(requiredSchema, hadoopConf) + override protected def getFilePath(file: PartitionedFile): Path = + file.filePath.toPath - // Sets flags for `ParquetToSparkSchemaConverter` - hadoopConf.setBoolean( - SQLConf.PARQUET_BINARY_AS_STRING.key, - sparkSession.sessionState.conf.isParquetBinaryAsString) - hadoopConf.setBoolean( - SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, - sparkSession.sessionState.conf.isParquetINT96AsTimestamp) + override protected def isVectorizedReaderEnabled(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + supportBatch(sparkSession, resultSchema) + + override protected def getPushDownStringPredicate(sqlConf: SQLConf): Boolean = + sqlConf.parquetFilterPushDownStringPredicate + + override protected def getReturningBatch(sparkSession: SparkSession, + resultSchema: StructType): Boolean = + sparkSession.sessionState.conf.parquetVectorizedReaderEnabled && + supportsColumnar(sparkSession, resultSchema).toString.equals("true") + + override protected def setParquetTimeConfs(hadoopConf: Configuration, sparkSession: SparkSession): Unit = { // Using string value of this conf to preserve compatibility across spark versions. hadoopConf.setBoolean( SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key, @@ -118,347 +71,5 @@ class Spark35LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu ) hadoopConf.setBoolean(SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED.key, sparkSession.sessionState.conf.parquetInferTimestampNTZEnabled) hadoopConf.setBoolean(SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key, sparkSession.sessionState.conf.legacyParquetNanosAsLong) - val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself, - // therefore it's safe to do schema projection here - if (!isNullOrEmpty(internalSchemaStr)) { - val prunedInternalSchemaStr = - pruneInternalSchema(internalSchemaStr, requiredSchema) - hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr) - } - - val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - - // TODO: if you move this into the closure it reverts to the default values. - // If true, enable using the custom RecordReader for parquet. This only works for - // a subset of the types (no complex types). - val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) - val sqlConf = sparkSession.sessionState.conf - val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled - val enableVectorizedReader: Boolean = supportBatch(sparkSession, resultSchema) - val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled - val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion - val capacity = sqlConf.parquetVectorizedReaderBatchSize - val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown - val pushDownDate = sqlConf.parquetFilterPushDownDate - val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp - val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal - val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringPredicate - val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold - val isCaseSensitive = sqlConf.caseSensitiveAnalysis - val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) - val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead - val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead - val timeZoneId = Option(sqlConf.sessionLocalTimeZone) - // Should always be set by FileSourceScanExec creating this. - // Check conf before checking option, to allow working around an issue by changing conf. - val returningBatch = sparkSession.sessionState.conf.parquetVectorizedReaderEnabled && - supportsColumnar(sparkSession, resultSchema).toString.equals("true") - - - (file: PartitionedFile) => { - assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size) - - val filePath = file.filePath.toPath - val split = new FileSplit(filePath, file.start, file.length, Array.empty[String]) - - val sharedConf = broadcastedHadoopConf.value.value - - // Fetch internal schema - val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA) - // Internal schema has to be pruned at this point - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - - var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent - - val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH) - val fileSchema = if (shouldUseInternalSchema) { - val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong; - val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST) - val storage = HoodieStorageUtils.getStorage(tablePath, HadoopFSUtils.getStorageConf(sharedConf)) - //TODO: HARDCODED TIMELINE OBJECT - val layout = TimelineLayout.fromVersion(TimelineLayoutVersion.CURR_LAYOUT_VERSION) - InternalSchemaCache.getInternalSchemaByVersionId( - commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits, - layout) - } else { - null - } - - lazy val footerFileMetaData = - ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData - // Try to push down filters when filter push-down is enabled. - val pushed = if (enableParquetFilterPushDown) { - val parquetSchema = footerFileMetaData.getSchema - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val parquetFilters = new ParquetFilters( - parquetSchema, - pushDownDate, - pushDownTimestamp, - pushDownDecimal, - pushDownStringStartWith, - pushDownInFilterThreshold, - isCaseSensitive, - datetimeRebaseSpec) - filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null))) - // Collects all converted Parquet filter predicates. Notice that not all predicates can be - // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` - // is used here. - .flatMap(parquetFilters.createFilter) - .reduceOption(FilterApi.and) - } else { - None - } - - // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' - // *only* if the file was created by something other than "parquet-mr", so check the actual - // writer here for this file. We have to do this per-file, as each file in the table may - // have different writers. - // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. - def isCreatedByParquetMr: Boolean = - footerFileMetaData.getCreatedBy().startsWith("parquet-mr") - - val convertTz = - if (timestampConversion && !isCreatedByParquetMr) { - Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) - } else { - None - } - - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - - // Clone new conf - val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value) - val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) { - val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema() - val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema) - - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json) - - SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema) - } else { - val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema) - if (!implicitTypeChangeInfo.isEmpty) { - shouldUseInternalSchema = true - hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json) - } - implicitTypeChangeInfo - } - - if (enableVectorizedReader && shouldUseInternalSchema && - !typeChangeInfos.values().forall(_.getLeft.isInstanceOf[AtomicType])) { - throw new IllegalArgumentException( - "Nested types with type changes(implicit or explicit) cannot be read in vectorized mode. " + - "To workaround this issue, set spark.sql.parquet.enableVectorizedReader=false.") - } - - val hadoopAttemptContext = - new TaskAttemptContextImpl(hadoopAttemptConf, attemptId) - - // Try to push down filters when filter push-down is enabled. - // Notice: This push-down is RowGroups level, not individual records. - if (pushed.isDefined) { - ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) - } - val taskContext = Option(TaskContext.get()) - if (enableVectorizedReader) { - val vectorizedReader = - if (shouldUseInternalSchema) { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new HoodieVectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity, - typeChangeInfos) - } else { - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - new VectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity) - } - - // SPARK-37089: We cannot register a task completion listener to close this iterator here - // because downstream exec nodes have already registered their listeners. Since listeners - // are executed in reverse order of registration, a listener registered here would close the - // iterator while downstream exec nodes are still running. When off-heap column vectors are - // enabled, this can cause a use-after-free bug leading to a segfault. - // - // Instead, we use FileScanRDD's task completion listener to close this iterator. - val iter = new RecordReaderIterator(vectorizedReader) - try { - vectorizedReader.initialize(split, hadoopAttemptContext) - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (shouldAppendPartitionValues) { - logDebug(s"Appending $partitionSchema ${file.partitionValues}") - vectorizedReader.initBatch(partitionSchema, file.partitionValues) - } else { - vectorizedReader.initBatch(StructType(Nil), InternalRow.empty) - } - - if (returningBatch) { - vectorizedReader.enableReturningBatches() - } - - // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. - iter.asInstanceOf[Iterator[InternalRow]] - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } else { - logDebug(s"Falling back to parquet-mr") - val int96RebaseSpec = - DataSourceUtils.int96RebaseSpec(footerFileMetaData.getKeyValueMetaData.get, int96RebaseModeInRead) - val datetimeRebaseSpec = - DataSourceUtils.datetimeRebaseSpec(footerFileMetaData.getKeyValueMetaData.get, datetimeRebaseModeInRead) - val readSupport = new HoodieParquetReadSupport( - convertTz, - enableVectorizedReader = false, - enableTimestampFieldRepair = true, - datetimeRebaseSpec, - int96RebaseSpec) - - val reader = if (pushed.isDefined && enableRecordFilter) { - val parquetFilter = FilterCompat.get(pushed.get, null) - new ParquetRecordReader[InternalRow](readSupport, parquetFilter) - } else { - new ParquetRecordReader[InternalRow](readSupport) - } - val iter = new RecordReaderIterator[InternalRow](reader) - try { - reader.initialize(split, hadoopAttemptContext) - - val fullSchema = DataTypeUtils.toAttributes(requiredSchema) ++ DataTypeUtils.toAttributes(partitionSchema) - val unsafeProjection = if (typeChangeInfos.isEmpty) { - GenerateUnsafeProjection.generate(fullSchema, fullSchema) - } else { - // find type changed. - val newSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) => - if (typeChangeInfos.containsKey(i)) { - StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata) - } else f - }) - val newFullSchema = DataTypeUtils.toAttributes(newSchema) ++ DataTypeUtils.toAttributes(partitionSchema) - val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) => - if (typeChangeInfos.containsKey(i)) { - val srcType = typeChangeInfos.get(i).getRight - val dstType = typeChangeInfos.get(i).getLeft - val needTimeZone = Cast.needsTimeZone(srcType, dstType) - Cast(attr, dstType, if (needTimeZone) timeZoneId else None) - } else attr - } - GenerateUnsafeProjection.generate(castSchema, newFullSchema) - } - - // NOTE: We're making appending of the partitioned values to the rows read from the - // data file configurable - if (!shouldAppendPartitionValues || partitionSchema.length == 0) { - // There is no partition columns - iter.map(unsafeProjection) - } else { - val joinedRow = new JoinedRow() - iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) - } - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } - } - } -} - -object Spark35LegacyHoodieParquetFileFormat { - - def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = { - val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr) - if (querySchemaOption.isPresent && requiredSchema.nonEmpty) { - val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get()) - SerDeHelper.toJson(prunedSchema) - } else { - internalSchemaStr - } - } - - private def rebuildFilterFromParquet(oldFilter: Filter, fileSchema: InternalSchema, querySchema: InternalSchema): Filter = { - if (fileSchema == null || querySchema == null) { - oldFilter - } else { - oldFilter match { - case eq: EqualTo => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eq.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eq.copy(attribute = newAttribute) - case eqs: EqualNullSafe => - val newAttribute = InternalSchemaUtils.reBuildFilterName(eqs.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else eqs.copy(attribute = newAttribute) - case gt: GreaterThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gt.copy(attribute = newAttribute) - case gtr: GreaterThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(gtr.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else gtr.copy(attribute = newAttribute) - case lt: LessThan => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lt.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lt.copy(attribute = newAttribute) - case lte: LessThanOrEqual => - val newAttribute = InternalSchemaUtils.reBuildFilterName(lte.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else lte.copy(attribute = newAttribute) - case i: In => - val newAttribute = InternalSchemaUtils.reBuildFilterName(i.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else i.copy(attribute = newAttribute) - case isn: IsNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isn.copy(attribute = newAttribute) - case isnn: IsNotNull => - val newAttribute = InternalSchemaUtils.reBuildFilterName(isnn.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else isnn.copy(attribute = newAttribute) - case And(left, right) => - And(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Or(left, right) => - Or(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema)) - case Not(child) => - Not(rebuildFilterFromParquet(child, fileSchema, querySchema)) - case ssw: StringStartsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ssw.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ssw.copy(attribute = newAttribute) - case ses: StringEndsWith => - val newAttribute = InternalSchemaUtils.reBuildFilterName(ses.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else ses.copy(attribute = newAttribute) - case sc: StringContains => - val newAttribute = InternalSchemaUtils.reBuildFilterName(sc.attribute, fileSchema, querySchema) - if (newAttribute.isEmpty) AlwaysTrue else sc.copy(attribute = newAttribute) - case AlwaysTrue => - AlwaysTrue - case AlwaysFalse => - AlwaysFalse - case _ => - AlwaysTrue - } - } } } From 4236e7064ec341959c52e0f5960ecb6c4e69d459 Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 14:36:40 +0800 Subject: [PATCH 183/255] fix(build): move TestSparkValueMetadataUtils to the release-branch stats package TestSparkValueMetadataUtils, added by 1eefd1b0a274 (#19219), was placed in org.apache.hudi.metadata.stats, where master keeps SparkValueMetadataUtils, ValueMetadata and ValueType, so it resolved all three same-package with no imports. release-1.2.1 still has those classes in org.apache.hudi.stats, so the references did not resolve and hudi-spark-client failed to test-compile with "cannot find symbol: class ValueMetadata / class ValueType". Move the test to org/apache/hudi/stats/ and set its package accordingly. That matches where the class under test lives here, puts it alongside the existing TestValueMetadata and TestValueType, and removes the otherwise empty org/apache/hudi/metadata/stats directory this branch does not use. The test only touches public API, so no imports are needed. --- .../hudi/{metadata => }/stats/TestSparkValueMetadataUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/{metadata => }/stats/TestSparkValueMetadataUtils.java (99%) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/stats/TestSparkValueMetadataUtils.java similarity index 99% rename from hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java rename to hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/stats/TestSparkValueMetadataUtils.java index 43c85b2e8b7a9..45f6ad3dd3fa8 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/stats/TestSparkValueMetadataUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/stats/TestSparkValueMetadataUtils.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hudi.metadata.stats; +package org.apache.hudi.stats; import org.apache.hudi.metadata.HoodieIndexVersion; From d63a00d5f01e2081b52315fb042ee741a6d4b32e Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Wed, 22 Jul 2026 02:38:55 -0700 Subject: [PATCH 184/255] [HUDI-18060] Improve error message when ordering field value is null (#18061) When records have a null value in the ordering (precombine) field, Hudi jobs previously failed with a cryptic "Ordering value is null for record" error that gave no actionable context. This change fails fast in HoodieCreateRecordUtils with a clear message identifying the offending ordering field and record key, and suggesting remediation. Merge modes that do not depend on the ordering value are exempted: COMMIT_TIME_ORDERING and OverwriteWithLatestAvroPayload fall back to OrderingValues.getDefault() instead of failing. The payload-class check is retained so table version 6 (which may not have a merge mode set) still bypasses the failure for OverwriteWithLatestAvroPayload. The gating flag is computed once at driver scope rather than per record to avoid repeated work on the per-record hot path. Closes #18060 (cherry picked from commit a18f22126dfa029b59d68cf53c4d1a3b75be0e33) --- .../apache/hudi/HoodieCreateRecordUtils.scala | 45 ++- .../hudi/TestHoodieCreateRecordUtils.scala | 291 ++++++++++++++++++ 2 files changed, 330 insertions(+), 6 deletions(-) create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCreateRecordUtils.scala diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala index df5f495a2be3a..a9e4f903b1946 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala @@ -20,7 +20,7 @@ package org.apache.hudi import org.apache.hudi.DataSourceWriteOptions.INSERT_DROP_DUPS import org.apache.hudi.avro.{AvroRecordContext, AvroSchemaCache, HoodieAvroUtils} -import org.apache.hudi.common.config.TypedProperties +import org.apache.hudi.common.config.{RecordMergeMode, TypedProperties} import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model._ import org.apache.hudi.common.model.WriteOperationType.isChangingRecords @@ -78,6 +78,12 @@ object HoodieCreateRecordUtils { val preppedSparkSqlMergeInto = args.preppedSparkSqlMergeInto val preppedWriteOperation = args.preppedWriteOperation val orderingFields = args.tableConfig.getOrderingFields + val recordMergeMode = args.tableConfig.getRecordMergeMode + val payloadClass = config.getPayloadClass + // Ordering values are not required for COMMIT_TIME_ORDERING or OverwriteWithLatestAvroPayload. + // Table version 6 may not have a merge mode set, so the payload class check is still needed. + val requiresOrderingValue = !((recordMergeMode == RecordMergeMode.COMMIT_TIME_ORDERING) + || classOf[OverwriteWithLatestAvroPayload].getName.equals(payloadClass)) val shouldDropPartitionColumns = config.getBoolean(DataSourceWriteOptions.DROP_PARTITION_COLUMNS) val recordType = config.getRecordMerger.getRecordType @@ -150,11 +156,8 @@ object HoodieCreateRecordUtils { avroRecWithoutMeta } val hoodieRecord = if (shouldCombine && !orderingFields.isEmpty) { - val orderingVal = OrderingValues.create( - orderingFields, - JFunction.toJavaFunction[String, Comparable[_]]( - field => HoodieAvroUtils.getNestedFieldVal(avroRec, field, false, - consistentLogicalTimestampEnabled).asInstanceOf[Comparable[_]])) + val orderingVal = getOrderingValue(orderingFields, avroRec, hoodieKey.getRecordKey, + consistentLogicalTimestampEnabled, requiresOrderingValue) HoodieRecordUtils.createHoodieRecord(processedRecord, orderingVal, hoodieKey, config.getPayloadClass, null, recordLocation, requiresPayload, isDelete) } else { @@ -282,4 +285,34 @@ object HoodieCreateRecordUtils { (new HoodieKey(recordKey, partitionPath), recordLocation) } + + /** + * Gets the ordering value from the ordering fields of an Avro record. + * When `requiresOrderingValue` is false (e.g., COMMIT_TIME_ORDERING or OverwriteWithLatestAvroPayload), + * null values are allowed and a default ordering value is used. + * Otherwise, throws IllegalArgumentException if any ordering field has a null value. + */ + private def getOrderingValue(orderingFields: java.util.List[String], + avroRec: GenericRecord, + recordKey: String, + consistentLogicalTimestampEnabled: Boolean, + requiresOrderingValue: Boolean): Comparable[_] = { + OrderingValues.create( + orderingFields, + JFunction.toJavaFunction[String, Comparable[_]](field => { + val fieldVal = HoodieAvroUtils.getNestedFieldVal(avroRec, field, false, consistentLogicalTimestampEnabled) + if (fieldVal == null) { + if (requiresOrderingValue) { + throw new IllegalArgumentException( + s"Ordering field '$field' has null value for record key '$recordKey'. " + + s"Please ensure all records have non-null values for the ordering field, " + + s"or use a payload class that doesn't require ordering (e.g., OverwriteWithLatestAvroPayload).") + } + // Return default ordering value for payloads that don't require ordering + OrderingValues.getDefault.asInstanceOf[Comparable[_]] + } else { + fieldVal.asInstanceOf[Comparable[_]] + } + })) + } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCreateRecordUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCreateRecordUtils.scala new file mode 100644 index 0000000000000..74e800534c436 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCreateRecordUtils.scala @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi + +import org.apache.hudi.common.config.RecordMergeMode +import org.apache.hudi.common.model.WriteOperationType +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.keygen.constant.KeyGeneratorOptions + +import org.apache.spark.SparkException +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.types._ +import org.junit.jupiter.api.{AfterAll, BeforeAll, Test} +import org.junit.jupiter.api.Assertions.{assertNotNull, assertTrue} + +/** + * Test cases for {@link HoodieCreateRecordUtils}. + */ +class TestHoodieCreateRecordUtils { + + private val SPARK_SCHEMA = StructType(Seq( + StructField("uuid", StringType, nullable = false), + StructField("name", StringType, nullable = false), + StructField("age", IntegerType, nullable = false), + StructField("ts", LongType, nullable = true), + StructField("partition", StringType, nullable = false) + )) + + // Common test constants + private val TEST_TABLE_NAME = "test_table" + private val RECORD_NAME = "TestRecord" + private val RECORD_NAMESPACE = "org.apache.hudi.test" + private val INSTANT_TIME = "20231031000000" + private val RECORD_KEY_FIELD = "uuid" + private val PARTITION_FIELD = "partition" + private val PRECOMBINE_FIELD = "ts" + + /** + * Helper method to create DataFrame from Row data + */ + private def createTestDataFrame(rows: Row*): org.apache.spark.sql.DataFrame = { + val spark = TestHoodieCreateRecordUtils.spark + spark.createDataFrame(spark.sparkContext.parallelize(rows), SPARK_SCHEMA) + } + + /** + * Helper method to get the root cause of an exception. + * Iterative implementation to avoid stack overflow and handle circular references. + * + * @param t The throwable to extract root cause from + * @return The root cause throwable + */ + private def getRootCause(t: Throwable): Throwable = { + var current = t + val visited = scala.collection.mutable.Set[Throwable]() + + while (current.getCause != null && !visited.contains(current)) { + visited += current + current = current.getCause + } + + current + } + + /** + * Helper method to create base parameters common to all tests. + * These are the mandatory properties required by SimpleKeyGenerator. + */ + private def createBaseParameters(): Map[String, String] = { + Map( + // KeyGeneratorOptions (used by some parts of the pipeline) + KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key() -> RECORD_KEY_FIELD, + KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key() -> PARTITION_FIELD, + // DataSourceWriteOptions (required by SimpleKeyGenerator) + DataSourceWriteOptions.RECORDKEY_FIELD.key() -> RECORD_KEY_FIELD, + DataSourceWriteOptions.PARTITIONPATH_FIELD.key() -> PARTITION_FIELD + ) + } + + /** + * Helper method to create common parameters for tests with precombine + */ + private def createParametersWithPrecombine(payloadClass: String = "org.apache.hudi.common.model.DefaultHoodieRecordPayload"): Map[String, String] = { + createBaseParameters() ++ Map( + DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> PRECOMBINE_FIELD, + DataSourceWriteOptions.PAYLOAD_CLASS_NAME.key() -> payloadClass, + HoodieWriteConfig.COMBINE_BEFORE_UPSERT.key() -> "true", + DataSourceWriteOptions.INSERT_DROP_DUPS.key() -> "false" + ) + } + + /** + * Helper method to create parameters for tests without precombine + */ + private def createParametersWithoutPrecombine(): Map[String, String] = { + createBaseParameters() ++ Map( + DataSourceWriteOptions.PAYLOAD_CLASS_NAME.key() -> "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload", + HoodieWriteConfig.COMBINE_BEFORE_INSERT.key() -> "false", + DataSourceWriteOptions.INSERT_DROP_DUPS.key() -> "false" + ) + } + + @Test + def testNullPrecombineFieldThrowsClearError(): Unit = { + val df = createTestDataFrame(Row("id1", "Alice", 25, null, "par1")) + val parameters = createParametersWithPrecombine() + + val exception = try { + // Attempt to write which will trigger HoodieCreateRecordUtils + df.write + .format("hudi") + .options(parameters) + .option(DataSourceWriteOptions.TABLE_NAME.key(), TEST_TABLE_NAME) + .option("hoodie.table.name", TEST_TABLE_NAME) + .option("path", TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine") + .mode("overwrite") + .save() + null + } catch { + case e: SparkException => + getRootCause(e) match { + case iae: IllegalArgumentException => iae + case other => other + } + case e: IllegalArgumentException => e + case e: Exception => + getRootCause(e) match { + case iae: IllegalArgumentException => iae + case _ => throw e + } + } + + assertNotNull(exception, "Expected IllegalArgumentException for null precombine field") + assertTrue(exception.isInstanceOf[IllegalArgumentException], + s"Expected IllegalArgumentException but got ${exception.getClass.getName}") + assertTrue(exception.getMessage.contains("has null value for record key"), + s"Exception message should mention null value for record key. Actual: ${exception.getMessage}") + assertTrue(exception.getMessage.contains("Please ensure all records have non-null values for the ordering field"), + s"Exception message should provide guidance. Actual: ${exception.getMessage}") + assertTrue(exception.getMessage.contains("OverwriteWithLatestAvroPayload"), + s"Exception message should suggest alternative payload class. Actual: ${exception.getMessage}") + } + + @Test + def testValidPrecombineFieldSucceeds(): Unit = { + val df = createTestDataFrame(Row("id1", "Alice", 25, 1000L, "par1")) + val parameters = createParametersWithPrecombine() + + // Should not throw exception + df.write + .format("hudi") + .options(parameters) + .option(DataSourceWriteOptions.TABLE_NAME.key(), TEST_TABLE_NAME) + .option("hoodie.table.name", TEST_TABLE_NAME) + .option("path", TestHoodieCreateRecordUtils.tempDir + "/test_valid_precombine") + .mode("overwrite") + .save() + + // Verify data was written + val result = TestHoodieCreateRecordUtils.spark.read + .format("hudi") + .load(TestHoodieCreateRecordUtils.tempDir + "/test_valid_precombine") + assertTrue(result.count() > 0, "Data should have been written successfully") + } + + @Test + def testNullPrecombineFieldErrorContainsRecordKey(): Unit = { + val testRecordKey = "test_key_123" + val df = createTestDataFrame(Row(testRecordKey, "Bob", 30, null, "par2")) + val parameters = createParametersWithPrecombine() + + val exception = try { + df.write + .format("hudi") + .options(parameters) + .option(DataSourceWriteOptions.TABLE_NAME.key(), TEST_TABLE_NAME) + .option("hoodie.table.name", TEST_TABLE_NAME) + .option("path", TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine_key") + .mode("overwrite") + .save() + null + } catch { + case e: Exception => + getRootCause(e) match { + case iae: IllegalArgumentException => iae + case _ => throw e + } + } + + assertNotNull(exception) + assertTrue(exception.getMessage.contains(testRecordKey), + s"Exception message should contain the record key '$testRecordKey' to help identify the problematic record. Actual: ${exception.getMessage}") + } + + @Test + def testNullPrecombineFieldWithOverwritePayloadSucceeds(): Unit = { + // OverwriteWithLatestAvroPayload should allow null precombine values + val df = createTestDataFrame(Row("id1", "Alice", 25, null, "par1")) + val parameters = createParametersWithPrecombine( + payloadClass = "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload") + + // Should not throw exception - OverwriteWithLatestAvroPayload doesn't require ordering values + df.write + .format("hudi") + .options(parameters) + .option(DataSourceWriteOptions.TABLE_NAME.key(), TEST_TABLE_NAME) + .option("hoodie.table.name", TEST_TABLE_NAME) + .option("path", TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine_overwrite") + .mode("overwrite") + .save() + + // Verify data was written + val result = TestHoodieCreateRecordUtils.spark.read + .format("hudi") + .load(TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine_overwrite") + assertTrue(result.count() > 0, "Data should have been written successfully with null precombine using OverwriteWithLatestAvroPayload") + } + + @Test + def testNullPrecombineFieldWithCommitTimeOrderingSucceeds(): Unit = { + // COMMIT_TIME_ORDERING merge mode should allow null precombine values + val df = createTestDataFrame(Row("id1", "Alice", 25, null, "par1")) + val parameters = createBaseParameters() ++ Map( + DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> PRECOMBINE_FIELD, + HoodieWriteConfig.COMBINE_BEFORE_UPSERT.key() -> "true", + DataSourceWriteOptions.INSERT_DROP_DUPS.key() -> "false", + DataSourceWriteOptions.RECORD_MERGE_MODE.key() -> RecordMergeMode.COMMIT_TIME_ORDERING.name() + ) + + // Should not throw exception - COMMIT_TIME_ORDERING doesn't require ordering values + df.write + .format("hudi") + .options(parameters) + .option(DataSourceWriteOptions.TABLE_NAME.key(), TEST_TABLE_NAME) + .option("hoodie.table.name", TEST_TABLE_NAME) + .option("path", TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine_commit_time") + .mode("overwrite") + .save() + + // Verify data was written + val result = TestHoodieCreateRecordUtils.spark.read + .format("hudi") + .load(TestHoodieCreateRecordUtils.tempDir + "/test_null_precombine_commit_time") + assertTrue(result.count() > 0, "Data should have been written successfully with null precombine using COMMIT_TIME_ORDERING") + } +} + +object TestHoodieCreateRecordUtils { + var spark: SparkSession = _ + var tempDir: String = _ + + @BeforeAll + def setupSpark(): Unit = { + tempDir = java.nio.file.Files.createTempDirectory("hudi_test_").toFile.getAbsolutePath + spark = SparkSession.builder() + .appName("TestHoodieCreateRecordUtils") + .master("local[2]") + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config("spark.sql.shuffle.partitions", "1") + .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.hudi.catalog.HoodieCatalog") + .getOrCreate() + } + + @AfterAll + def teardownSpark(): Unit = { + if (spark != null) { + spark.stop() + } + // Clean up temp directory + if (tempDir != null) { + org.apache.commons.io.FileUtils.deleteQuietly(new java.io.File(tempDir)) + } + } +} From 4b6081999c70012b7be547c123fe593252e77abf Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 22 Jul 2026 17:03:41 +0700 Subject: [PATCH 185/255] fix(hive-sync): keep HMS lock heartbeat alive and release its thread pool on close (#19334) * fix(hive-sync): keep HMS lock heartbeat alive and release its thread pool on close Fixes three defects in HiveMetastoreBasedLockProvider and its Heartbeat (package transaction/lock): - Heartbeat.run() no longer rethrows on failure. Under ScheduledExecutorService.scheduleAtFixedRate a thrown exception permanently cancels all subsequent executions (observable only through the unread ScheduledFuture), so one transient HMS/network hiccup silently stopped lock renewal while the writer still believed it held the lock. It now logs a warning with the cause and lets the next tick retry; the previous throw also discarded the cause. - close() moves executor.shutdown() into a finally block so the 2-thread scheduled pool is always released even when unlock()/Hive.closeCurrent() throws, and passes the caught exception to log.error instead of dropping it. - acquireLock recovery path: when the client times out on Future.get but the lock is granted server-side and recovered via checkLock, a heartbeat is now scheduled (via the extracted scheduleHeartbeat()) so the recovered lock is renewed instead of being left to expire mid-write. Adds unit tests TestHeartbeat and TestHiveMetastoreBasedLockProviderClose. * addressed review comments: drop the hardcoded thread count from the executor shutdown comment --------- Co-authored-by: Vova Kolmakov (cherry picked from commit 9f8376e3723d6e51f5e1a27f52f396b6181a6f6f) --- .../hudi/hive/transaction/lock/Heartbeat.java | 10 +- .../lock/HiveMetastoreBasedLockProvider.java | 28 +++-- .../hive/transaction/lock/TestHeartbeat.java | 55 +++++++++ ...stHiveMetastoreBasedLockProviderClose.java | 113 ++++++++++++++++++ 4 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java index f91b660380447..bf0037faf6df1 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java @@ -19,10 +19,10 @@ package org.apache.hudi.hive.transaction.lock; -import org.apache.hudi.exception.HoodieLockException; - +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +@Slf4j class Heartbeat implements Runnable { private final IMetaStoreClient client; private final long lockId; @@ -37,7 +37,11 @@ public void run() { try { client.heartbeat(0, lockId); } catch (Exception e) { - throw new HoodieLockException(String.format("Failed to heartbeat for lock: %d", lockId)); + // Do not rethrow. This task is scheduled via ScheduledExecutorService.scheduleAtFixedRate, + // where a thrown exception permanently cancels all subsequent executions and is only + // observable through the (unread) ScheduledFuture. Swallowing a transient failure here keeps + // the lock heartbeated on the next tick instead of silently stopping renewal altogether. + log.warn("Failed to heartbeat for lock: {}", lockId, e); } } } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java index 80a95801fbdbc..c83267be5319c 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java @@ -167,9 +167,12 @@ public void close() { future.cancel(false); } Hive.closeCurrent(); - executor.shutdown(); } catch (Exception e) { - log.error(generateLogStatement(org.apache.hudi.common.lock.LockState.FAILED_TO_RELEASE, generateLogSuffixString())); + log.error(generateLogStatement(org.apache.hudi.common.lock.LockState.FAILED_TO_RELEASE, generateLogSuffixString()), e); + } finally { + // Always release the heartbeat thread pool, even if unlock/closeCurrent above threw, + // otherwise its scheduled threads leak for the lifetime of the JVM. + executor.shutdown(); } } @@ -192,17 +195,15 @@ private void acquireLockInternal(long time, TimeUnit unit, LockComponent lockCom final LockRequest lockRequestFinal = lockRequest; this.lock = executor.submit(() -> hiveClient.lock(lockRequestFinal)) .get(time, unit); - - // refresh lock in case that certain commit takes a long time. - Heartbeat heartbeat = new Heartbeat(hiveClient, lock.getLockid()); - long heartbeatIntervalMs = lockConfiguration.getConfig() - .getLong(LOCK_HEARTBEAT_INTERVAL_MS_KEY, DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS); - future = executor.scheduleAtFixedRate(heartbeat, heartbeatIntervalMs / 2, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + scheduleHeartbeat(); } catch (InterruptedException | TimeoutException e) { if (this.lock == null || this.lock.getState() != LockState.ACQUIRED) { LockResponse lockResponse = this.hiveClient.checkLock(lockRequest.getTxnid()); if (lockResponse.getState() == LockState.ACQUIRED) { this.lock = lockResponse; + // The lock was granted server-side even though the client timed out waiting on the + // future; it still needs a heartbeat, otherwise a long-running commit lets HMS expire it. + scheduleHeartbeat(); } else { throw e; } @@ -219,6 +220,17 @@ private void acquireLockInternal(long time, TimeUnit unit, LockComponent lockCom } } + /** + * Schedules a periodic {@link Heartbeat} to refresh the currently held lock in case a commit + * takes a long time. Must be called only after {@link #lock} has been set. + */ + private void scheduleHeartbeat() { + Heartbeat heartbeat = new Heartbeat(hiveClient, lock.getLockid()); + long heartbeatIntervalMs = lockConfiguration.getConfig() + .getLong(LOCK_HEARTBEAT_INTERVAL_MS_KEY, DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS); + future = executor.scheduleAtFixedRate(heartbeat, heartbeatIntervalMs / 2, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + } + private void checkRequiredProps(final LockConfiguration lockConfiguration) { ValidationUtils.checkArgument(lockConfiguration.getConfig().getString(HIVE_DATABASE_NAME_PROP_KEY) != null); ValidationUtils.checkArgument(lockConfiguration.getConfig().getString(HIVE_TABLE_NAME_PROP_KEY) != null); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java new file mode 100644 index 0000000000000..ca2386f7a28f5 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.transaction.lock; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.thrift.TException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +class TestHeartbeat { + + @Test + void runDoesNotRethrowWhenHeartbeatFails() throws TException { + IMetaStoreClient client = mock(IMetaStoreClient.class); + doThrow(new TException("transient failure")).when(client).heartbeat(anyLong(), anyLong()); + + Heartbeat heartbeat = new Heartbeat(client, 7L); + + // Rethrowing here would cancel every subsequent execution of a scheduleAtFixedRate task, + // silently stopping lock renewal. The fix must swallow the failure so the next tick retries. + assertDoesNotThrow(heartbeat::run); + } + + @Test + void runHeartbeatsTheLockOnSuccess() throws TException { + IMetaStoreClient client = mock(IMetaStoreClient.class); + + new Heartbeat(client, 99L).run(); + + verify(client, times(1)).heartbeat(0L, 99L); + } +} diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java new file mode 100644 index 0000000000000..817c1407f3b9e --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.transaction.lock; + +import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.config.TypedProperties; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockLevel; +import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.LockType; +import org.apache.thrift.TException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.apache.hudi.common.config.LockConfiguration.HIVE_DATABASE_NAME_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.HIVE_TABLE_NAME_PROP_KEY; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link HiveMetastoreBasedLockProvider#close()} that exercise the thread-pool + * shutdown path with a mocked {@link IMetaStoreClient}, without a live metastore or ZooKeeper. + */ +class TestHiveMetastoreBasedLockProviderClose { + + private static final String DB = "testdb"; + private static final String TABLE = "testtable"; + + private LockConfiguration lockConfiguration; + private LockComponent lockComponent; + + @BeforeEach + void setUp() { + TypedProperties props = new TypedProperties(); + props.setProperty(HIVE_DATABASE_NAME_PROP_KEY, DB); + props.setProperty(HIVE_TABLE_NAME_PROP_KEY, TABLE); + lockConfiguration = new LockConfiguration(props); + lockComponent = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, DB); + lockComponent.setTablename(TABLE); + } + + @Test + void closeShutsDownExecutorEvenWhenUnlockThrows() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(42L)); + doThrow(new TException("boom")).when(client).unlock(anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + // A failing unlock() must not prevent the heartbeat thread pool from being shut down. + assertDoesNotThrow(provider::close); + assertTrue(executorOf(provider).isShutdown(), + "executor must be shut down even when unlock() throws"); + } + + @Test + void closeShutsDownExecutorOnNormalPath() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(1L)); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + provider.close(); + + verify(client).unlock(1L); + assertTrue(executorOf(provider).isShutdown()); + } + + private static LockResponse acquiredLock(long lockId) { + LockResponse response = new LockResponse(); + response.setLockid(lockId); + response.setState(LockState.ACQUIRED); + return response; + } + + private static ScheduledExecutorService executorOf(HiveMetastoreBasedLockProvider provider) throws Exception { + Field field = HiveMetastoreBasedLockProvider.class.getDeclaredField("executor"); + field.setAccessible(true); + return (ScheduledExecutorService) field.get(provider); + } +} From ef9a7f347abd568fe723556943d615909dfd8a6d Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Wed, 22 Jul 2026 15:47:46 +0530 Subject: [PATCH 186/255] fix(utilities): guard Source.releaseResources() against transient RDD unpersist failures (#19328) releaseResources() runs in StreamSync.syncOnce()'s finally block, after the write/commit has already completed. A transient Spark NullPointerException in BlockManagerMaster.removeRdd (thrown when the SparkContext is mid-teardown or stopping and the driver endpoint is null) during the post-write RDD unpersist was propagating out and failing otherwise-successful ingestion rounds. A cleanup-only failure must not fail the round: wrap the unpersist in try/catch and log at WARN. Extract the unpersist into a protected unpersistCachedSourceRdd() seam (VisibleForTesting) and add TestSource covering both the swallow and happy paths. (cherry picked from commit 76a95f5d629bceff89eab081c768438f2fef1e87) --- .../apache/hudi/utilities/sources/Source.java | 12 +++ .../hudi/utilities/sources/TestSource.java | 73 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSource.java diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java index 75cef40c1ee5b..de2ce192401ea 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Either; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.utilities.callback.SourceCommitCallback; import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.streamer.DefaultStreamContext; @@ -172,6 +173,17 @@ private synchronized void persist(T data) { @Override public void releaseResources() { + // Cleanup runs after the write/commit; a transient Spark failure while unpersisting + // must not fail an already-successful round. + try { + unpersistCachedSourceRdd(); + } catch (Exception e) { + log.warn("Failed to unpersist cached source RDD during releaseResources; ignoring", e); + } + } + + @VisibleForTesting + protected void unpersistCachedSourceRdd() { if (cachedSourceRdd != null && cachedSourceRdd.isLeft()) { cachedSourceRdd.asLeft().unpersist(); } else if (cachedSourceRdd != null && cachedSourceRdd.isRight()) { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSource.java new file mode 100644 index 0000000000000..e6e4897e05f13 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSource.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.sources; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.util.Option; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Unit tests for {@link Source#releaseResources()}. + * + *

    releaseResources() is invoked from StreamSync.syncOnce()'s finally block, after the + * write/commit has already completed. A transient Spark failure while unpersisting the + * cached source RDD (e.g. a NullPointerException from BlockManagerMaster.removeRdd when + * the SparkContext is mid-teardown/stopping) must not fail an otherwise-successful + * ingestion round. + */ +public class TestSource { + + /** + * Minimal concrete {@link Source} whose unpersist step always fails, so the + * swallow-on-failure behaviour of releaseResources() can be exercised without a + * live SparkContext. + */ + private static class MockSourceWithUnpersistenceFailure extends Source { + private int unpersistCalls = 0; + + MockSourceWithUnpersistenceFailure(TypedProperties props) { + super(props, null, null, null); + } + + @Override + protected InputBatch fetchNewData(Option lastCkptStr, long sourceLimit) { + // Not exercised by these tests; releaseResources() is tested directly. + return null; + } + + @Override + protected void unpersistCachedSourceRdd() { + unpersistCalls++; + throw new NullPointerException("simulated BlockManagerMaster.removeRdd NPE"); + } + } + + @Test + public void releaseResourcesSwallowsTransientUnpersistFailure() { + MockSourceWithUnpersistenceFailure source = + new MockSourceWithUnpersistenceFailure(new TypedProperties()); + assertDoesNotThrow(source::releaseResources, + "A transient unpersist failure in cleanup must not propagate out of releaseResources()"); + assertEquals(1, source.unpersistCalls, "unpersist should have been attempted exactly once"); + } +} From 696fe6c4c3550e3b55faa797c608417afbd971e5 Mon Sep 17 00:00:00 2001 From: Davis-Zhang-Onehouse <169106455+Davis-Zhang-Onehouse@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:35:14 -0700 Subject: [PATCH 187/255] chore(utilities): add SQS backlog / in-flight visibility to S3 events source (#19333) * logging: add SQS backlog / in-flight visibility to S3 events source Adds a lifecycle log line to S3EventsSource.onCommit recording how many SQS messages are being deleted at commit and the checkpoint, improving operational visibility into the SQS-backed S3 events ingestion path. Placeholder / first increment of a broader observability change tracked for an Onehouse internal hotfix; the full receive-loop / in-flight logging will be upstreamed here subsequently. * chore(utilities): drop redundant class/method prefix from onCommit log SLF4J emits the logger name already (every log4j2 pattern in the repo uses %c), so the "S3EventsSource.onCommit:" prefix duplicated it. Also names the queue as SQS in the message. --------- Co-authored-by: voon (cherry picked from commit e4071ce5407656e6acd6214b77fcc66813bce69b) --- .../org/apache/hudi/utilities/sources/S3EventsSource.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java index 6d96542d0bbb0..b6b66d3fb4188 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java @@ -35,6 +35,8 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; import java.io.Closeable; @@ -52,6 +54,8 @@ */ public class S3EventsSource extends RowSource implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(S3EventsSource.class); + private final S3EventsMetaSelector pathSelector; private final SchemaProvider schemaProvider; private final List processedMessages = new ArrayList<>(); @@ -109,6 +113,8 @@ public void close() throws IOException { @Override public void onCommit(String lastCkptStr) { + LOG.info("Deleting {} processed messages from SQS queue, checkpoint={}.", + processedMessages.size(), lastCkptStr); pathSelector.deleteProcessedMessages(sqs, pathSelector.queueUrl, processedMessages); processedMessages.clear(); } From f69f68eab6d2bbf705cf34d27b1680bad4c8f162 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Thu, 23 Jul 2026 05:26:00 +0530 Subject: [PATCH 188/255] fix(reader): derive pre-v9 CDC delete markers from the effective payload class (#19348) Co-authored-by: Y Ethan Guo (cherry picked from commit fb07a2dfa5c8b03b714d67d85644407e1519bb57) --- .../apache/hudi/io/TestHoodieWriteHandle.java | 1 + .../hudi/common/table/HoodieTableConfig.java | 12 ++- .../apache/hudi/common/util/ConfigUtils.java | 8 +- .../hudi/common/util/TestConfigUtils.java | 83 ++++++++++++++++++- .../common/table/TestHoodieTableConfig.java | 4 +- .../TestPayloadDeprecationFlow.scala | 72 +++++++++++++++- 6 files changed, 172 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestHoodieWriteHandle.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestHoodieWriteHandle.java index b37f5989e57c3..394fa7b292c37 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestHoodieWriteHandle.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestHoodieWriteHandle.java @@ -81,6 +81,7 @@ public void setUp() { MockitoAnnotations.initMocks(this); when(mockHoodieTable.getMetaClient()).thenReturn(mockMetaClient); when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig); + when(mockTableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(mockWriteConfig.getRecordMerger()).thenReturn(mockRecordMerger); // Set up a basic schema for the write config diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 76f687b7963a8..1ed99b5d11a0f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -1371,14 +1371,20 @@ public Option getPartitionMetafileFormat() { return Option.empty(); } - public Map getTableMergeProperties() { + /** + * Returns the record-merge properties for this table, deriving the pre-v9 delete markers from the + * given effective payload class rather than the one persisted in this table config. Callers on the + * write path pass the write-config payload class ({@code hoodie.datasource.write.payload.class}), + * which for a pre-v9 table may be the only place the payload class is set. + */ + public Map getTableMergeProperties(String payloadClass) { Map configs = ConfigUtils.extractWithPrefix(this.props, RECORD_MERGE_PROPERTY_PREFIX); if (getTableVersion().lesserThan(HoodieTableVersion.NINE)) { // Convert legacy payload properties do delete key and delete marker properties - if (getPayloadClass().equals(AWSDmsAvroPayload.class.getName())) { + if (payloadClass.equals(AWSDmsAvroPayload.class.getName())) { configs.put(DELETE_KEY, OP_FIELD); configs.put(DELETE_MARKER, DELETE_OPERATION_VALUE); - } else if (getPayloadClass().equals(MySqlDebeziumAvroPayload.class.getName()) || getPayloadClass().equals(PostgresDebeziumAvroPayload.class.getName())) { + } else if (payloadClass.equals(MySqlDebeziumAvroPayload.class.getName()) || payloadClass.equals(PostgresDebeziumAvroPayload.class.getName())) { configs.put(DELETE_KEY, DebeziumConstants.FLATTENED_OP_COL_NAME); configs.put(DELETE_MARKER, DebeziumConstants.DELETE_OP); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java index 6fb81390ccda9..463842096d43f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java @@ -151,7 +151,13 @@ public static TypedProperties supplementOrderingFields(TypedProperties props, Li * Ensures that the prefixed merge properties are populated for mergers. */ public static TypedProperties getMergeProps(TypedProperties props, HoodieTableConfig tableConfig) { - Map mergeProps = tableConfig.getTableMergeProperties(); + // Prefer the payload class persisted in the table config, falling back to the reader/write props + // (e.g. hoodie.datasource.write.payload.class) when the table never persisted it. This keeps the + // persisted payload authoritative (no query-side shadowing) while pre-v9 delete markers still derive. + String payloadClass = tableConfig.getPayloadClassIfPresent() + .orElseGet(() -> HoodieRecordPayload.getPayloadClassNameIfPresent(props) + .orElseGet(tableConfig::getPayloadClass)); + Map mergeProps = tableConfig.getTableMergeProperties(payloadClass); if (mergeProps.isEmpty()) { return props; } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java index 1c862803c5f23..d44ddf755e352 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java @@ -24,8 +24,13 @@ import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieReaderConfig; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.AWSDmsAvroPayload; import org.apache.hudi.common.model.HoodiePayloadProps; +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.model.debezium.DebeziumConstants; +import org.apache.hudi.common.model.debezium.PostgresDebeziumAvroPayload; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.util.collection.ExternalSpillableMap.DiskMapType; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; @@ -41,6 +46,8 @@ import java.util.Map; import java.util.stream.Stream; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER; import static org.apache.hudi.common.table.HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -476,4 +483,78 @@ void testBuildFileGroupReaderPropertiesIncludesMetadataFileCacheConfig() { assertEquals("200000", fileGroupReaderProps.getProperty(HoodieReaderConfig.HFILE_BLOCK_CACHE_SIZE.key())); assertEquals("7", fileGroupReaderProps.getProperty(HoodieReaderConfig.HFILE_BLOCK_CACHE_TTL_MINUTES.key())); } -} \ No newline at end of file + + @Test + void testGetMergePropsPrefersPersistedTableConfigPayloadClass() { + // A payload override in the write props must not shadow the class persisted in the table config: + // the persisted Debezium payload wins, so the pre-v9 delete markers are derived from it. + HoodieTableConfig tableConfig = preV9TableConfig(PostgresDebeziumAvroPayload.class.getName()); + TypedProperties props = new TypedProperties(); + props.put("hoodie.datasource.write.payload.class", OverwriteWithLatestAvroPayload.class.getName()); + + TypedProperties merged = ConfigUtils.getMergeProps(props, tableConfig); + + assertEquals(DebeziumConstants.FLATTENED_OP_COL_NAME, merged.getString(DELETE_KEY)); + assertEquals(DebeziumConstants.DELETE_OP, merged.getString(DELETE_MARKER)); + } + + @Test + void testGetMergePropsFallsBackToWritePropsPayloadClass() { + // Pre-v9 table that never persisted a payload class: resolve it from the write props so the + // delete-marker derivation still fires. + HoodieTableConfig tableConfig = preV9TableConfig(null); + TypedProperties props = new TypedProperties(); + props.put("hoodie.datasource.write.payload.class", PostgresDebeziumAvroPayload.class.getName()); + + TypedProperties merged = ConfigUtils.getMergeProps(props, tableConfig); + + assertEquals(DebeziumConstants.FLATTENED_OP_COL_NAME, merged.getString(DELETE_KEY)); + assertEquals(DebeziumConstants.DELETE_OP, merged.getString(DELETE_MARKER)); + } + + @Test + void testGetMergePropsDerivesAwsDmsDeleteMarkers() { + HoodieTableConfig tableConfig = preV9TableConfig(AWSDmsAvroPayload.class.getName()); + + TypedProperties merged = ConfigUtils.getMergeProps(new TypedProperties(), tableConfig); + + assertEquals(AWSDmsAvroPayload.OP_FIELD, merged.getString(DELETE_KEY)); + assertEquals(AWSDmsAvroPayload.DELETE_OPERATION_VALUE, merged.getString(DELETE_MARKER)); + } + + @Test + void testGetMergePropsDerivesNoDeleteMarkersForVersionNine() { + // Version 9+ tables carry the merge configs directly, so the legacy delete markers are not + // derived even for a CDC payload. + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.getProps().put(HoodieTableConfig.VERSION.key(), String.valueOf(HoodieTableVersion.NINE.versionCode())); + tableConfig.getProps().put(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), PostgresDebeziumAvroPayload.class.getName()); + + TypedProperties merged = ConfigUtils.getMergeProps(new TypedProperties(), tableConfig); + + assertFalse(merged.containsKey(DELETE_KEY)); + assertFalse(merged.containsKey(DELETE_MARKER)); + } + + @Test + void testGetMergePropsDerivesNoDeleteMarkersForNonCdcPayload() { + // Pre-v9 table with a non-CDC payload and no override: no delete markers, props returned as-is. + HoodieTableConfig tableConfig = preV9TableConfig(OverwriteWithLatestAvroPayload.class.getName()); + TypedProperties props = new TypedProperties(); + props.put("normal.key", "val"); + + TypedProperties merged = ConfigUtils.getMergeProps(props, tableConfig); + + assertFalse(merged.containsKey(DELETE_KEY)); + assertEquals("val", merged.getString("normal.key")); + } + + private static HoodieTableConfig preV9TableConfig(String payloadClass) { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.getProps().put(HoodieTableConfig.VERSION.key(), String.valueOf(HoodieTableVersion.EIGHT.versionCode())); + if (payloadClass != null) { + tableConfig.getProps().put(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), payloadClass); + } + return tableConfig; + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java index eebb5a77a8be2..2784ee00a5887 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java @@ -395,7 +395,7 @@ void testDefinedTableConfigs() { void testTableMergeProperties() throws IOException { // for out of the box, there are no merge properties HoodieTableConfig config = new HoodieTableConfig(storage, metaPath); - assertTrue(config.getTableMergeProperties().isEmpty()); + assertTrue(config.getTableMergeProperties(config.getPayloadClass()).isEmpty()); // delete and re-create w/ merge properties storage.deleteFile(cfgPath); @@ -414,7 +414,7 @@ void testTableMergeProperties() throws IOException { Map expectedProps = new HashMap<>(); expectedProps.put("key1","value1"); expectedProps.put("key2","value2"); - assertEquals(expectedProps, config.getTableMergeProperties()); + assertEquals(expectedProps, config.getTableMergeProperties(config.getPayloadClass())); } private static Stream testInferMergingConfigsForPreV9Table() { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala index 76e23f886f537..58fab2afcb9a0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala @@ -35,7 +35,7 @@ import org.apache.hudi.testutils.SparkClientFunctionalTestHarness import org.apache.spark.sql.SaveMode import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.{Arguments, MethodSource} +import org.junit.jupiter.params.provider.{Arguments, MethodSource, ValueSource} import org.scalatest.Assertions.assertThrows import scala.jdk.CollectionConverters._ @@ -494,6 +494,76 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { .build() } + /** + * COW pre-v9 Postgres-Debezium table whose payload class is set only via the write config + * (hoodie.datasource.write.payload.class), not the table properties. A Debezium delete + * (_change_operation_type='d') for a key absent from the base file must be classified as a delete + * and become a no-op, rather than failing when the reader derives the delete markers. + * + * stripPayloadClass=true removes the payload class from the table properties (the failing case + * before the fix); stripPayloadClass=false keeps it and acts as the control. + */ + @ParameterizedTest + @ValueSource(strings = Array("true", "false")) + def testDebeziumDeleteForAbsentKeyWithPayloadClassNotInTableConfig(stripPayloadClassStr: String): Unit = { + val stripPayloadClass = stripPayloadClassStr.toBoolean + val payloadClazz = classOf[PostgresDebeziumAvroPayload].getName + // Payload class supplied only as a write option, not as a table property. + val opts: Map[String, String] = Map( + HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key() -> payloadClazz, + HoodieMetadataConfig.ENABLE.key() -> "false") + // Single-bucket index so the delete key hashes to the existing file group and is merged against + // its base file, rather than creating a fresh insert file group. + val indexOpts: Map[String, String] = Map( + "hoodie.index.type" -> "BUCKET", + "hoodie.index.bucket.engine" -> "SIMPLE", + "hoodie.bucket.index.num.buckets" -> "1", + "hoodie.bucket.index.hash.field" -> "_event_lsn") + + val columns = Seq("ts", "_event_lsn", "rider", "driver", "fare", "Op", "_event_seq", + DebeziumConstants.FLATTENED_FILE_COL_NAME, DebeziumConstants.FLATTENED_POS_COL_NAME, DebeziumConstants.FLATTENED_OP_COL_NAME) + + // 1. Insert base rows (lsn 1,2,3) into a COW table at table version 8. + val data = Seq( + (10, 1L, "rider-A", "driver-A", 19.10, "i", "10.1", 10, 1, "i"), + (10, 2L, "rider-B", "driver-B", 27.70, "i", "10.1", 10, 1, "i"), + (10, 3L, "rider-C", "driver-C", 33.90, "i", "10.1", 10, 1, "i")) + spark.createDataFrame(data).toDF(columns: _*).write.format("hudi") + .option(RECORDKEY_FIELD.key(), "_event_lsn") + .option(HoodieTableConfig.ORDERING_FIELDS.key(), "_event_lsn") + .option(TABLE_TYPE.key(), HoodieTableType.COPY_ON_WRITE.name()) + .option(DataSourceWriteOptions.TABLE_NAME.key(), "test_table") + .option(OPERATION.key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + .option(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), "8") + .options(indexOpts) + .options(opts) + .mode(SaveMode.Overwrite) + .save(basePath) + + var metaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(storageConf()).build() + assertEquals(8, metaClient.getTableConfig.getTableVersion.versionCode()) + + if (stripPayloadClass) { + // Remove any persisted payload-class keys so the payload class lives only in the write config. + val payloadKeys = metaClient.getTableConfig.getProps.asScala.keys + .filter(k => k.toString.contains("payload.class")).map(_.toString).toSet + HoodieTableConfig.delete(metaClient.getStorage, metaClient.getMetaPath, payloadKeys.asJava) + metaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(storageConf()).build() + assertFalse(metaClient.getTableConfig.getPayloadClassIfPresent.isPresent) + } + + // 2. Upsert a Debezium delete ('d') for a key ABSENT from the base file (lsn 99). + val deleteData = Seq( + (12, 99L, "rider-Z", "driver-Z", 20.10, "D", "12.1", 12, 1, "d")) + performUpsert(deleteData, columns, indexOpts, opts, basePath, + tableVersion = Some("8"), orderingFields = Some("_event_lsn")) + + // 3. Base rows intact and the absent-key delete is a no-op. + val df = spark.read.format("hudi").load(basePath) + assertEquals(3, df.count()) + assertEquals(0, df.filter("_event_lsn = 99").count()) + } + /** * Helper method to perform upsert operations with configurable options */ From 028357b11b1139bb27eed944878319d45b6a4961 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 23 Jul 2026 09:55:23 +0800 Subject: [PATCH 189/255] test(common): add LSM file group read path coverage (#19347) Partial backport: 2 of the 6 test files apply to release-1.2.1. Dropped, with reasons: - TestHoodieLsmFileGroupReader, TestLsmFileGroupRecordIterator and TestSpillableLsmRecordIterator cover org.apache.hudi.common.table.read.lsm, a package that does not exist on this branch. It arrives with 6170ac905273 (#18987, "feat: Add a lsm-tree based FG reader"), which is not backported. - TestPositionBasedFileGroupRecordBuffer exercises master-only log-block APIs: HoodieLogBlock#getRecordPositionList() (here only the Roaring64NavigableMap getRecordPositions() exists) and the HoodieDeleteBlock#getRecordsToDelete(HoodieReaderContext) overload. Porting it would mean rewriting the test against this branch's API rather than backporting it. Kept TestBufferedRecordMergerFactory and TestIncrementalQueryAnalyzer, whose classes under test both exist here. Applied verbatim; no adaptation needed. (cherry picked from commit e7b0ba818df93476be4f339b74a2aae970046494) --- .../read/TestBufferedRecordMergerFactory.java | 84 +++++++++++++++++ .../read/TestIncrementalQueryAnalyzer.java | 89 +++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestIncrementalQueryAnalyzer.java diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java new file mode 100644 index 0000000000000..8015d0dcd8f93 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.table.read; + +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieReaderContext; +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.table.PartialUpdateMode; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestBufferedRecordMergerFactory { + + @Test + void testSelectsMergerForEveryConfiguredMode() { + HoodieReaderContext context = mock(HoodieReaderContext.class); + when(context.getRecordContext()).thenReturn(mock(RecordContext.class)); + HoodieSchema schema = HoodieSchema.create(HoodieSchemaType.STRING); + TypedProperties props = new TypedProperties(); + Option recordMerger = Option.of(mock(HoodieRecordMerger.class)); + + assertMerger("CommitTimeRecordMerger", context, RecordMergeMode.COMMIT_TIME_ORDERING, false, + recordMerger, schema, Option.empty(), props, Option.empty()); + assertMerger("CommitTimePartialRecordMerger", context, RecordMergeMode.COMMIT_TIME_ORDERING, false, + recordMerger, schema, Option.empty(), props, Option.of(PartialUpdateMode.IGNORE_DEFAULTS)); + assertMerger("EventTimeRecordMerger", context, RecordMergeMode.EVENT_TIME_ORDERING, false, + recordMerger, schema, Option.empty(), props, Option.empty()); + assertMerger("EventTimePartialRecordMerger", context, RecordMergeMode.EVENT_TIME_ORDERING, false, + recordMerger, schema, Option.empty(), props, Option.of(PartialUpdateMode.FILL_UNAVAILABLE)); + assertMerger("PartialUpdateBufferedRecordMerger", context, RecordMergeMode.EVENT_TIME_ORDERING, true, + recordMerger, schema, Option.empty(), props, Option.empty()); + assertMerger("CustomRecordMerger", context, RecordMergeMode.CUSTOM, false, + recordMerger, schema, Option.empty(), props, Option.empty()); + assertMerger("CustomPayloadRecordMerger", context, RecordMergeMode.CUSTOM, false, + recordMerger, schema, Option.of(Pair.of("table.Payload", "incoming.Payload")), props, Option.empty()); + assertMerger("ExpressionPayloadRecordMerger", context, RecordMergeMode.CUSTOM, false, + recordMerger, schema, Option.of(Pair.of("table.Payload", "org.apache.spark.sql.hudi.command.payload.ExpressionPayload")), props, Option.empty()); + + assertEquals("CustomPayloadRecordMerger", BufferedRecordMergerFactory.create( + context, RecordMergeMode.CUSTOM, false, recordMerger, Option.of("payload.Class"), schema, props, Option.empty()) + .getClass().getSimpleName()); + } + + private static void assertMerger(String expected, + HoodieReaderContext context, + RecordMergeMode mode, + boolean partialMerging, + Option recordMerger, + HoodieSchema schema, + Option> payloadClasses, + TypedProperties props, + Option partialUpdateMode) { + assertEquals(expected, BufferedRecordMergerFactory.create( + context, mode, partialMerging, recordMerger, schema, payloadClasses, props, partialUpdateMode) + .getClass().getSimpleName()); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestIncrementalQueryAnalyzer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestIncrementalQueryAnalyzer.java new file mode 100644 index 0000000000000..4ed4c34e41e0a --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestIncrementalQueryAnalyzer.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.table.read; + +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.log.InstantRange; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestIncrementalQueryAnalyzer { + + @Test + void testQueryContextRangeEdges() { + HoodieTimeline timeline = mock(HoodieTimeline.class); + HoodieInstant active = mock(HoodieInstant.class); + when(active.getCompletionTime()).thenReturn("20240102000000"); + IncrementalQueryAnalyzer.QueryContext earliestToLatest = IncrementalQueryAnalyzer.QueryContext.create( + null, null, Arrays.asList("001", "002"), Collections.emptyList(), Collections.singletonList(active), timeline, null); + + assertFalse(earliestToLatest.isEmpty()); + assertEquals("002", earliestToLatest.getLastInstant()); + assertTrue(earliestToLatest.isConsumingFromEarliest()); + assertTrue(earliestToLatest.isConsumingToLatest()); + assertTrue(earliestToLatest.getInstantRange().isEmpty()); + assertEquals("20240102000000", earliestToLatest.getMaxCompletionTime()); + assertEquals(Collections.singletonList(active), earliestToLatest.getInstants()); + + IncrementalQueryAnalyzer.QueryContext boundedEarliest = IncrementalQueryAnalyzer.QueryContext.create( + null, "002", Arrays.asList("001", "002"), Collections.emptyList(), Collections.emptyList(), timeline, null); + HoodieInstant latestActive = mock(HoodieInstant.class); + when(latestActive.getCompletionTime()).thenReturn("20240103000000"); + when(timeline.getInstantsAsStream()).thenReturn(Stream.of(latestActive)); + InstantRange boundedRange = boundedEarliest.getInstantRange().get(); + assertTrue(boundedEarliest.isConsumingFromEarliest()); + assertFalse(boundedEarliest.isConsumingToLatest()); + assertTrue(boundedRange.isInRange("001")); + assertTrue(boundedRange.isInRange("002")); + assertEquals("20240103000000", boundedEarliest.getMaxCompletionTime()); + assertNull(boundedEarliest.getArchivedTimeline()); + + IncrementalQueryAnalyzer.QueryContext exact = IncrementalQueryAnalyzer.QueryContext.create( + "001", "002", Arrays.asList("001", "002"), Collections.emptyList(), Collections.emptyList(), timeline, null); + assertFalse(exact.isConsumingFromEarliest()); + assertTrue(exact.getInstantRange().get().isInRange("001")); + assertFalse(exact.getInstantRange().get().isInRange("003")); + assertThrows(IllegalStateException.class, IncrementalQueryAnalyzer.QueryContext.EMPTY::getLastInstant); + } + + @Test + void testBuilderRequiresMetaClientAndRangeType() { + assertThrows(NullPointerException.class, () -> IncrementalQueryAnalyzer.builder() + .rangeType(InstantRange.RangeType.CLOSED_CLOSED) + .build()); + assertThrows(NullPointerException.class, () -> IncrementalQueryAnalyzer.builder() + .metaClient(mock(HoodieTableMetaClient.class)) + .build()); + } +} From b3756517b3223fc53200db0ad7107cbb55d49f6e Mon Sep 17 00:00:00 2001 From: Shihuan Liu Date: Wed, 22 Jul 2026 18:55:54 -0700 Subject: [PATCH 190/255] fix(flink): make hudi-flink-bundle built with flink-bundle-shade-hive usable for Hive sync (#19330) (cherry picked from commit 76f81ec558211bce6074529578b86189b4b00e27) --- packaging/hudi-flink-bundle/pom.xml | 200 ++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml index 0c995e939c233..9fb78c57e11f1 100644 --- a/packaging/hudi-flink-bundle/pom.xml +++ b/packaging/hudi-flink-bundle/pom.xml @@ -652,6 +652,106 @@ ${flink.bundle.hive.scope} + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + + + + + org.antlr:antlr-runtime + org.antlr:ST4 + + + + + + com.google.common. + ${flink.bundle.shade.prefix}com.google.common. + + + + com.google.protobuf. + ${flink.bundle.shade.prefix}com.google.protobuf. + + + org.apache.commons.lang3. + ${flink.bundle.shade.prefix}org.apache.commons.lang3. + + + org.apache.commons.lang. + ${flink.bundle.shade.prefix}org.apache.commons.lang. + + + org.apache.thrift. + ${flink.bundle.shade.prefix}org.apache.thrift. + + + org.apache.orc. + ${flink.bundle.shade.prefix}org.apache.orc. + + + org.json. + ${flink.bundle.shade.prefix}org.json. + + + org.codehaus.jackson. + ${flink.bundle.shade.prefix}org.codehaus.jackson. + + + com.facebook.fb303. + ${flink.bundle.shade.prefix}com.facebook.fb303. + + + au.com.bytecode.opencsv. + ${flink.bundle.shade.prefix}au.com.bytecode.opencsv. + + + javaewah. + ${flink.bundle.shade.prefix}javaewah. + + + javolution. + ${flink.bundle.shade.prefix}javolution. + + + jodd. + ${flink.bundle.shade.prefix}jodd. + + + + + + org.apache.hive:hive-exec + + org/apache/avro/** + org/apache/orc/** + org/apache/parquet/** + com/google/protobuf/** + io/airlift/compress/** + + + + + + + + + flink-bundle-shade-hive3 @@ -673,6 +773,106 @@ ${flink.bundle.hive.scope} + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + + + + + org.antlr:antlr-runtime + org.antlr:ST4 + + + + + + com.google.common. + ${flink.bundle.shade.prefix}com.google.common. + + + + com.google.protobuf. + ${flink.bundle.shade.prefix}com.google.protobuf. + + + org.apache.commons.lang3. + ${flink.bundle.shade.prefix}org.apache.commons.lang3. + + + org.apache.commons.lang. + ${flink.bundle.shade.prefix}org.apache.commons.lang. + + + org.apache.thrift. + ${flink.bundle.shade.prefix}org.apache.thrift. + + + org.apache.orc. + ${flink.bundle.shade.prefix}org.apache.orc. + + + org.json. + ${flink.bundle.shade.prefix}org.json. + + + org.codehaus.jackson. + ${flink.bundle.shade.prefix}org.codehaus.jackson. + + + com.facebook.fb303. + ${flink.bundle.shade.prefix}com.facebook.fb303. + + + au.com.bytecode.opencsv. + ${flink.bundle.shade.prefix}au.com.bytecode.opencsv. + + + javaewah. + ${flink.bundle.shade.prefix}javaewah. + + + javolution. + ${flink.bundle.shade.prefix}javolution. + + + jodd. + ${flink.bundle.shade.prefix}jodd. + + + + + + org.apache.hive:hive-exec + + org/apache/avro/** + org/apache/orc/** + org/apache/parquet/** + com/google/protobuf/** + io/airlift/compress/** + + + + + + + + + hudi-platform-service From 1c9fd002ba0a04af968d0226c686716aeedf951c Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Thu, 23 Jul 2026 10:30:27 +0530 Subject: [PATCH 191/255] fix: Handle map/array-nested leaf columns in column stats collection during MOR log-append (#19126) * fix: Handle map/array nested leaves in Avro column-stats value navigation Collecting column stats for a primitive nested inside a MAP or ARRAY (e.g. `my_map.key_value.value`, `my_array.list.element`) crashed the MOR inline log-append path at table version 9: IllegalStateException: Cannot get field from schema type: MAP at HoodieSchema.getField at AvroRecordContext.getFieldValueFromIndexedRecord at HoodieAvroIndexedRecord.getColumnValueAsJava at HoodieTableMetadataUtil.collectColumnRangeFieldValueV2 at HoodieInlineLogAppendHandle.collectColumnStats #17694 taught the schema-side navigator (HoodieSchema.getNestedField) and the base-file (Parquet) path to resolve the Parquet-style `.key_value.key`, `.key_value.value` and `.list.element` synthetic accessors, so such leaves pass isColumnTypeSupported. The record value-side navigator (AvroRecordContext.getFieldValueFromIndexedRecord), used by the column-stats V2 collection path (collectColumnRangeFieldValueV2 -> getColumnValueAsJava), was never updated: it assumes every path segment is a RECORD field and calls HoodieSchema.getField on the MAP/ARRAY schema, which throws. Make the value navigator return null when a path segment cannot be resolved as a plain RECORD field (a MAP/ARRAY intermediate or a null intermediate), instead of throwing. This mirrors HoodieAvroUtils.getNestedFieldVal, which is why the V1 path at table version 6 already tolerated these paths. A map/array leaf is multi-valued per record and has no single value to fold into a min/max, so returning null (no stats from the record path) is correct; statistics for such leaves are still collected from the base-file (Parquet) footer path added in #17694. Also guard the intermediate downcast so a non-record value (java Map/List) degrades to null rather than throwing ClassCastException. * Rename test constant COMPLEX_SCHEMA to MAP_AND_ARRAY_SCHEMA for clarity * Move the schema constants to the top of TestAvroRecordContext --------- Co-authored-by: voon (cherry picked from commit f25e0799c5b73e8b77d777910dab78859406b1c5) --- .../apache/hudi/avro/AvroRecordContext.java | 10 ++- .../hudi/avro/TestAvroRecordContext.java | 67 ++++++++++++++++--- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java index fceeeaf84b97a..dfc25910037cd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java @@ -79,6 +79,14 @@ public static Object getFieldValueFromIndexedRecord( String[] path = fieldName.split("\\."); for (int i = 0; i < path.length; i++) { currentSchema = currentSchema.getNonNullType(); + // Value navigation here can only descend through RECORD fields. Column-stats field paths + // that traverse a MAP (".key_value.key" / ".key_value.value") or ARRAY (".list.element") + // synthetic accessor, or that hit a null intermediate value, cannot be resolved to a single + // value and yield null instead of throwing. This mirrors HoodieAvroUtils.getNestedFieldVal; + // statistics for such nested leaves are still collected from the base-file (Parquet) path. + if (currentRecord == null || !currentSchema.hasFields()) { + return null; + } Option fieldOpt = currentSchema.getField(path[i]); if (fieldOpt.isEmpty()) { return null; @@ -89,7 +97,7 @@ public static Object getFieldValueFromIndexedRecord( return value; } currentSchema = field.schema(); - currentRecord = (IndexedRecord) value; + currentRecord = value instanceof IndexedRecord ? (IndexedRecord) value : null; } return null; } diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java index 04c7ae3c2bda7..56309cd815cb1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroRecordContext.java @@ -28,6 +28,9 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; import static org.apache.hudi.avro.AvroRecordContext.getFieldValueFromIndexedRecord; @@ -37,6 +40,23 @@ class TestAvroRecordContext { + private static final Schema RECORD_SCHEMA = new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"top\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"int\"}," + + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"address\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"address\",\"fields\":[" + + "{\"name\":\"city\",\"type\":\"string\"}," + + "{\"name\":\"zip\",\"type\":[\"null\",\"int\"],\"default\":null}]}],\"default\":null}," + + "{\"name\":\"multi\",\"type\":[\"null\",\"string\",\"int\"],\"default\":null}]}"); + + private static final Schema MAP_AND_ARRAY_SCHEMA = new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"complex\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"int\"}," + + "{\"name\":\"str_map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"string\"}],\"default\":null}," + + "{\"name\":\"int_array\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}," + + "{\"name\":\"rec_map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\"," + + "\"name\":\"inner\",\"fields\":[{\"name\":\"x\",\"type\":\"int\"}]}}],\"default\":null}]}"); + private static Stream testConvertValueToEngineType() { return Stream.of( Arguments.of(1L, 1L), @@ -52,15 +72,6 @@ void testConvertValueToEngineType(Comparable input, Comparable expected) { assertEquals(expected, actual); } - private static final Schema RECORD_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"top\",\"fields\":[" - + "{\"name\":\"id\",\"type\":\"int\"}," - + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}," - + "{\"name\":\"address\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"address\",\"fields\":[" - + "{\"name\":\"city\",\"type\":\"string\"}," - + "{\"name\":\"zip\",\"type\":[\"null\",\"int\"],\"default\":null}]}],\"default\":null}," - + "{\"name\":\"multi\",\"type\":[\"null\",\"string\",\"int\"],\"default\":null}]}"); - private static GenericRecord buildRecord() { GenericRecord address = new GenericData.Record(RECORD_SCHEMA.getField("address").schema().getTypes().get(1)); address.put("city", new Utf8("sf")); @@ -94,11 +105,45 @@ void testGetFieldValueNested() { @Test void testGetFieldValueErrorCases() { GenericRecord record = buildRecord(); - // a union that is not [null, T] does not support field lookups - assertThrows(IllegalStateException.class, () -> getFieldValueFromIndexedRecord(record, "multi.sub")); + // a union that is not [null, T] cannot be navigated into; instead of throwing, the value + // navigator returns null so callers (e.g. column-stats collection) degrade gracefully, + // matching HoodieAvroUtils.getNestedFieldVal + assertNull(getFieldValueFromIndexedRecord(record, "multi.sub")); + // an empty field name is still rejected up front assertThrows(IllegalArgumentException.class, () -> getFieldValueFromIndexedRecord(record, "")); } + @Test + void testGetFieldValueMapAndArrayLeavesReturnNull() { + GenericRecord record = new GenericData.Record(MAP_AND_ARRAY_SCHEMA); + record.put("id", 7); + Map strMap = new HashMap<>(); + strMap.put(new Utf8("a"), new Utf8("v1")); + record.put("str_map", strMap); + record.put("int_array", Arrays.asList(3, 1, 2)); + // rec_map left null + + // top-level scalar still resolves normally + assertEquals(7, getFieldValueFromIndexedRecord(record, "id")); + + // Parquet-style synthetic accessors that traverse a MAP (".key_value.key/value") or an + // ARRAY (".list.element") cannot be resolved to a single value and must return null instead + // of throwing. Regression: these previously threw + // IllegalStateException "Cannot get field from schema type: MAP" during MOR log-append + // column-stats collection. Such nested leaves still get statistics from the base-file path. + assertNull(getFieldValueFromIndexedRecord(record, "str_map.key_value.key")); + assertNull(getFieldValueFromIndexedRecord(record, "str_map.key_value.value")); + assertNull(getFieldValueFromIndexedRecord(record, "int_array.list.element")); + // a deep path descending through a MAP into a record field also degrades to null + assertNull(getFieldValueFromIndexedRecord(record, "rec_map.key_value.value.x")); + + // and null when the complex field itself is absent/null + GenericRecord empty = new GenericData.Record(MAP_AND_ARRAY_SCHEMA); + empty.put("id", 0); + assertNull(getFieldValueFromIndexedRecord(empty, "str_map.key_value.value")); + assertNull(getFieldValueFromIndexedRecord(empty, "int_array.list.element")); + } + @Test void testGetFieldValueAcrossEqualSchemaInstances() { // records from different files carry equal but distinct schema instances; both must intern to From a03241d7911840162f8342976da3077a8abf061e Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Thu, 23 Jul 2026 14:35:41 +0700 Subject: [PATCH 192/255] test(spark): Fix flaky TestSparkFilterHelper by giving it its own SparkSession (#19356) testConvertInExpression calls org.apache.spark.sql.functions.expr(...), which lazily builds a SessionState on the active SparkSession. TestSparkFilterHelper extended HoodieSparkClientTestHarness directly without wiring @BeforeEach initSparkContexts(), so it relied on a leaked active session from a prior test in the same Surefire fork and intermittently failed with "LiveListenerBus is stopped" when that session's context was already stopped. Add @BeforeEach/@AfterEach that create and stop the session, matching HoodieClientTestBase and the sibling TestHoodieDataSourceHelper. Co-authored-by: Vova Kolmakov (cherry picked from commit 5a42f8957f0971593572663ba9dedb94398545e3) --- .../org/apache/hudi/TestSparkFilterHelper.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestSparkFilterHelper.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestSparkFilterHelper.scala index 775704241de2b..6e26b642bb8c3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestSparkFilterHelper.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestSparkFilterHelper.scala @@ -28,12 +28,22 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.junit.jupiter.api.{Assertions, Test} +import org.junit.jupiter.api.{AfterEach, Assertions, BeforeEach, Test} import scala.collection.JavaConverters._ class TestSparkFilterHelper extends HoodieSparkClientTestHarness with SparkAdapterSupport { + @BeforeEach + def setUp(): Unit = { + initSparkContexts() + } + + @AfterEach + def tearDown(): Unit = { + cleanupSparkContexts() + } + @Test def testConvertInExpression(): Unit = { val filterExpr = sparkAdapter.translateFilter( From 500ccc964454bfee7a5825152f45ac7c803adeae Mon Sep 17 00:00:00 2001 From: 201573 <78600363+201573@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:17:11 +0800 Subject: [PATCH 193/255] docs(docker): document build_docker_images.sh flags (#18687) * docs(docker): document build_docker_images.sh flags * docs(docker): clarify multi-arch prerequisites * docs(docker): align demo example with setup_demo.sh * docs(docker): mention setup_demo.sh compose override * docs(docker): address review findings on demo build notes - note the default versions built when no flags are passed - clarify build_docker_images.sh builds with docker directly, unlike build_local_docker_images.sh which goes through Maven - warn that a plain setup_demo.sh run pulls Docker Hub images over locally built ones; use ./setup_demo.sh dev to keep local images - use shell language tag on the buildx code fence for consistency --------- Co-authored-by: voon (cherry picked from commit 30a4ba33e38595eb6c1ef8327cba7ae0296aa92a) --- docker/README.md | 107 +++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 63 deletions(-) diff --git a/docker/README.md b/docker/README.md index f655f42dca8b5..f445a87fb4d4b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -59,6 +59,37 @@ To build all docker images locally, you can run the script: ./build_local_docker_images.sh ``` +To build the Docker demo images with `docker` directly, rather than through the Maven build like +`build_local_docker_images.sh` above, run the script from under `/docker`: + +```shell +# With no flags, builds Hadoop 2.8.4 / Spark 3.5.3 / Hive 2.3.10, matching +# docker-compose_hadoop284_hive2310_spark353_{amd64,arm64}.yml +./build_docker_images.sh +``` + +You can override the Hadoop, Spark, and Hive versions from the command line. If you plan to use `setup_demo.sh`, +build the image set matching the default compose files first. For other flows, use one of the supported version +combinations under `docker/compose`. + +```shell +# Matches setup_demo.sh and +# docker-compose_hadoop334_hive313_spark353_{amd64,arm64}.yml +./build_docker_images.sh --hadoop-version 3.3.4 --spark-version 3.5.3 --hive-version 3.1.3 + +# Another supported combination is +# docker-compose_hadoop340_hive313_spark401_{amd64,arm64}.yml +./build_docker_images.sh --hadoop-version 3.4.0 --spark-version 4.0.1 --hive-version 3.1.3 +``` + +`setup_demo.sh` currently defaults to `docker-compose_hadoop334_hive313_spark353_{amd64,arm64}.yml`. If you build a +different image set for the demo flow, update `COMPOSE_FILE_NAME` in `setup_demo.sh` to point to the matching compose +file before running the script. Run `./setup_demo.sh dev` to use your locally built images; a plain run pulls the +Docker Hub images over them. + +By default, the script builds images for the current machine architecture and derives the version tag from the root +`pom.xml`. Use `--version-tag` to set an explicit tag if needed. + To build a single image target, you can run ```shell @@ -120,15 +151,10 @@ Please refer to the [Docker Demo Docs page](https://hudi.apache.org/docs/docker_ ## Building Multi-Arch Images -NOTE: The steps below require some code changes. Support for multi-arch builds in a fully automated manner is being -tracked by [HUDI-3601](https://issues.apache.org/jira/browse/HUDI-3601). +The `build_docker_images.sh` script supports multi-arch image builds through Docker `buildx`. First ensure a `buildx` +builder is set up locally: -By default, the docker images are built for x86_64 (amd64) architecture. Docker `buildx` allows you to build multi-arch -images, link them together with a manifest file, and push them all to a registry – with a single command. Let's say we -want to build for arm64 architecture. First we need to ensure that `buildx` setup is done locally. Please follow the -below steps (referred from https://www.docker.com/blog/multi-arch-images): - -``` +```shell # List builders ~ ❯❯❯ docker buildx ls NAME/NODE DRIVER/ENDPOINT STATUS PLATFORMS @@ -155,63 +181,18 @@ Status: running Platforms: linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6 ``` -Now goto `/docker/hoodie/hadoop` and change the `Dockerfile` to pull dependent images corresponding to -arm64. For example, in [base/Dockerfile](./hoodie/hadoop/base/Dockerfile) (which pulls jdk11 image), change the -line `FROM openjdk:11-jdk-slim-bullseye` to `FROM arm64v8/openjdk:11-jdk-slim-bullseye`. +Then run the script from under `/docker`: -Then, from under `/docker/hoodie/hadoop` directory, execute the following command to build as well as -push the image to the dockerhub repo: - -``` -# Run under hoodie/hadoop, the is optional, "latest" by default -docker buildx build --platform -t /[:] --push - -# For example, to build the Java 11 base image -docker buildx build base_java11 --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-base-java11:linux-arm64-0.10.1 --push -``` - -Note: the base image is now tagged per Java variant (`-base-java11` / `-base-java17`). Downstream Dockerfiles -select the variant via the `BASE_IMAGE_TAG` build arg (default `java11`). If you also need the Java 17 base for -arm64, repeat the build against `base_java17` and tag it as `...-base-java17:`. - -Once the base image is pushed then you could do something similar for other images. -Change [hive](./hoodie/hadoop/hive_base/Dockerfile) dockerfile to pull the base image with tag corresponding to -linux/arm64 platform. - -``` -# Change below line in the Dockerfile -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest -# as shown below (pin to the same Java variant you built above, e.g. java11) -FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-java11:linux-arm64-0.10.1 +```shell +./build_docker_images.sh --multi-arch -# and then build & push from under hoodie/hadoop dir -docker buildx build hive_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:linux-arm64-0.10.1 --push +# Example with explicit component versions +./build_docker_images.sh --hadoop-version 3.4.0 --spark-version 4.0.1 --hive-version 3.1.3 --multi-arch ``` -Similarly, for images that are dependent on hive (e.g. [base spark](./hoodie/hadoop/spark_base/Dockerfile) -, [sparkmaster](./hoodie/hadoop/sparkmaster/Dockerfile), [sparkworker](./hoodie/hadoop/sparkworker/Dockerfile) -and [sparkadhoc](./hoodie/hadoop/sparkadhoc/Dockerfile)), change the corresponding Dockerfile to pull the base hive -image with tag corresponding to arm64. Then build and push using `docker buildx` command. - -For the sake of completeness, here is a [patch](https://gist.github.com/xushiyan/cec16585e884cf0693250631a1d10ec2) which -shows what changes to make in Dockerfiles (assuming tag is named `linux-arm64-0.10.1`), and below is the list -of `docker buildx` commands. - -``` -docker buildx build base_java11 --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-base-java11:linux-arm64-0.10.1 --push -docker buildx build datanode --platform linux/arm64 --build-arg BASE_IMAGE_TAG=java11 -t apachehudi/hudi-hadoop_2.8.4-datanode:linux-arm64-0.10.1 --push -docker buildx build historyserver --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-history:linux-arm64-0.10.1 --push -docker buildx build hive_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:linux-arm64-0.10.1 --push -docker buildx build namenode --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-namenode:linux-arm64-0.10.1 --push -docker buildx build prestobase --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:linux-arm64-0.10.1 --push -docker buildx build spark_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkbase_2.4.4:linux-arm64-0.10.1 --push -docker buildx build sparkadhoc --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:linux-arm64-0.10.1 --push -docker buildx build sparkmaster --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:linux-arm64-0.10.1 --push -docker buildx build sparkworker --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:linux-arm64-0.10.1 --push -``` +When `--multi-arch` is enabled, the script builds and pushes the amd64 and arm64 variants in one pass. Use +`--version-tag ` to override the image tag used for the push. -Once all the required images are pushed to the dockerhub repos, then we need to do one additional change -in [docker compose](./compose/docker-compose_hadoop284_hive233_spark244.yml) file. -Apply [this patch](https://gist.github.com/codope/3dd986de5e54f0650dd74b6032e4456c) to the docker compose file so -that [setup_demo](./setup_demo.sh) pulls images with the correct tag for arm64. And now we should be ready to run the -setup script and follow the docker demo. +Note that `--multi-arch` uses `docker buildx build --push` and the image names in the script are hardcoded to the +`apachehudi/...` Docker Hub repositories, so this flow requires push access to those repositories. No Dockerfile +changes are needed for the current amd64 plus arm64 image set in this repository. From a828431603335e19eae86965c69ffa7a8f32de18 Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 23 Jul 2026 16:22:21 +0800 Subject: [PATCH 194/255] fix(metadata-table): follow-ups for the zero-size file skip (#18611) (#19355) * test(metadata): add Scala test for upsert after skipping zero-size file on MDT initialize Adds testUpsertAfterSkippingZeroSizeFileOnInitialize to TestRecordLevelIndex to cover the scenario where SKIP_ZERO_SIZE_FILES_ON_INITIALIZE is enabled: - corrupts the sole base file to zero-size and forces MDT rebootstrap - verifies upsert succeeds and writes to a new file group (not the zero-size one) - validates RLI completeness and that the zero-size fileId is absent from MDT Uses GLOBAL_RECORD_LEVEL_INDEX so bootstrap always creates the minimum 10 file groups for record_index even when all data files were skipped as zero-size. * fix(metadata-table): address review follow-ups for the zero-size file skip - honor hoodie.metadata.skip.zero.size.files.on.initialize in the restore-sync relisting so restore does not re-add files that were skipped at initialization - document that skipped files remain on storage untracked by MDT and the cleaner, and must be removed manually - correct the config sinceVersion to 1.3.0 - move the metric name into HoodieMetadataMetrics and only emit it when files were actually skipped - aggregate the per-file WARN into one WARN per partition, with the individual paths at DEBUG --------- Co-authored-by: Lokesh Jain (cherry picked from commit 5f19cf32b039eb64828af8752346cdde31444a24) --- .../HoodieBackedTableMetadataWriter.java | 16 +-- .../common/config/HoodieMetadataConfig.java | 6 +- .../hudi/metadata/HoodieMetadataMetrics.java | 1 + .../metadata/HoodieTableMetadataUtil.java | 6 +- .../functional/TestRecordLevelIndex.scala | 102 ++++++++++++++++++ 5 files changed, 122 insertions(+), 9 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index afb5ec210db0f..93be569667fa1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1090,9 +1090,10 @@ private HoodieTableMetaClient initializeMetaClient() throws IOException { * * @param initializationTime Files which have a timestamp after this are neglected * @param pendingDataInstants Pending instants on data set - * @param skipZeroSizeFiles Whether zero-size data files should be skipped during listing. This should only be - * enabled on the initialize path; other callers (e.g. restore) must pass false so that - * files already tracked in the metadata table are not spuriously deleted. + * @param skipZeroSizeFiles Whether zero-size data files should be skipped during listing, per + * hoodie.metadata.skip.zero.size.files.on.initialize. Both the initialize path and the + * restore-sync relisting pass the config so zero-size files are treated consistently; + * otherwise restore would re-add to the metadata table the files skipped at initialization. * @return List consisting of {@code DirectoryInfo} for each partition found. */ private List listAllPartitionsFromFilesystem(String initializationTime, Set pendingDataInstants, boolean skipZeroSizeFiles) { @@ -1146,8 +1147,10 @@ private List listAllPartitionsFromFilesystem(String initializatio } } - final long zeroSizeCount = totalZeroSizeFiles; - metrics.ifPresent(m -> m.incrementMetric("skipped_zero_size_files_on_initialize", zeroSizeCount)); + if (totalZeroSizeFiles > 0) { + final long zeroSizeCount = totalZeroSizeFiles; + metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.SKIPPED_ZERO_SIZE_FILES_ON_INITIALIZE_STR, zeroSizeCount)); + } return partitionsToBootstrap; } @@ -1780,7 +1783,8 @@ public void update(HoodieRestoreMetadata restoreMetadata, String instantTime) { // Restore requires the existing pipelines to be shutdown. So we can safely scan the dataset to find the current // list of files in the filesystem. - List dirInfoList = listAllPartitionsFromFilesystem(instantTime, Collections.emptySet(), false); + List dirInfoList = listAllPartitionsFromFilesystem(instantTime, Collections.emptySet(), + dataWriteConfig.getMetadataConfig().shouldSkipZeroSizeFilesOnInitialize()); Map dirInfoMap = dirInfoList.stream().collect(Collectors.toMap(DirectoryInfo::getRelativePath, Function.identity())); dirInfoList.clear(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index f6757a5b71258..6daac88388549 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -199,9 +199,11 @@ public final class HoodieMetadataConfig extends HoodieConfig { .key(METADATA_PREFIX + ".skip.zero.size.files.on.initialize") .defaultValue(false) .markAdvanced() - .sinceVersion("1.2.0") + .sinceVersion("1.3.0") .withDocumentation("When enabled, zero-size data files encountered while listing the data table during " - + "metadata table initialization are skipped instead of being recorded in the metadata table."); + + "metadata table initialization and restore sync are skipped instead of being recorded in the metadata " + + "table. Skipped files remain on storage and are not tracked by the metadata table or the cleaner; " + + "remove them manually. The metadata validator will report them as inconsistencies."); public static final ConfigProperty FILE_LISTING_PARALLELISM_VALUE = ConfigProperty .key("hoodie.file.listing.parallelism") diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java index 32f7eca343f36..75d208655af6c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java @@ -67,6 +67,7 @@ public class HoodieMetadataMetrics implements Serializable { public static final String INITIALIZE_STR = "initialize"; public static final String REBOOTSTRAP_STR = "rebootstrap_count"; public static final String BOOTSTRAP_ERR_STR = "bootstrap_error"; + public static final String SKIPPED_ZERO_SIZE_FILES_ON_INITIALIZE_STR = "skipped_zero_size_files_on_initialize"; // Stats names public static final String STAT_TOTAL_BASE_FILE_SIZE = "totalBaseFileSizeInBytes"; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index f0a0fe3f98176..af7b94391810d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -3165,12 +3165,16 @@ public DirectoryInfo(String relativePath, List pathInfos, Strin if (pathInfo.getLength() > 0 || !skipZeroSizeFiles) { filenameToSizeMap.put(pathInfo.getPath().getName(), pathInfo.getLength()); } else { - log.warn("Skipping zero-size data file during MDT bootstrap: {}", pathInfo.getPath()); + log.debug("Skipping zero-size data file: {}", pathInfo.getPath()); zeroSizeFileCount++; } } } } + if (zeroSizeFileCount > 0) { + log.warn("Skipped {} zero-size data files while listing partition {}; they remain on storage and are not tracked in the metadata table", + zeroSizeFileCount, relativePath); + } } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala index 12a6ad0d6da56..b215c4e53dd16 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala @@ -776,6 +776,108 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix } } + /** + * Tests that when a zero-size base file is skipped during MDT bootstrap, a subsequent upsert + * still succeeds and produces consistent data. Because the skipped file group is absent from MDT, + * the upsert treats those records as new inserts and writes them to a new file group. The test + * verifies the final record count and that every RLI entry points to a real, readable file. + */ + @Test + def testUpsertAfterSkippingZeroSizeFileOnInitialize(): Unit = { + // Use a single-partition data generator so all inserts land in one parquet file. + val singlePartitionDataGen = HoodieTestDataGenerator.createTestGeneratorFirstPartition() + val singlePartition = HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH + val insertedRecords = 10 + val inserts = singlePartitionDataGen.generateInserts("001", insertedRecords) + val insertDf = toDataset(spark, inserts) + + val baseOptions = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> HoodieTableType.COPY_ON_WRITE.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", + HoodieCompactionConfig.INLINE_COMPACT.key() -> "false") + + insertDf.write.format("hudi") + .options(baseOptions) + .mode(SaveMode.Overwrite) + .save(basePath) + assertEquals(insertedRecords, spark.read.format("hudi").load(basePath).count()) + + // Confirm there is exactly one parquet base file in the single partition. + val partitionPath = new StoragePath(basePath, singlePartition) + val baseFilesBeforeCorruption = storage.listDirectEntries(partitionPath).asScala + .filter(_.getPath.getName.endsWith(".parquet")) + .toSeq + assertEquals(1, baseFilesBeforeCorruption.size, "Expected exactly one parquet file in the single partition") + val zeroSizeFileId = FSUtils.getFileId(baseFilesBeforeCorruption.head.getPath.getName) + + // Replace the only base file with an empty (zero-size) file. + replaceOneBaseFileWithEmpty(Seq(singlePartition)) + + // Delete MDT to force a full rebootstrap. + metaClient = HoodieTableMetaClient.reload(metaClient) + HoodieTableMetadataUtil.deleteMetadataTable(metaClient, context, false) + assertFalse(storage.exists(new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(basePath))), + "Metadata table should be absent before rebootstrap") + + // The upsert triggers MDT rebootstrap (since MDT was deleted). Skip config is set so + // the zero-size file is skipped during bootstrap. Because RLI has no entries for these + // records (they were in the skipped file), the upsert treats them as new inserts. + // Use global RLI so that the MDT bootstraps at least minFileGroupCount (10) file groups + // for record_index even when all data files were skipped as zero-size. + metaClient.reloadActiveTimeline() + val latestSchema = new TableSchemaResolver(metaClient).getTableSchemaFromLatestCommit(false).get().toString + val optionsWithSkip = baseOptions ++ Map( + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true", + HoodieIndexConfig.INDEX_TYPE.key() -> "GLOBAL_RECORD_LEVEL_INDEX", + HoodieMetadataConfig.SKIP_ZERO_SIZE_FILES_ON_INITIALIZE.key() -> "true", + HoodieWriteConfig.AVRO_SCHEMA_STRING.key() -> latestSchema) + + // Reuse the same 10 records for the upsert. Since RLI has no entries for these record keys + // (the original file was zero-size and skipped during bootstrap), the upsert treats them as + // new inserts and writes them to a brand-new file group — NOT the zero-size one. + val upsertDf = toDataset(spark, inserts) + // Upsert should succeed — any exception here fails the test. + upsertDf.write.format("hudi") + .options(optionsWithSkip) + .option(DataSourceWriteOptions.OPERATION.key(), UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + + // Data consistency check: all 10 records must be readable in the new file group. + // (The zero-size base file contributes 0 readable records; the new file group has 10.) + val readDf = spark.read.format("hudi").load(basePath) + assertEquals(insertedRecords, readDf.count(), + "All records should be readable after upsert into a new file group") + + // RLI consistency check: every live record must be indexed at the location where it actually lives. + metaClient = HoodieTableMetaClient.reload(metaClient) + val writeConfig = getWriteConfig(optionsWithSkip) + val postUpsertMetadata = metadataWriter(writeConfig).getTableMetadata.asInstanceOf[HoodieBackedTableMetadata] + + // MDT files partition must not track the zero-size file group. + val allMdtFiles = getFilesInAllPartitions(postUpsertMetadata) + assertFalse(allMdtFiles.exists(_.getPath.getName.contains(zeroSizeFileId)), + s"MDT should not contain the zero-size file group $zeroSizeFileId after bootstrap with skip enabled") + + // Global RLI: look up all record keys without a partition hint. + val recordKeys = inserts.asScala.map(_.getRecordKey).asJava.stream().collect(Collectors.toList()) + val postUpsertLocations = readRecordIndex(postUpsertMetadata, recordKeys, HOption.empty()) + assertEquals(insertedRecords, postUpsertLocations.size, + "All upserted records should have an RLI entry after upsert") + val df = readDf.collect() + validateDFWithLocations(df, postUpsertLocations, singlePartition) + + // The zero-size file group must not have been selected as the write target — its file ID + // should not appear in any of the post-upsert RLI locations. + assertFalse(postUpsertLocations.values.exists(_.getFileId == zeroSizeFileId), + "The zero-size file group should not have been picked as a write target during upsert") + } + private def replaceOneBaseFileWithEmpty(partitionPaths: Seq[String]): String = { val candidateBaseFile = partitionPaths.view.flatMap { partition => storage.listDirectEntries(new StoragePath(basePath, partition)).asScala From a51b3a481fb939737447998db962e55e08cdf64b Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 23 Jul 2026 17:42:02 +0800 Subject: [PATCH 195/255] fix(flink): normalize row logical conversions and improve coverage (#19351) * test(flink): improve row conversion coverage Adapted for release-1.2.1: dropped three of the seven new test files. TestRowDataAvroRoundTrip, TestVectorConversionUtils and TestParquetRowDataWriter all exercise Flink VECTOR support, which this branch does not have. VECTOR is Spark-only in 1.2.1: HoodieSchemaType.VECTOR exists in hudi-common and the Spark modules implement it, but there is not a single HoodieSchemaType.VECTOR reference under hudi-flink-datasource or hudi-flink-client -- HoodieSchemaConverter only carries a guard that throws "Unsupported VECTOR element type". Specifically, they need: - AvroToRowDataConverters#createRowConverter(HoodieSchema, RowType, boolean) and org.apache.hudi.util.VectorConversionUtils, from aac975c9ff34 (#18723, "feat(flink): Support reading VECTOR columns from Parquet and Avro format") - ParquetRowDataWriter(RecordConsumer, boolean, HoodieSchema); this branch's constructor takes a RowType, from 7af4fb9acfae (#18877, "feat(flink): Support writing VECTOR columns for flink writer") Backporting that pair (47 files) to satisfy test coverage would add a new Flink engine capability to a patch release, so the tests are dropped instead. The production fix is unaffected: the AvroToRowDataConverters date-conversion correction and the HoodieFlinkRecord change apply as-is, along with the four test files that do not touch VECTOR (TestHoodieFlinkInternalRowSerializer, TestHoodieFlinkRecord, TestParquetSchemaConverter, TestRowDataUtils). (cherry picked from commit 427efb2654296cd78c93e2602ca06f6f82bf0733) --- .../hudi/client/model/HoodieFlinkRecord.java | 6 +- .../hudi/util/AvroToRowDataConverters.java | 3 +- .../TestHoodieFlinkInternalRowSerializer.java | 125 +++++++++++++++++ .../client/model/TestHoodieFlinkRecord.java | 116 ++++++++++++++++ .../parquet/TestParquetSchemaConverter.java | 63 +++++++++ .../apache/hudi/util/TestRowDataUtils.java | 127 ++++++++++++++++++ 6 files changed, 435 insertions(+), 5 deletions(-) create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkInternalRowSerializer.java create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestRowDataUtils.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java index b44397abc53cb..74990bc5835d1 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java @@ -46,7 +46,6 @@ import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.utils.JoinedRowData; import java.io.ByteArrayOutputStream; @@ -170,11 +169,10 @@ public Object convertColumnValueForLogicalType(HoodieSchema fieldSchema, return LocalDate.ofEpochDay(((Integer) fieldValue).longValue()); } else if (schemaType == HoodieSchemaType.TIMESTAMP && keepConsistentLogicalTimestamp) { HoodieSchema.Timestamp timestampSchema = (HoodieSchema.Timestamp) fieldSchema; - TimestampData ts = (TimestampData) fieldValue; if (timestampSchema.getPrecision() == HoodieSchema.TimePrecision.MILLIS) { - return ts.getMillisecond(); + return fieldValue; } else if (timestampSchema.getPrecision() == HoodieSchema.TimePrecision.MICROS) { - return ts.getMillisecond() / 1000; + return ((Long) fieldValue) / 1000; } } else if (schemaType == HoodieSchemaType.DECIMAL) { return ((DecimalData) fieldValue).toBigDecimal(); diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java index d63ce350b487c..0da058bfd91b4 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java @@ -340,7 +340,8 @@ public static JodaConverter getConverter() { public long convertDate(Object object) { final org.joda.time.LocalDate value = (org.joda.time.LocalDate) object; - return value.toDate().getTime(); + return LocalDate.of( + value.getYear(), value.getMonthOfYear(), value.getDayOfMonth()).toEpochDay(); } public int convertTime(Object object) { diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkInternalRowSerializer.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkInternalRowSerializer.java new file mode 100644 index 0000000000000..60227372c7f18 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkInternalRowSerializer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.model; + +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.RowType; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestHoodieFlinkInternalRowSerializer { + + private static final RowType ROW_TYPE = (RowType) DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(10, 2)), + DataTypes.FIELD("created", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)), + DataTypes.FIELD("tags", DataTypes.ARRAY(DataTypes.STRING()))) + .getLogicalType(); + + @Test + void testRoundTripCopyAndStreamCopy() throws Exception { + HoodieFlinkInternalRowSerializer serializer = new HoodieFlinkInternalRowSerializer(ROW_TYPE); + GenericRowData payload = GenericRowData.of( + StringData.fromString("alice"), + DecimalData.fromBigDecimal(new BigDecimal("12.34"), 10, 2), + TimestampData.fromInstant(Instant.parse("2025-02-03T04:05:06.123456Z")), + new GenericArrayData(new Object[] {StringData.fromString("x"), null})); + HoodieFlinkInternalRow original = new HoodieFlinkInternalRow( + "key", "partition", "file", "instant", "I", false, payload); + + DataOutputSerializer output = new DataOutputSerializer(128); + serializer.serialize(original, output); + HoodieFlinkInternalRow restored = serializer.deserialize( + new DataInputDeserializer(output.getCopyOfBuffer())); + assertDataRecord(restored); + + HoodieFlinkInternalRow copied = serializer.copy(original); + assertNotSame(original, copied); + assertNotSame(original.getRowData(), copied.getRowData()); + assertDataRecord(copied); + + DataOutputSerializer copiedOutput = new DataOutputSerializer(128); + serializer.copy(new DataInputDeserializer(output.getCopyOfBuffer()), copiedOutput); + assertDataRecord(serializer.deserialize(new DataInputDeserializer(copiedOutput.getCopyOfBuffer()))); + } + + @Test + void testIndexRecordRoundTripAndSerializerContract() throws Exception { + HoodieFlinkInternalRowSerializer serializer = new HoodieFlinkInternalRowSerializer(ROW_TYPE); + HoodieFlinkInternalRow indexRecord = + new HoodieFlinkInternalRow("key", "partition", "file", "instant"); + DataOutputSerializer output = new DataOutputSerializer(64); + serializer.serialize(indexRecord, output); + + HoodieFlinkInternalRow restored = serializer.deserialize( + indexRecord, new DataInputDeserializer(output.getCopyOfBuffer())); + assertTrue(restored.isIndexRecord()); + assertEquals("key", restored.getRecordKey()); + assertEquals("partition", restored.getPartitionPath()); + assertEquals("file", restored.getFileId()); + assertEquals("instant", restored.getInstantTime()); + assertEquals("", restored.getOperationType()); + + DataOutputSerializer copiedOutput = new DataOutputSerializer(64); + serializer.copy(new DataInputDeserializer(output.getCopyOfBuffer()), copiedOutput); + assertTrue(serializer.deserialize( + new DataInputDeserializer(copiedOutput.getCopyOfBuffer())).isIndexRecord()); + + assertFalse(serializer.isImmutableType()); + assertEquals(-1, serializer.getLength()); + assertEquals(serializer, serializer.duplicate()); + assertEquals(serializer.hashCode(), serializer.duplicate().hashCode()); + assertFalse(serializer.equals(null)); + assertFalse(serializer.equals("serializer")); + assertThrows(UnsupportedOperationException.class, serializer::createInstance); + assertThrows(UnsupportedOperationException.class, + () -> serializer.copy(indexRecord, indexRecord)); + assertThrows(UnsupportedOperationException.class, serializer::snapshotConfiguration); + } + + private static void assertDataRecord(HoodieFlinkInternalRow record) { + assertFalse(record.isIndexRecord()); + assertEquals("key", record.getRecordKey()); + assertEquals("partition", record.getPartitionPath()); + assertEquals("file", record.getFileId()); + assertEquals("instant", record.getInstantTime()); + assertEquals("I", record.getOperationType()); + assertEquals("alice", record.getRowData().getString(0).toString()); + assertEquals(new BigDecimal("12.34"), record.getRowData().getDecimal(1, 10, 2).toBigDecimal()); + assertEquals(Instant.parse("2025-02-03T04:05:06.123456Z"), + record.getRowData().getTimestamp(2, 6).toInstant()); + assertEquals("x", record.getRowData().getArray(3).getString(0).toString()); + assertTrue(record.getRowData().getArray(3).isNullAt(1)); + } +} diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkRecord.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkRecord.java index 6744e2d14830b..9753628da4101 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkRecord.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/model/TestHoodieFlinkRecord.java @@ -20,20 +20,33 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieOperation; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.OrderingValues; +import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; import java.util.Arrays; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Unit tests for {@link HoodieFlinkRecord}. @@ -266,4 +279,107 @@ public void testUpdateMetaFieldWithoutOperationField() { HoodieFlinkRecord updatedRecord = (HoodieFlinkRecord) record.updateMetaField(schema, 0, "20240101000001"); assertEquals(HoodieOperation.INSERT, updatedRecord.getOperation()); } + + @Test + public void testRecordContract() { + HoodieKey key = new HoodieKey("id-001", "partition-1"); + GenericRowData row = GenericRowData.of(StringData.fromString("id-001")); + HoodieFlinkRecord record = new HoodieFlinkRecord( + key, HoodieOperation.INSERT, 100L, row, true); + + assertEquals(HoodieRecord.HoodieRecordType.FLINK, record.getRecordType()); + assertEquals(key, record.newInstance().getKey()); + assertEquals("new-key", record.newInstance( + new HoodieKey("new-key", "p"), HoodieOperation.UPDATE_AFTER).getRecordKey()); + assertEquals("another-key", record.newInstance( + new HoodieKey("another-key", "p")).getRecordKey()); + assertFalse(record.shouldIgnore(null, new Properties())); + assertSame(record, record.copy()); + assertTrue(record.getMetadata().isEmpty()); + + HoodieFlinkRecord empty = new HoodieFlinkRecord( + key, HoodieOperation.INSERT, (RowData) null); + assertTrue(empty.checkIsDelete(null, new Properties())); + } + + @Test + public void testConvertColumnValueForLogicalType() { + TimestampData timestamp = TimestampData.fromInstant( + Instant.parse("2025-02-03T04:05:06.123456Z")); + HoodieFlinkRecord record = new HoodieFlinkRecord(GenericRowData.of(timestamp)); + + assertNull(record.convertColumnValueForLogicalType( + HoodieSchema.create(HoodieSchemaType.STRING), null, true)); + assertEquals(LocalDate.ofEpochDay(2), record.convertColumnValueForLogicalType( + HoodieSchema.createDate(), 2, true)); + + HoodieSchema millisSchema = HoodieSchema.createTimestampMillis(); + HoodieSchema millisRecordSchema = HoodieSchema.createRecord( + "millis_record", null, null, + Arrays.asList(HoodieSchemaField.of("event_time", millisSchema))); + Object millisValue = record.getColumnValueAsJava( + millisRecordSchema, "event_time", new Properties()); + assertEquals(timestamp.getMillisecond(), millisValue); + assertEquals(timestamp.getMillisecond(), record.convertColumnValueForLogicalType( + millisSchema, millisValue, true)); + + HoodieSchema microsSchema = HoodieSchema.createTimestampMicros(); + HoodieSchema microsRecordSchema = HoodieSchema.createRecord( + "micros_record", null, null, + Arrays.asList(HoodieSchemaField.of("event_time", microsSchema))); + Object microsValue = record.getColumnValueAsJava( + microsRecordSchema, "event_time", new Properties()); + long expectedMicros = timestamp.toInstant().getEpochSecond() * 1_000_000 + + timestamp.toInstant().getNano() / 1_000; + assertEquals(expectedMicros, microsValue); + assertEquals(timestamp.getMillisecond(), record.convertColumnValueForLogicalType( + microsSchema, microsValue, true)); + + assertEquals(new BigDecimal("12.34"), record.convertColumnValueForLogicalType( + HoodieSchema.createDecimal("decimal", null, null, 10, 2, 5), + DecimalData.fromBigDecimal(new BigDecimal("12.34"), 10, 2), true)); + assertSame(millisValue, record.convertColumnValueForLogicalType( + millisSchema, millisValue, false)); + } + + @Test + public void testUnsupportedOperations() { + HoodieKey key = new HoodieKey("id-001", "partition-1"); + GenericRowData row = GenericRowData.of(StringData.fromString("id-001")); + HoodieFlinkRecord record = new HoodieFlinkRecord( + key, HoodieOperation.INSERT, 100L, row, true); + assertThrows(UnsupportedOperationException.class, + () -> record.writeRecordPayload(row, null, null)); + assertThrows(UnsupportedOperationException.class, + () -> record.readRecordPayload(null, null)); + assertThrows(UnsupportedOperationException.class, + () -> record.getColumnValues(null, null, false)); + assertThrows(UnsupportedOperationException.class, + () -> record.joinWith(record, null)); + assertThrows(UnsupportedOperationException.class, + () -> record.wrapIntoHoodieRecordPayloadWithParams( + null, null, Option.empty(), false, Option.empty(), false, Option.empty())); + assertThrows(UnsupportedOperationException.class, + () -> record.wrapIntoHoodieRecordPayloadWithKeyGen(null, null, Option.empty())); + assertThrows(UnsupportedOperationException.class, + () -> record.truncateRecordKey(null, null, null)); + } + + @Test + public void testKeyLookupAndAvroMaterialization() { + HoodieSchema schema = HoodieSchema.createRecord("test", null, null, Arrays.asList( + HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.STRING)), + HoodieSchemaField.of("value", HoodieSchema.create(HoodieSchemaType.INT)))); + HoodieFlinkRecord record = new HoodieFlinkRecord(GenericRowData.of( + StringData.fromString("id-001"), 42)); + + assertEquals("id-001", record.getRecordKey(schema, "id")); + // The second lookup exercises the cached record-key path. + assertEquals("id-001", record.getRecordKey(schema, "id")); + assertEquals("id-001", record.toIndexedRecord(schema, new Properties()) + .get().getData().get(0).toString()); + assertTrue(record.getAvroBytes(schema, new Properties()).size() > 0); + assertEquals(OrderingValues.getDefault(), + record.getOrderingValueAsJava(schema, new Properties(), null)); + } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java index 403b840684965..c588466f41c9a 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/parquet/TestParquetSchemaConverter.java @@ -230,6 +230,69 @@ void testConvertTimestampTypes() { assertThat(messageType.toString(), is(expected)); } + @Test + void testConvertAnnotatedPrimitiveParquetTypes() { + MessageType parquet = new MessageType("annotated", + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.REQUIRED) + .as(LogicalTypeAnnotation.decimalType(2, 8)).named("decimal32"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.intType(8, true)).named("tiny"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.intType(16, true)).named("small"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.intType(32, true)).named("number"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.dateType()).named("day"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MILLIS)).named("time"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.decimalType(3, 12)).named("decimal64"), + Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, Type.Repetition.OPTIONAL) + .named("timestamp96"), + Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, Type.Repetition.OPTIONAL) + .length(8).as(LogicalTypeAnnotation.decimalType(4, 16)).named("decimal_fixed")); + + RowType converted = ParquetSchemaConverter.convertToRowType(parquet); + assertEquals("DECIMAL(8, 2) NOT NULL", converted.getTypeAt(0).asSummaryString()); + assertEquals("TINYINT", converted.getTypeAt(1).asSummaryString()); + assertEquals("SMALLINT", converted.getTypeAt(2).asSummaryString()); + assertEquals("INT", converted.getTypeAt(3).asSummaryString()); + assertEquals("DATE", converted.getTypeAt(4).asSummaryString()); + assertEquals("TIME(0)", converted.getTypeAt(5).asSummaryString()); + assertEquals("DECIMAL(12, 3)", converted.getTypeAt(6).asSummaryString()); + assertEquals("TIMESTAMP(9)", converted.getTypeAt(7).asSummaryString()); + assertEquals("DECIMAL(16, 4)", converted.getTypeAt(8).asSummaryString()); + } + + @Test + void testConvertLocalZonedTimestampToParquet() { + RowType rowType = (RowType) DataTypes.ROW( + DataTypes.FIELD("local_millis", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("local_micros", DataTypes.TIMESTAMP_LTZ(6))).notNull().getLogicalType(); + MessageType parquet = ParquetSchemaConverter.convertToParquetMessageType("local", rowType); + assertEquals(LogicalTypeAnnotation.timestampType( + false, LogicalTypeAnnotation.TimeUnit.MILLIS), + parquet.getType("local_millis").getLogicalTypeAnnotation()); + assertEquals(LogicalTypeAnnotation.timestampType( + false, LogicalTypeAnnotation.TimeUnit.MICROS), + parquet.getType("local_micros").getLogicalTypeAnnotation()); + } + + @Test + void testConvertBinaryDateAndTimeToParquet() { + RowType rowType = (RowType) DataTypes.ROW( + DataTypes.FIELD("payload", DataTypes.BYTES()), + DataTypes.FIELD("day", DataTypes.DATE()), + DataTypes.FIELD("time", DataTypes.TIME(3))).notNull().getLogicalType(); + MessageType parquet = ParquetSchemaConverter.convertToParquetMessageType("primitives", rowType); + assertEquals(PrimitiveType.PrimitiveTypeName.BINARY, + parquet.getType("payload").asPrimitiveType().getPrimitiveTypeName()); + assertEquals(LogicalTypeAnnotation.dateType(), + parquet.getType("day").getLogicalTypeAnnotation()); + assertEquals(LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MILLIS), + parquet.getType("time").getLogicalTypeAnnotation()); + } + /** * A Parquet group with metadata + value binary fields but NO VARIANT annotation must be * treated as a plain ROW. Only the Parquet {@code VARIANT} annotation triggers variant diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestRowDataUtils.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestRowDataUtils.java new file mode 100644 index 0000000000000..0249e74028cf0 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestRowDataUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.util; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.TimestampType; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class TestRowDataUtils { + + @Test + void testJavaAndFlinkValueConverters() { + assertNull(RowDataUtils.NULL_GETTER.getFieldOrNull(GenericRowData.of(1))); + assertNull(RowDataUtils.javaValFunc(DataTypes.NULL().getLogicalType(), true).apply("ignored")); + assertEquals(7, RowDataUtils.javaValFunc(DataTypes.TINYINT().getLogicalType(), true).apply((byte) 7)); + assertEquals(8, RowDataUtils.javaValFunc(DataTypes.SMALLINT().getLogicalType(), true).apply((short) 8)); + assertEquals(2, RowDataUtils.javaValFunc(DataTypes.DATE().getLogicalType(), true).apply(2)); + assertEquals("text", RowDataUtils.javaValFunc(DataTypes.STRING().getLogicalType(), true) + .apply(StringData.fromString("text"))); + assertArrayEquals(new byte[] {1, 2}, ((ByteBuffer) RowDataUtils.javaValFunc( + DataTypes.BYTES().getLogicalType(), true).apply(new byte[] {1, 2})).array()); + assertEquals(new BigDecimal("12.30"), RowDataUtils.javaValFunc( + DataTypes.DECIMAL(8, 2).getLogicalType(), true) + .apply(DecimalData.fromBigDecimal(new BigDecimal("12.30"), 8, 2))); + + assertEquals((byte) 7, RowDataUtils.flinkValFunc(DataTypes.TINYINT().getLogicalType(), true).apply((byte) 7)); + assertEquals((short) 8, RowDataUtils.flinkValFunc(DataTypes.SMALLINT().getLogicalType(), true).apply((short) 8)); + assertEquals(3, RowDataUtils.flinkValFunc(DataTypes.DATE().getLogicalType(), true) + .apply(LocalDate.ofEpochDay(3))); + assertEquals("text", RowDataUtils.flinkValFunc(DataTypes.STRING().getLogicalType(), true) + .apply("text").toString()); + ByteBuffer buffer = ByteBuffer.wrap(new byte[] {3, 4}); + assertSame(buffer, RowDataUtils.flinkValFunc(DataTypes.BYTES().getLogicalType(), true).apply(buffer)); + assertEquals(new BigDecimal("45.60"), ((DecimalData) RowDataUtils.flinkValFunc( + DataTypes.DECIMAL(8, 2).getLogicalType(), true).apply(new BigDecimal("45.60"))).toBigDecimal()); + } + + @Test + void testTimestampConvertersAtMillisAndMicros() { + TimestampData timestamp = TimestampData.fromInstant(Instant.parse("2025-02-03T04:05:06.123456Z")); + long millis = timestamp.toInstant().toEpochMilli(); + long micros = timestamp.toInstant().getEpochSecond() * 1_000_000 + 123456; + + assertEquals(millis, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP_LTZ(3).getLogicalType(), true) + .apply(timestamp)); + assertEquals(micros, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP_LTZ(6).getLogicalType(), true) + .apply(timestamp)); + assertEquals(millis, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP(3).getLogicalType(), true) + .apply(timestamp)); + assertEquals(micros, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP(6).getLogicalType(), true) + .apply(timestamp)); + long localMillis = (long) RowDataUtils.javaValFunc( + DataTypes.TIMESTAMP(3).getLogicalType(), false).apply(timestamp); + long localMicros = (long) RowDataUtils.javaValFunc( + DataTypes.TIMESTAMP(6).getLogicalType(), false).apply(timestamp); + + assertEquals(Instant.ofEpochMilli(millis), ((TimestampData) RowDataUtils.flinkValFunc( + DataTypes.TIMESTAMP_LTZ(3).getLogicalType(), true).apply(millis)).toInstant()); + assertEquals(timestamp.toInstant(), ((TimestampData) RowDataUtils.flinkValFunc( + DataTypes.TIMESTAMP_LTZ(6).getLogicalType(), true).apply(micros)).toInstant()); + assertEquals(millis, ((TimestampData) RowDataUtils.flinkValFunc( + DataTypes.TIMESTAMP(3).getLogicalType(), false).apply(millis)).toTimestamp().getTime()); + assertEquals(timestamp.toInstant(), ((TimestampData) RowDataUtils.flinkValFunc( + DataTypes.TIMESTAMP(6).getLogicalType(), true).apply(micros)).toInstant()); + assertEquals(localMillis, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP(3).getLogicalType(), false) + .apply(RowDataUtils.flinkValFunc(DataTypes.TIMESTAMP(3).getLogicalType(), false).apply(localMillis))); + assertEquals(localMicros, RowDataUtils.javaValFunc(DataTypes.TIMESTAMP(6).getLogicalType(), false) + .apply(RowDataUtils.flinkValFunc(DataTypes.TIMESTAMP(6).getLogicalType(), false).apply(localMicros))); + + assertThrows(UnsupportedOperationException.class, + () -> RowDataUtils.javaValFunc(DataTypes.TIMESTAMP(9).getLogicalType(), true)); + assertThrows(UnsupportedOperationException.class, + () -> RowDataUtils.flinkValFunc(DataTypes.TIMESTAMP_LTZ(9).getLogicalType(), true)); + } + + @Test + void testGenericConversionAndPrecision() { + assertNull(RowDataUtils.convertValueToFlinkType(null)); + assertEquals("value", RowDataUtils.convertValueToFlinkType("value").toString()); + assertEquals(new BigDecimal("1.20"), ((DecimalData) RowDataUtils.convertValueToFlinkType( + new BigDecimal("1.20"))).toBigDecimal()); + Timestamp timestamp = Timestamp.valueOf("2025-02-03 04:05:06.123456"); + assertEquals(timestamp, ((TimestampData) RowDataUtils.convertValueToFlinkType(timestamp)).toTimestamp()); + assertEquals(2, RowDataUtils.convertValueToFlinkType(LocalDate.ofEpochDay(2))); + assertArrayEquals(new byte[] {9, 8}, + (byte[]) RowDataUtils.convertValueToFlinkType(ByteBuffer.wrap(new byte[] {9, 8}))); + Object marker = new Object(); + assertSame(marker, RowDataUtils.convertValueToFlinkType(marker)); + assertEquals(3, RowDataUtils.precision(new TimestampType(3))); + assertEquals(6, RowDataUtils.precision(new LocalZonedTimestampType(6))); + assertThrows(AssertionError.class, + () -> RowDataUtils.precision(DataTypes.INT().getLogicalType())); + } +} From 67b1072602bc96d700d7d0ae81f37cd1da01b357 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 23 Jul 2026 18:12:29 +0800 Subject: [PATCH 196/255] fix(flink): deduplicate delete keys and cover write paths (#19354) Adapted for release-1.2.1: - Dropped the TestFlinkWriteHandleFactory changes. That test file does not exist here; it arrives with 9f53c46d0de8 (#19307, "feat(flink): enable LSM end-to-end for Flink by default"), which is not backported, and it exercises RowDataInlineLogWriteHandle, also absent. - TestHoodieFlinkTableActionRouting: mock HoodieAppendHandle instead of HoodieInlineLogAppendHandle. Master split that handle for native logs (#19067) into HoodieNativeLogAppendHandle / HoodieInlineLogAppendHandle; this branch has no native log format, and HoodieFlinkMergeOnReadTable#handleInsertsForLogCompaction constructs HoodieAppendHandle. Mocking the class actually constructed keeps the assertion meaningful. - TestHoodieFlinkTableServiceClient: kept this branch's org.apache.hudi.client.transaction.lock.InProcessLockProvider import, which is already present, rather than master's core.transaction.lock path; added only the HoodieException import the commit needs. The delete-key deduplication fix itself (FlinkDeleteHelper, JavaDeleteHelper) applied unchanged. (cherry picked from commit 4d11a60552e43aae88e5dfcfbbdb31d287505273) --- .../action/commit/FlinkDeleteHelper.java | 7 +- .../hudi/client/TestFlinkWriteClient.java | 36 ++ .../TestFlinkWriteClientFunctional.java | 491 ++++++++++++++++++ .../TestHoodieFlinkTableServiceClient.java | 185 +++++++ .../TestHoodieFlinkTableActionRouting.java | 263 ++++++++++ .../action/commit/TestFlinkDeleteHelper.java | 174 +++++++ .../table/action/commit/JavaDeleteHelper.java | 16 +- .../action/commit/TestJavaDeleteHelper.java | 72 +++ 8 files changed, 1231 insertions(+), 13 deletions(-) create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClientFunctional.java create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/TestHoodieFlinkTableActionRouting.java create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/commit/TestFlinkDeleteHelper.java create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaDeleteHelper.java diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java index 7dfc8a336c325..733b79fa9987e 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; @@ -65,16 +66,16 @@ public static FlinkDeleteHelper newInstance() { public List deduplicateKeys(List keys, HoodieTable>, List, List> table, int parallelism) { boolean isIndexingGlobal = table.getIndex().isGlobal(); if (isIndexingGlobal) { - HashSet recordKeys = keys.stream().map(HoodieKey::getRecordKey).collect(Collectors.toCollection(HashSet::new)); + HashSet recordKeys = new HashSet<>(); List deduplicatedKeys = new LinkedList<>(); keys.forEach(x -> { - if (recordKeys.contains(x.getRecordKey())) { + if (recordKeys.add(x.getRecordKey())) { deduplicatedKeys.add(x); } }); return deduplicatedKeys; } else { - HashSet set = new HashSet<>(keys); + LinkedHashSet set = new LinkedHashSet<>(keys); keys.clear(); keys.addAll(set); return keys; diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java index d0ae6a30a54a4..bf41c13426bbf 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java @@ -23,11 +23,15 @@ import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.EngineType; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; @@ -39,10 +43,12 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.util.Collections; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -157,4 +163,34 @@ public void testCleanResourcesCleansMetadataTableHeartbeatForStreamingMetadataWr assertFalse(HoodieHeartbeatClient.heartbeatExists( metaClient.getStorage(), metadataTableBasePath, instantTime)); } + + @Test + void testUnsupportedWriteEntryPointsAndInvalidTableServiceFailFast() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withEngineType(EngineType.FLINK) + .withEmbeddedTimelineServerEnabled(false) + .build(); + writeClient = new HoodieFlinkWriteClient(context, writeConfig); + + assertThrows(HoodieNotSupportedException.class, () -> writeClient.bootstrap(Option.empty())); + assertThrows(HoodieNotSupportedException.class, + () -> writeClient.insertPreppedRecords(Collections.emptyList(), "001")); + assertThrows(HoodieNotSupportedException.class, + () -> writeClient.bulkInsert(Collections.emptyList(), "001")); + assertThrows(HoodieNotSupportedException.class, + () -> writeClient.bulkInsert(Collections.emptyList(), "001", Option.empty())); + assertThrows(HoodieNotSupportedException.class, + () -> writeClient.cluster("001", false)); + assertThrows(HoodieException.class, + () -> writeClient.delete(Collections.singletonList(new HoodieKey("id", "partition")), "001")); + assertThrows(HoodieException.class, + () -> writeClient.deletePrepped(Collections.emptyList(), "001")); + assertThrows(IllegalArgumentException.class, + () -> writeClient.completeTableService(TableServiceType.CLEAN, null, null, "001")); + assertFalse(writeClient.loadActiveTimelineOnTableInit()); + writeClient.waitForCleaningFinish(); + writeClient.cleanHandles(); + assertNotNull(writeClient.getHoodieTable(false)); + } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClientFunctional.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClientFunctional.java new file mode 100644 index 0000000000000..4d5e2670db144 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClientFunctional.java @@ -0,0 +1,491 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.avro.model.HoodieClusteringPlan; +import org.apache.hudi.client.clustering.plan.strategy.FlinkSizeBasedClusteringPlanStrategyRecently; +import org.apache.hudi.client.common.HoodieFlinkEngineContext; +import org.apache.hudi.client.model.HoodieFlinkRecord; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieOperation; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.ClusteringUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieClusteringConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.FlinkCreateHandle; +import org.apache.hudi.io.FlinkMergeHandle; +import org.apache.hudi.io.FlinkWriteHandleFactory; +import org.apache.hudi.io.HoodieWriteMergeHandle; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.table.HoodieFlinkTable; +import org.apache.hudi.table.action.HoodieWriteMetadata; +import org.apache.hudi.table.action.commit.BucketInfo; +import org.apache.hudi.table.action.commit.BucketType; +import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Functional coverage for the Flink client write boundary. + * + *

    The datasource bucket assigner hands this client records that already carry a target file group. + * These tests construct that same input directly so client and handle behavior can be exercised without + * depending on the datasource module. + */ +class TestFlinkWriteClientFunctional extends HoodieFlinkClientTestHarness { + + private static final String PARTITION_PATH = "2026/07/23"; + private static final String FILE_ID = "f0"; + private static final String SCHEMA = "{" + + "\"type\":\"record\"," + + "\"name\":\"flink_write_test\"," + + "\"fields\":[" + + "{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"name\",\"type\":\"string\"}," + + "{\"name\":\"ts\",\"type\":\"long\"}" + + "]}"; + + private HoodieWriteConfig writeConfig; + + @BeforeEach + void setUp() { + initPath(); + initFileSystem(); + } + + @AfterEach + void tearDown() throws IOException { + cleanupResources(); + } + + static Stream tableTypesAndCdc() { + return Stream.of( + Arguments.of(HoodieTableType.COPY_ON_WRITE, false), + Arguments.of(HoodieTableType.COPY_ON_WRITE, true), + Arguments.of(HoodieTableType.MERGE_ON_READ, false), + Arguments.of(HoodieTableType.MERGE_ON_READ, true)); + } + + @ParameterizedTest + @MethodSource("tableTypesAndCdc") + void testInsertAndUpsertWriteFilesAndCommitMetadata(HoodieTableType tableType, boolean cdcEnabled) + throws IOException { + initWriteClient(tableType, cdcEnabled, false); + + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + List firstBatch = new ArrayList<>(Arrays.asList( + insertRecord("id1", "one", 1L), + insertRecord("id2", "two", 2L))); + if (tableType == HoodieTableType.MERGE_ON_READ) { + firstBatch.add(insertRecord("id3", "three", 3L)); + } + List firstInsertStatuses = writeClient.insert(firstBatch, insertInstant); + assertWriteStatuses(firstInsertStatuses, firstBatch.size()); + + // A second COW mini-batch for the same bucket exercises the incremental/replace handle. + List insertStatuses = tableType == HoodieTableType.COPY_ON_WRITE + ? writeClient.insert(Arrays.asList(insertRecord("id3", "three", 3L)), insertInstant) + : firstInsertStatuses; + if (tableType == HoodieTableType.COPY_ON_WRITE && cdcEnabled) { + insertStatuses = writeClient.upsert( + Arrays.asList(updateRecord("id1", "one-mini-batch-update", 4L, insertInstant)), + insertInstant); + } + assertWriteStatuses(insertStatuses, 3); + assertTrue(writeClient.commit(insertInstant, insertStatuses)); + assertCommitMetadata(insertInstant, tableType, 3); + + writeClient.cleanHandles(); + String updateInstant = writeClient.startCommit(); + transitionToInflight(updateInstant); + List updates = Arrays.asList( + updateRecord("id1", "one-updated", 11L, insertInstant), + deleteRecord("id2", 12L, insertInstant)); + List updateStatuses = writeClient.upsert(updates, updateInstant); + long expectedWrites = tableType == HoodieTableType.COPY_ON_WRITE ? 2 : 1; + assertWriteStatuses(updateStatuses, expectedWrites); + assertEquals(1, + updateStatuses.stream().map(WriteStatus::getStat).mapToLong(stat -> stat.getNumDeletes()).sum()); + assertTrue(writeClient.commit(updateInstant, updateStatuses)); + assertCommitMetadata(updateInstant, tableType, expectedWrites); + + if (tableType == HoodieTableType.COPY_ON_WRITE && cdcEnabled) { + assertTrue(updateStatuses.stream() + .map(WriteStatus::getStat) + .anyMatch(stat -> stat.getCdcStats() != null && !stat.getCdcStats().isEmpty())); + } + } + + @Test + void testCopyOnWriteCleansRetryFiles() throws IOException { + context = new HoodieFlinkEngineContext( + new HoodieFlinkEngineContext.DefaultTaskContextSupplier() { + @Override + public Supplier getAttemptIdSupplier() { + return () -> 1L; + } + }); + initWriteClient(HoodieTableType.COPY_ON_WRITE, false, false); + + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + StoragePath staleInsertPath = createInvalidRetryFile(insertInstant); + List insertStatuses = writeClient.insert( + Collections.singletonList(insertRecord("id1", "one", 1L)), insertInstant); + assertWriteStatuses(insertStatuses, 1); + assertFalse(metaClient.getStorage().exists(staleInsertPath)); + assertTrue(writeClient.commit(insertInstant, insertStatuses)); + + writeClient.cleanHandles(); + String updateInstant = writeClient.startCommit(); + transitionToInflight(updateInstant); + StoragePath staleUpdatePath = createInvalidRetryFile(updateInstant); + List updateStatuses = writeClient.upsert( + Collections.singletonList(updateRecord("id1", "one-updated", 2L, insertInstant)), + updateInstant); + assertWriteStatuses(updateStatuses, 1); + assertFalse(metaClient.getStorage().exists(staleUpdatePath)); + assertTrue(writeClient.commit(updateInstant, updateStatuses)); + } + + @Test + void testCopyOnWriteHandleRolloverAndGracefulCloseCleanup() throws IOException { + initWriteClient(HoodieTableType.COPY_ON_WRITE, false, false); + + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + writeClient.insert(Arrays.asList( + insertRecord("id1", "one", 1L), + insertRecord("id2", "two", 2L)), insertInstant); + List insertStatuses = writeClient.insert( + Collections.singletonList(insertRecord("id3", "three", 3L)), insertInstant); + assertWriteStatuses(insertStatuses, 3); + + HoodieFlinkTable table = writeClient.getHoodieTable(); + FlinkCreateHandle rolloverHandle = new FlinkCreateHandle( + writeConfig, insertInstant, table, PARTITION_PATH, FILE_ID, table.getTaskContextSupplier()); + assertTrue(rolloverHandle.canWrite(insertRecord("id4", "four", 4L))); + assertNotEquals(insertStatuses.get(0).getStat().getPath(), rolloverHandle.getWritePath().toString()); + rolloverHandle.closeGracefully(); + // Closing gracefully is intentionally idempotent. + rolloverHandle.closeGracefully(); + + FlinkCreateHandle failingHandle = new FlinkCreateHandle( + writeConfig, insertInstant, table, PARTITION_PATH, FILE_ID, table.getTaskContextSupplier()) { + @Override + public List close() { + super.close(); + throw new IllegalStateException("expected close failure"); + } + }; + StoragePath failedCreatePath = failingHandle.getWritePath(); + failingHandle.closeGracefully(); + assertFalse(metaClient.getStorage().exists(failedCreatePath)); + assertTrue(writeClient.commit(insertInstant, insertStatuses)); + + String mergeInstant = writeClient.startCommit(); + transitionToInflight(mergeInstant); + table = writeClient.getHoodieTable(); + FlinkMergeHandle failingMergeHandle = new FlinkMergeHandle( + writeConfig, + mergeInstant, + table, + Collections.emptyList().iterator(), + PARTITION_PATH, + FILE_ID, + table.getTaskContextSupplier()) { + @Override + public List close() { + super.close(); + throw new IllegalStateException("expected close failure"); + } + }; + StoragePath failedMergePath = failingMergeHandle.getWritePath(); + failingMergeHandle.closeGracefully(); + assertFalse(metaClient.getStorage().exists(failedMergePath)); + } + + @Test + void testScheduleClusteringFromRecentlyWrittenPartition() throws IOException { + initWriteClient(HoodieTableType.COPY_ON_WRITE, false, true); + + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + List statuses = writeClient.insert(Arrays.asList( + insertRecord("id1", "one", 1L), + insertRecord("id2", "two", 2L)), insertInstant); + assertTrue(writeClient.commit(insertInstant, statuses)); + + Option clusteringInstant = writeClient.scheduleClustering(Option.empty()); + assertTrue(clusteringInstant.isPresent()); + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieInstant requestedInstant = metaClient.getInstantGenerator() + .getClusteringCommitRequestedInstant(clusteringInstant.get()); + HoodieClusteringPlan clusteringPlan = ClusteringUtils + .getClusteringPlan(metaClient, requestedInstant).get().getRight(); + assertEquals(1, clusteringPlan.getInputGroups().size()); + assertEquals(PARTITION_PATH, + clusteringPlan.getInputGroups().get(0).getSlices().get(0).getPartitionPath()); + } + + @Test + void testPreppedWriteEntryPointsCommitMetadata() throws IOException { + initWriteClient(HoodieTableType.COPY_ON_WRITE, false, false); + + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + List insertStatuses = writeClient.insert(Arrays.asList( + insertRecord("id1", "one", 1L), + insertRecord("id2", "two", 2L)), insertInstant); + assertTrue(writeClient.commit(insertInstant, insertStatuses)); + + writeClient.cleanHandles(); + String upsertInstant = writeClient.startCommit(); + transitionToInflight(upsertInstant); + List upsertStatuses = writeClient.upsertPreppedRecords( + Collections.singletonList(updateRecord("id1", "one-prepped", 3L, insertInstant)), + upsertInstant); + assertWriteStatuses(upsertStatuses, 2); + assertTrue(writeClient.commit(upsertInstant, upsertStatuses)); + assertCommitMetadata(upsertInstant, HoodieTableType.COPY_ON_WRITE, 2); + + writeClient.cleanHandles(); + String bulkInsertInstant = writeClient.startCommit(); + transitionToInflight(bulkInsertInstant); + List bulkInsertStatuses = writeClient.bulkInsertPreppedRecords( + Collections.singletonList(insertRecord("id3", "three", 4L)), + bulkInsertInstant, + Option.empty()); + assertWriteStatuses(bulkInsertStatuses, 1); + assertTrue(writeClient.commit(bulkInsertInstant, bulkInsertStatuses)); + assertCommitMetadata(bulkInsertInstant, HoodieTableType.COPY_ON_WRITE, 1); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + void testClientRoutesOverwriteAndDeleteActionsAfterPriorCommit() throws IOException { + initWriteClient(HoodieTableType.COPY_ON_WRITE, false, false); + String insertInstant = writeClient.startCommit(); + transitionToInflight(insertInstant); + List insertStatuses = writeClient.insert( + Collections.singletonList(insertRecord("id1", "one", 1L)), insertInstant); + assertTrue(writeClient.commit(insertInstant, insertStatuses)); + Map> replacedFileIds = + writeClient.getPartitionToReplacedFileIds(WriteOperationType.INSERT_OVERWRITE, insertStatuses); + assertEquals(Collections.singleton(FILE_ID), + new HashSet<>(replacedFileIds.get(PARTITION_PATH))); + + HoodieFlinkTable table = mock(HoodieFlinkTable.class); + when(table.getMetaClient()).thenReturn(metaClient); + HoodieWriteMetadata> metadata = new HoodieWriteMetadata<>(); + metadata.setWriteStatuses(Collections.emptyList()); + when(table.insertOverwrite(any(), any(), any(), anyString(), any())).thenReturn(metadata); + when(table.insertOverwriteTable(any(), any(), any(), anyString(), any())).thenReturn(metadata); + when(table.delete(any(), anyString(), any())).thenReturn(metadata); + when(table.deletePrepped(any(), anyString(), any())).thenReturn(metadata); + when(table.deletePartitions(any(), anyString(), any())).thenReturn(metadata); + + HoodieFlinkWriteClient routingClient = new HoodieFlinkWriteClient(context, writeConfig) { + @Override + protected org.apache.hudi.table.HoodieTable createTable( + HoodieWriteConfig config, HoodieTableMetaClient ignoredMetaClient) { + return table; + } + }; + FlinkWriteHandleFactory.Factory handleFactory = mock(FlinkWriteHandleFactory.Factory.class); + FlinkCreateHandle writeHandle = mock(FlinkCreateHandle.class); + when(handleFactory.create(any(), any(), any(), anyString(), any(), any())).thenReturn(writeHandle); + BucketInfo bucketInfo = new BucketInfo(BucketType.INSERT, FILE_ID, PARTITION_PATH); + List deleteKeys = Collections.singletonList(new HoodieKey("id1", PARTITION_PATH)); + List preppedDeletes = Collections.emptyList(); + List partitions = Collections.singletonList(PARTITION_PATH); + + try (MockedStatic factory = Mockito.mockStatic(FlinkWriteHandleFactory.class)) { + factory.when(() -> FlinkWriteHandleFactory.getFactory(any(), any(), anyBoolean())) + .thenReturn(handleFactory); + assertTrue(routingClient.insertOverwrite( + Collections.emptyList().iterator(), bucketInfo, "001").isEmpty()); + assertTrue(routingClient.insertOverwriteTable( + Collections.emptyList().iterator(), bucketInfo, "002").isEmpty()); + assertTrue(routingClient.delete(deleteKeys, "003").isEmpty()); + assertTrue(routingClient.deletePrepped(preppedDeletes, "004").isEmpty()); + assertTrue(routingClient.deletePartitions(partitions, "005").isEmpty()); + + verify(table).insertOverwrite(eq(context), eq(writeHandle), eq(bucketInfo), eq("001"), any()); + verify(table).insertOverwriteTable(eq(context), eq(writeHandle), eq(bucketInfo), eq("002"), any()); + verify(table).delete(eq(context), eq("003"), eq(deleteKeys)); + verify(table).deletePrepped(eq(context), eq("004"), eq(preppedDeletes)); + verify(table).deletePartitions(eq(context), eq("005"), eq(partitions)); + } finally { + routingClient.close(); + } + } + + private HoodieRecord insertRecord(String key, String name, long ts) { + return record(key, name, ts, HoodieOperation.INSERT, "I"); + } + + private HoodieRecord updateRecord(String key, String name, long ts, String instantTime) { + return record(key, name, ts, HoodieOperation.UPDATE_AFTER, instantTime); + } + + private HoodieRecord deleteRecord(String key, long ts, String instantTime) { + return record(key, "deleted", ts, HoodieOperation.DELETE, instantTime); + } + + private HoodieRecord record( + String key, String name, long ts, HoodieOperation operation, String locationInstant) { + GenericRowData row = GenericRowData.of( + StringData.fromString(key), StringData.fromString(name), ts); + HoodieFlinkRecord record = new HoodieFlinkRecord( + new HoodieKey(key, PARTITION_PATH), operation, ts, row); + record.setCurrentLocation(new HoodieRecordLocation(locationInstant, FILE_ID)); + return record; + } + + private void initWriteClient( + HoodieTableType tableType, boolean cdcEnabled, boolean useRecentClusteringStrategy) + throws IOException { + Properties tableProperties = new Properties(); + tableProperties.setProperty(HoodieTableConfig.CDC_ENABLED.key(), Boolean.toString(cdcEnabled)); + tableProperties.setProperty( + HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE.key(), + HoodieCDCSupplementalLoggingMode.DATA_BEFORE_AFTER.name()); + tableProperties.setProperty(HoodieTableConfig.RECORDKEY_FIELDS.key(), "id"); + tableProperties.setProperty(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path"); + tableProperties.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(), "ts"); + tableProperties.setProperty( + HoodieWriteConfig.MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE.key(), "false"); + metaClient = HoodieTableMetaClient.newTableBuilder() + .setTableName("flink_write_client_test") + .setTableType(tableType) + .fromProperties(tableProperties) + .initTable(storageConf, basePath); + + HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withEngineType(EngineType.FLINK) + .withSchema(SCHEMA) + .withProperties(tableProperties) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .withMergeHandleClassName(HoodieWriteMergeHandle.class.getName()); + if (useRecentClusteringStrategy) { + builder.withClusteringConfig(HoodieClusteringConfig.newBuilder() + .withEngineType(EngineType.FLINK) + .withClusteringPlanStrategyClass(FlinkSizeBasedClusteringPlanStrategyRecently.class.getName()) + .withClusteringPlanSmallFileLimit(Long.MAX_VALUE) + .withClusteringMaxNumGroups(10) + .withClusteringSortColumns("id") + .build()); + } + writeConfig = builder.build(); + writeClient = new HoodieFlinkWriteClient<>(context, writeConfig); + } + + private void assertWriteStatuses(List statuses, long expectedRecords) { + assertFalse(statuses.isEmpty()); + assertTrue(statuses.stream().noneMatch(WriteStatus::hasErrors)); + assertEquals(expectedRecords, + statuses.stream().map(WriteStatus::getStat).mapToLong(stat -> stat.getNumWrites()).sum()); + statuses.forEach(status -> { + assertEquals(PARTITION_PATH, status.getStat().getPartitionPath()); + assertNotNull(status.getStat().getPath()); + }); + } + + private void transitionToInflight(String instantTime) { + metaClient.reloadActiveTimeline(); + metaClient.getActiveTimeline().transitionRequestedToInflight( + metaClient.getCommitActionType(), instantTime); + } + + private StoragePath createInvalidRetryFile(String instantTime) throws IOException { + StoragePath partitionPath = new StoragePath(metaClient.getBasePath(), PARTITION_PATH); + metaClient.getStorage().createDirectory(partitionPath); + String fileName = FSUtils.makeBaseFileName( + instantTime, + FSUtils.makeWriteToken(0, 1, 0), + FILE_ID, + metaClient.getTableConfig().getBaseFileFormat().getFileExtension()); + StoragePath retryPath = new StoragePath(partitionPath, fileName); + metaClient.getStorage().create(retryPath).close(); + return retryPath; + } + + private void assertCommitMetadata(String instantTime, HoodieTableType tableType, long expectedRecords) + throws IOException { + metaClient = HoodieTableMetaClient.reload(metaClient); + String action = metaClient.getCommitActionType(); + HoodieInstant instant = metaClient.getActiveTimeline() + .getTimelineOfActions(Collections.singleton(action)) + .filterCompletedInstants() + .getInstantsAsStream() + .filter(candidate -> candidate.requestedTime().equals(instantTime)) + .findFirst() + .orElseThrow(() -> new AssertionError("Missing completed instant " + instantTime)); + assertEquals(expectedRecords, + metaClient.getActiveTimeline().readCommitMetadata(instant).fetchTotalRecordsWritten()); + } +} diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java index e54793d3926ad..fde3711d9cb5b 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java @@ -22,25 +22,42 @@ import org.apache.hudi.client.common.HoodieFlinkEngineContext; import org.apache.hudi.client.transaction.lock.InProcessLockProvider; import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metadata.FlinkHoodieBackedTableMetadataWriter; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.table.HoodieFlinkTable; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.table.action.HoodieWriteMetadata; +import org.apache.hudi.table.action.compact.CompactHelpers; +import org.apache.hudi.table.marker.WriteMarkers; +import org.apache.hudi.table.marker.WriteMarkersFactory; import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; import org.mockito.Mockito; import java.io.IOException; +import java.util.Collections; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -75,11 +92,13 @@ void testInitMetadataTableRespectsStreamingWriteFlag(boolean metadataStreamingWr HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); HoodieTimeline inflightAndRequestedTimeline = mock(HoodieTimeline.class); when(table.getActiveTimeline()).thenReturn(activeTimeline); + when(table.getMetaClient()).thenReturn(metaClient); when(activeTimeline.filterInflightsAndRequested()).thenReturn(inflightAndRequestedTimeline); when(inflightAndRequestedTimeline.lastInstant()).thenReturn(Option.empty()); FlinkHoodieBackedTableMetadataWriter metadataWriter = mock(FlinkHoodieBackedTableMetadataWriter.class); when(metadataWriter.isInitialized()).thenReturn(true); + when(metadataWriter.hasPartitionsStateChanged()).thenReturn(true); TestableHoodieFlinkTableServiceClient tableServiceClient = new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); @@ -98,6 +117,135 @@ void testInitMetadataTableRespectsStreamingWriteFlag(boolean metadataStreamingWr verify(table, never()).maybeDeleteMetadataTable(); } + @Test + void testInitMetadataTableWrapsMetadataWriterFailure() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withLockConfig(HoodieLockConfig.newBuilder() + .withLockProvider(InProcessLockProvider.class) + .build()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()) + .build(); + HoodieFlinkTable table = mock(HoodieFlinkTable.class); + HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); + HoodieTimeline pendingTimeline = mock(HoodieTimeline.class); + when(table.getActiveTimeline()).thenReturn(activeTimeline); + when(activeTimeline.filterInflightsAndRequested()).thenReturn(pendingTimeline); + when(pendingTimeline.lastInstant()).thenReturn(Option.empty()); + + TestableHoodieFlinkTableServiceClient client = + new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); + try (MockedStatic writerFactory = + Mockito.mockStatic(FlinkHoodieBackedTableMetadataWriter.class)) { + writerFactory.when(() -> FlinkHoodieBackedTableMetadataWriter.create(any(), any(), any(), any())) + .thenThrow(new IllegalStateException("expected metadata writer failure")); + assertThrows(HoodieException.class, client::initMetadataTable); + } finally { + client.close(); + } + } + + @Test + void testMetadataDisabledDeletesStaleMetadataTable() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + HoodieFlinkTable table = mock(HoodieFlinkTable.class); + TestableHoodieFlinkTableServiceClient client = + new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); + try { + client.initMetadataTable(); + } finally { + client.close(); + } + + verify(table).maybeDeleteMetadataTable(); + verify(table, never()).deleteMetadataIndexIfNecessary(); + } + + @Test + void testOutputConversionAndNoOpHooks() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + TestableHoodieFlinkTableServiceClient client = + new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), mock(HoodieTable.class)); + try { + WriteStatus status = new WriteStatus(false, 0.0); + status.setStat(new HoodieWriteStat()); + HoodieWriteMetadata> metadata = new HoodieWriteMetadata<>(); + metadata.setWriteStatuses(Collections.singletonList(status)); + + client.callTriggerWritesAndFetchWriteStats(metadata); + assertSame(metadata, client.callConvertToOutputMetadata(metadata)); + client.callHandleWriteErrors(Collections.singletonList(status.getStat())); + // cluster() is intentionally unimplemented for Flink. + assertNull(client.cluster("001", false)); + assertNotNull(client.createRealTable()); + } finally { + client.close(); + } + } + + @Test + void testCompleteCompactionCommitsAndCleansMarkers() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + HoodieTable table = mock(HoodieTable.class); + when(table.getInstantGenerator()).thenReturn(metaClient.getInstantGenerator()); + HoodieCommitMetadata metadata = new HoodieCommitMetadata(); + TestableHoodieFlinkTableServiceClient client = + new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); + CompactHelpers compactHelpers = mock(CompactHelpers.class); + WriteMarkers writeMarkers = mock(WriteMarkers.class); + + try (MockedStatic helpersFactory = Mockito.mockStatic(CompactHelpers.class); + MockedStatic markersFactory = Mockito.mockStatic(WriteMarkersFactory.class)) { + helpersFactory.when(CompactHelpers::getInstance).thenReturn(compactHelpers); + markersFactory.when(() -> WriteMarkersFactory.get(any(), any(), any())).thenReturn(writeMarkers); + client.callCompleteCompaction(metadata, table, "20260723120000000"); + } finally { + client.close(); + } + + verify(compactHelpers).completeInflightCompaction(table, "20260723120000000", metadata); + verify(writeMarkers).quietDeleteMarkerDir(any(), any(Integer.class)); + } + + @Test + void testCompleteClusteringCommitsAndCleansMarkers() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath()) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + HoodieFlinkTable table = mock(HoodieFlinkTable.class); + HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); + when(table.getActiveTimeline()).thenReturn(activeTimeline); + when(table.getInstantGenerator()).thenReturn(metaClient.getInstantGenerator()); + HoodieInstant clusteringInstant = mock(HoodieInstant.class); + HoodieReplaceCommitMetadata metadata = new HoodieReplaceCommitMetadata(); + TestableHoodieFlinkTableServiceClient client = + new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); + WriteMarkers writeMarkers = mock(WriteMarkers.class); + + try (MockedStatic clusteringUtils = Mockito.mockStatic(ClusteringUtils.class); + MockedStatic markersFactory = Mockito.mockStatic(WriteMarkersFactory.class)) { + clusteringUtils.when(() -> ClusteringUtils.getInflightClusteringInstant( + "20260723120000001", activeTimeline, metaClient.getInstantGenerator())) + .thenReturn(Option.of(clusteringInstant)); + markersFactory.when(() -> WriteMarkersFactory.get(any(), any(), any())).thenReturn(writeMarkers); + client.callCompleteClustering(metadata, table, "20260723120000001"); + } finally { + client.close(); + } + + verify(writeMarkers).quietDeleteMarkerDir(any(), any(Integer.class)); + } + private static class TestableHoodieFlinkTableServiceClient extends HoodieFlinkTableServiceClient { private final HoodieTable mockedTable; @@ -113,5 +261,42 @@ protected TestableHoodieFlinkTableServiceClient(HoodieFlinkEngineContext context protected HoodieTable createTable(HoodieWriteConfig config, StorageConfiguration storageConf, boolean skipValidation) { return mockedTable; } + + private void callTriggerWritesAndFetchWriteStats(HoodieWriteMetadata> metadata) { + triggerWritesAndFetchWriteStats(metadata); + } + + private HoodieWriteMetadata> callConvertToOutputMetadata( + HoodieWriteMetadata> metadata) { + return convertToOutputMetadata(metadata); + } + + private void callHandleWriteErrors(java.util.List writeStats) { + handleWriteErrors(writeStats, TableServiceType.COMPACT); + } + + private HoodieTable createRealTable() { + return super.createTable(config, storageConf, false); + } + + private void callCompleteCompaction( + HoodieCommitMetadata metadata, HoodieTable table, String instantTime) { + completeCompaction(metadata, table, instantTime, Collections.emptyList()); + } + + private void callCompleteClustering( + HoodieReplaceCommitMetadata metadata, HoodieTable table, String instantTime) { + completeClustering(metadata, table, instantTime); + } + + @Override + protected void finalizeWrite(HoodieTable table, String instantTime, java.util.List stats) { + // The test covers Flink orchestration; base finalize-write behavior is covered by client-common tests. + } + + @Override + protected void writeTableMetadata(HoodieTable table, String instantTime, HoodieCommitMetadata metadata) { + // The test covers Flink orchestration; metadata writer behavior is covered independently. + } } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/TestHoodieFlinkTableActionRouting.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/TestHoodieFlinkTableActionRouting.java new file mode 100644 index 0000000000000..4c859468e59aa --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/TestHoodieFlinkTableActionRouting.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.table; + +import org.apache.hudi.avro.model.HoodieRollbackPlan; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.io.HoodieAppendHandle; +import org.apache.hudi.io.HoodieCreateHandle; +import org.apache.hudi.io.HoodieWriteHandle; +import org.apache.hudi.table.action.BaseActionExecutor; +import org.apache.hudi.table.action.HoodieWriteMetadata; +import org.apache.hudi.table.action.clean.CleanPlanActionExecutor; +import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor; +import org.apache.hudi.table.action.commit.BucketInfo; +import org.apache.hudi.table.action.commit.BucketType; +import org.apache.hudi.table.action.commit.FlinkDeletePreppedCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkInsertCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkInsertOverwriteCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkInsertOverwriteTableCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkInsertPreppedCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkPartitionTTLActionExecutor; +import org.apache.hudi.table.action.commit.FlinkUpsertCommitActionExecutor; +import org.apache.hudi.table.action.commit.FlinkUpsertPreppedCommitActionExecutor; +import org.apache.hudi.table.action.commit.delta.FlinkUpsertDeltaCommitActionExecutor; +import org.apache.hudi.table.action.commit.delta.FlinkUpsertPreppedDeltaCommitActionExecutor; +import org.apache.hudi.table.action.compact.RunCompactionActionExecutor; +import org.apache.hudi.table.action.compact.ScheduleCompactionActionExecutor; +import org.apache.hudi.table.action.rollback.BaseRollbackPlanActionExecutor; +import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests Flink table action routing and explicitly unsupported engine APIs. */ +@SuppressWarnings({"rawtypes", "unchecked"}) +class TestHoodieFlinkTableActionRouting extends HoodieFlinkClientTestHarness { + + @BeforeEach + void setUp() { + initPath(); + initFileSystem(); + } + + @AfterEach + void tearDown() throws IOException { + cleanupResources(); + } + + @Test + void testCopyOnWriteUnsupportedActionsFailFast() throws IOException { + initMetaClient(HoodieTableType.COPY_ON_WRITE); + HoodieFlinkCopyOnWriteTable table = new HoodieFlinkCopyOnWriteTable(config(), context, metaClient); + + assertUnsupported(() -> table.upsert(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.insert(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.bulkInsert(context, "001", Collections.emptyList(), Option.empty())); + assertUnsupported(() -> table.delete(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.deletePrepped(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.upsertPrepped(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.insertPrepped(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.bulkInsertPrepped( + context, "001", Collections.emptyList(), Option.empty())); + assertUnsupported(() -> table.insertOverwrite(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.insertOverwriteTable(context, "001", Collections.emptyList())); + assertUnsupported(() -> table.scheduleCompaction(context, "001", Option.empty())); + assertUnsupported(() -> table.compact(context, "001")); + assertUnsupported(() -> table.cluster(context, "001")); + assertUnsupported(() -> table.bootstrap(context, Option.empty())); + assertUnsupported(() -> table.rollbackBootstrap(context, "001")); + assertUnsupported(() -> table.scheduleIndexing(context, "001", Collections.emptyList(), Collections.emptyList())); + assertUnsupported(() -> table.index(context, "001")); + assertUnsupported(() -> table.savepoint(context, "001", "user", "comment")); + assertUnsupported(() -> table.scheduleRestore(context, "002", "001")); + assertUnsupported(() -> table.restore(context, "002", "001")); + } + + @Test + void testCopyOnWriteRoutesSupportedPlanningActions() throws IOException { + initMetaClient(HoodieTableType.COPY_ON_WRITE); + HoodieFlinkCopyOnWriteTable table = new HoodieFlinkCopyOnWriteTable(config(), context, metaClient); + + try (MockedConstruction ignored = Mockito.mockConstruction( + ClusteringPlanActionExecutor.class, + (executor, constructionContext) -> when(executor.execute()).thenReturn(Option.empty()))) { + assertFalse(table.scheduleClustering(context, "001", Option.empty()).isPresent()); + } + try (MockedConstruction ignored = Mockito.mockConstruction( + CleanPlanActionExecutor.class, + (executor, constructionContext) -> when(executor.execute()).thenReturn(Option.empty()))) { + assertFalse(table.createCleanerPlan(context, Option.empty()).isPresent()); + } + try (MockedConstruction ignored = Mockito.mockConstruction( + FlinkPartitionTTLActionExecutor.class, + (executor, constructionContext) -> { + HoodieWriteMetadata> metadata = new HoodieWriteMetadata<>(); + metadata.setWriteStatuses(Collections.emptyList()); + when(executor.execute()).thenReturn(metadata); + })) { + assertEquals(Collections.emptyList(), table.managePartitionTTL(context, "002").getWriteStatuses()); + } + Option rollbackPlan = Option.of(mock(HoodieRollbackPlan.class)); + assertResultPropagated(BaseRollbackPlanActionExecutor.class, rollbackPlan, + () -> table.scheduleRollback( + context, "003", mock(HoodieInstant.class), false, false, false)); + } + + @Test + void testCopyOnWriteRoutesWriteActionsAndCompactionInsert() throws IOException { + initMetaClient(HoodieTableType.COPY_ON_WRITE); + HoodieFlinkCopyOnWriteTable table = new HoodieFlinkCopyOnWriteTable(config(), context, metaClient); + HoodieWriteHandle writeHandle = mock(HoodieWriteHandle.class); + BucketInfo bucketInfo = new BucketInfo(BucketType.INSERT, "file-1", "partition"); + + assertWriteMetadataPropagated(FlinkUpsertCommitActionExecutor.class, + () -> table.upsert(context, writeHandle, bucketInfo, "001", Collections.emptyIterator())); + assertWriteMetadataPropagated(FlinkInsertCommitActionExecutor.class, + () -> table.insert(context, writeHandle, bucketInfo, "001", Collections.emptyIterator())); + assertWriteMetadataPropagated(FlinkDeletePreppedCommitActionExecutor.class, + () -> table.deletePrepped(context, writeHandle, bucketInfo, "001", Collections.emptyList())); + assertWriteMetadataPropagated(FlinkUpsertPreppedCommitActionExecutor.class, + () -> table.upsertPrepped(context, writeHandle, bucketInfo, "001", Collections.emptyList())); + assertWriteMetadataPropagated(FlinkInsertPreppedCommitActionExecutor.class, + () -> table.insertPrepped(context, writeHandle, bucketInfo, "001", Collections.emptyList())); + assertWriteMetadataPropagated(FlinkInsertOverwriteCommitActionExecutor.class, + () -> table.insertOverwrite(context, writeHandle, bucketInfo, "001", Collections.emptyIterator())); + assertWriteMetadataPropagated(FlinkInsertOverwriteTableCommitActionExecutor.class, + () -> table.insertOverwriteTable(context, writeHandle, bucketInfo, "001", Collections.emptyIterator())); + + try (MockedConstruction ignored = Mockito.mockConstruction(HoodieCreateHandle.class)) { + assertEquals(Collections.emptyList(), + table.handleInsert("001", "partition", "file-1", Collections.emptyMap()).next()); + } + } + + @Test + void testMergeOnReadValidatesHandlesAndRoutesScheduling() throws IOException { + initMetaClient(HoodieTableType.MERGE_ON_READ); + HoodieFlinkMergeOnReadTable table = new HoodieFlinkMergeOnReadTable(config(), context, metaClient); + HoodieWriteHandle writeHandle = mock(HoodieWriteHandle.class); + BucketInfo bucketInfo = new BucketInfo(BucketType.UPDATE, "file-1", "partition"); + + assertThrows(IllegalArgumentException.class, + () -> table.upsert(context, writeHandle, bucketInfo, "001", Collections.emptyIterator())); + assertThrows(IllegalArgumentException.class, + () -> table.upsertPrepped(context, writeHandle, bucketInfo, "001", Collections.emptyList())); + + try (MockedConstruction mocked = Mockito.mockConstruction( + ScheduleCompactionActionExecutor.class, + (executor, constructionContext) -> when(executor.execute()).thenReturn(Option.empty()))) { + assertFalse(table.scheduleCompaction(context, "002", Option.empty()).isPresent()); + assertFalse(table.scheduleLogCompaction(context, "003", Option.empty()).isPresent()); + assertEquals(2, mocked.constructed().size()); + } + } + + @Test + void testMergeOnReadRoutesAppendAndCompactionActions() throws IOException { + initMetaClient(HoodieTableType.MERGE_ON_READ); + HoodieFlinkMergeOnReadTable table = new HoodieFlinkMergeOnReadTable(config(), context, metaClient); + HoodieAppendHandle appendHandle = mock(HoodieAppendHandle.class); + BucketInfo bucketInfo = new BucketInfo(BucketType.UPDATE, "file-1", "partition"); + + assertWriteMetadataPropagated(FlinkUpsertDeltaCommitActionExecutor.class, + () -> table.upsert(context, appendHandle, bucketInfo, "001", Collections.emptyIterator())); + assertWriteMetadataPropagated(FlinkUpsertPreppedDeltaCommitActionExecutor.class, + () -> table.upsertPrepped(context, appendHandle, bucketInfo, "001", Collections.emptyList())); + assertWriteMetadataPropagated(FlinkUpsertDeltaCommitActionExecutor.class, + () -> table.insert(context, appendHandle, bucketInfo, "001", Collections.emptyIterator())); + + HoodieWriteMetadata compactionMetadata = new HoodieWriteMetadata(); + compactionMetadata.setWriteStatuses(HoodieListData.eager(Collections.emptyList())); + try (MockedConstruction ignored = Mockito.mockConstruction( + RunCompactionActionExecutor.class, + (executor, constructionContext) -> when(executor.execute()).thenReturn(compactionMetadata))) { + assertEquals(Collections.emptyList(), table.compact(context, "002").getWriteStatuses()); + assertEquals(Collections.emptyList(), table.logCompact(context, "003").getWriteStatuses()); + } + + try (MockedConstruction ignored = Mockito.mockConstruction( + HoodieAppendHandle.class, + (handle, constructionContext) -> when(handle.close()).thenReturn(Collections.emptyList()))) { + assertEquals(Collections.emptyList(), + table.handleInsertsForLogCompaction( + "004", "partition", "file-1", Collections.emptyMap(), Collections.emptyMap()).next()); + } + } + + private void assertWriteMetadataPropagated( + Class executorClass, Supplier invocation) { + HoodieWriteMetadata> metadata = new HoodieWriteMetadata<>(); + metadata.setWriteStatuses(Collections.emptyList()); + assertResultPropagated(executorClass, metadata, invocation); + } + + private void assertResultPropagated( + Class executorClass, Object expected, Supplier invocation) { + try (MockedConstruction mocked = Mockito.mockConstruction( + executorClass, + (executor, constructionContext) -> when(executor.execute()).thenReturn(expected))) { + assertSame(expected, invocation.get()); + assertEquals(1, mocked.constructed().size()); + } + } + + private HoodieWriteConfig config() { + return HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withEngineType(EngineType.FLINK) + .withWriteTableVersion(HoodieTableVersion.NINE.versionCode()) + .build(); + } + + private void assertUnsupported(ThrowingRunnable runnable) { + assertThrows(HoodieNotSupportedException.class, runnable::run); + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } +} diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/commit/TestFlinkDeleteHelper.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/commit/TestFlinkDeleteHelper.java new file mode 100644 index 0000000000000..3b9b6e8bc423d --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/commit/TestFlinkDeleteHelper.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.table.action.commit; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.EmptyHoodieRecordPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.table.action.HoodieWriteMetadata; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for {@link FlinkDeleteHelper}. */ +@SuppressWarnings({"rawtypes", "unchecked"}) +class TestFlinkDeleteHelper { + + @Test + void testDeduplicateKeysForGlobalAndPartitionedIndexes() { + FlinkDeleteHelper helper = FlinkDeleteHelper.newInstance(); + assertSame(helper, FlinkDeleteHelper.newInstance()); + + HoodieTable table = mock(HoodieTable.class); + HoodieIndex index = mock(HoodieIndex.class); + when(table.getIndex()).thenReturn(index); + List keys = new ArrayList<>(Arrays.asList( + new HoodieKey("id1", "p1"), + new HoodieKey("id1", "p2"), + new HoodieKey("id2", "p1"), + new HoodieKey("id2", "p1"))); + + when(index.isGlobal()).thenReturn(true); + List globalResult = helper.deduplicateKeys(keys, table, 1); + assertEquals(Arrays.asList("id1", "id2"), globalResult.stream() + .map(HoodieKey::getRecordKey).collect(Collectors.toList())); + assertEquals(Arrays.asList("p1", "p1"), globalResult.stream() + .map(HoodieKey::getPartitionPath).collect(Collectors.toList())); + assertNotSame(keys, globalResult); + + when(index.isGlobal()).thenReturn(false); + List partitionedResult = helper.deduplicateKeys(keys, table, 1); + assertSame(keys, partitionedResult); + assertEquals(Arrays.asList( + new HoodieKey("id1", "p1"), + new HoodieKey("id1", "p2"), + new HoodieKey("id2", "p1")), partitionedResult); + } + + @Test + void testExecuteTagsExistingRecordsAndDelegatesDelete() { + HoodieTable table = mock(HoodieTable.class); + HoodieIndex index = mock(HoodieIndex.class); + HoodieEngineContext context = mock(HoodieEngineContext.class); + BaseCommitActionExecutor executor = mock(BaseCommitActionExecutor.class); + when(table.getIndex()).thenReturn(index); + when(index.isGlobal()).thenReturn(false); + + when(index.tagLocation(any(HoodieData.class), eq(context), eq(table))).thenAnswer(invocation -> { + HoodieData> records = invocation.getArgument(0); + List> tagged = records.collectAsList(); + tagged.get(0).setCurrentLocation(new HoodieRecordLocation("001", "file-1")); + return HoodieListData.eager(tagged); + }); + + HoodieWriteMetadata> expected = new HoodieWriteMetadata<>(); + expected.setWriteStatuses(Collections.singletonList(new WriteStatus(false, 0.0))); + when(executor.execute(any(List.class))).thenReturn(expected); + + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .withPath("/tmp/flink-delete-helper") + .combineDeleteInput(true) + .build(); + List keys = new ArrayList<>(Arrays.asList( + new HoodieKey("id1", "p1"), new HoodieKey("id1", "p1"), new HoodieKey("missing", "p1"))); + + HoodieWriteMetadata> result = FlinkDeleteHelper.newInstance() + .execute("002", keys, context, config, table, executor); + + assertSame(expected, result); + assertTrue(result.getIndexLookupDuration().isPresent()); + verify(executor).execute(any(List.class)); + verify(executor, never()).saveWorkloadProfileMetadataToInflight(any(), any()); + } + + @Test + void testExecuteWithNoExistingRecordsCreatesEmptyMetadata() { + HoodieTable table = mock(HoodieTable.class); + HoodieIndex index = mock(HoodieIndex.class); + HoodieEngineContext context = mock(HoodieEngineContext.class); + BaseCommitActionExecutor executor = mock(BaseCommitActionExecutor.class); + when(table.getIndex()).thenReturn(index); + when(index.tagLocation(any(HoodieData.class), eq(context), eq(table))) + .thenAnswer(invocation -> invocation.getArgument(0)); + + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .withPath("/tmp/flink-delete-helper") + .combineDeleteInput(false) + .build(); + HoodieWriteMetadata> result = FlinkDeleteHelper.newInstance().execute( + "003", Collections.singletonList(new HoodieKey("missing", "p1")), + context, config, table, executor); + + assertTrue(result.getWriteStatuses().isEmpty()); + verify(executor).saveWorkloadProfileMetadataToInflight(any(), eq("003")); + verify(executor).runPrecommitValidators(result); + verify(executor, never()).execute(any(List.class)); + } + + @Test + void testExecutePreservesOrWrapsFailures() { + HoodieTable table = mock(HoodieTable.class); + HoodieEngineContext context = mock(HoodieEngineContext.class); + BaseCommitActionExecutor executor = mock(BaseCommitActionExecutor.class); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .withPath("/tmp/flink-delete-helper") + .build(); + List keys = Collections.singletonList(new HoodieKey("id1", "p1")); + + HoodieUpsertException upsertException = new HoodieUpsertException("expected"); + when(table.getIndex()).thenThrow(upsertException); + HoodieUpsertException firstFailure = assertThrows(HoodieUpsertException.class, + () -> FlinkDeleteHelper.newInstance().execute("004", keys, context, config, table, executor)); + assertSame(upsertException, firstFailure); + + reset(table); + when(table.getIndex()).thenThrow(new IllegalStateException("boom")); + HoodieUpsertException wrapped = assertThrows(HoodieUpsertException.class, + () -> FlinkDeleteHelper.newInstance().execute("005", keys, context, config, table, executor)); + assertTrue(wrapped.getCause() instanceof IllegalStateException); + } +} diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java index bc077c12c00ce..45c2637259dda 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java @@ -38,7 +38,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; +import java.util.LinkedHashSet; import java.util.List; import java.util.stream.Collectors; @@ -64,16 +64,12 @@ public List deduplicateKeys(List keys, int parallelism) { boolean isIndexingGlobal = table.getIndex().isGlobal(); if (isIndexingGlobal) { - HashSet recordKeys = keys.stream().map(HoodieKey::getRecordKey).collect(Collectors.toCollection(HashSet::new)); - List deduplicatedKeys = new LinkedList<>(); - keys.forEach(x -> { - if (recordKeys.contains(x.getRecordKey())) { - deduplicatedKeys.add(x); - } - }); - return deduplicatedKeys; + HashSet recordKeys = new HashSet<>(); + return keys.stream() + .filter(key -> recordKeys.add(key.getRecordKey())) + .collect(Collectors.toList()); } else { - HashSet set = new HashSet<>(keys); + LinkedHashSet set = new LinkedHashSet<>(keys); keys.clear(); keys.addAll(set); return keys; diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaDeleteHelper.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaDeleteHelper.java new file mode 100644 index 0000000000000..fab7d0423163f --- /dev/null +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaDeleteHelper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.commit; + +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests for {@link JavaDeleteHelper}. */ +@SuppressWarnings({"rawtypes", "unchecked"}) +class TestJavaDeleteHelper { + + @Test + void testDeduplicateKeysForGlobalAndPartitionedIndexes() { + JavaDeleteHelper helper = JavaDeleteHelper.newInstance(); + assertSame(helper, JavaDeleteHelper.newInstance()); + + HoodieTable table = mock(HoodieTable.class); + HoodieIndex index = mock(HoodieIndex.class); + when(table.getIndex()).thenReturn(index); + List keys = new ArrayList<>(Arrays.asList( + new HoodieKey("id1", "p1"), + new HoodieKey("id1", "p2"), + new HoodieKey("id2", "p1"), + new HoodieKey("id2", "p1"))); + + when(index.isGlobal()).thenReturn(true); + List globalResult = helper.deduplicateKeys(keys, table, 1); + assertEquals(Arrays.asList("id1", "id2"), globalResult.stream() + .map(HoodieKey::getRecordKey).collect(Collectors.toList())); + assertEquals(Arrays.asList("p1", "p1"), globalResult.stream() + .map(HoodieKey::getPartitionPath).collect(Collectors.toList())); + assertNotSame(keys, globalResult); + + when(index.isGlobal()).thenReturn(false); + List partitionedResult = helper.deduplicateKeys(keys, table, 1); + assertSame(keys, partitionedResult); + assertEquals(Arrays.asList( + new HoodieKey("id1", "p1"), + new HoodieKey("id1", "p2"), + new HoodieKey("id2", "p1")), partitionedResult); + } +} From 349bbe038498cba274c2fca24e11d1480f4ec705 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 23 Jul 2026 21:27:15 -0700 Subject: [PATCH 197/255] fix(timeline-service): fail marker creation requests when marker flush fails (#19368) MarkerDirState.flushMarkersToFile closed the marker file stream with closeQuietly, so a close()-time flush failure (e.g. S3A, which performs the object PUT in close()) was silently swallowed. The pending marker creation request was then acknowledged as successful, letting the writer create a data file with no durable marker. Marker-based rollback then missed such files, and a retry under the same instant could leave duplicate file groups behind. - Close the writer within try-with-resources so a close() failure propagates as HoodieIOException instead of being swallowed. - On a flush failure, roll back the markers buffered by the failed batch from in-memory state and fail the pending futures exceptionally, so a retried request can recreate the markers. - Release the marker file index in a finally block so a failure doesn't leak it. Adds TestMarkerDirState covering the success, flush-failure, and retry-after-failure paths. (cherry picked from commit ebbfcc500dc138fb32d33e2778d6ba3ed0b88c37) --- .../handlers/marker/MarkerDirState.java | 122 ++++++++----- .../handlers/marker/TestMarkerDirState.java | 170 ++++++++++++++++++ 2 files changed, 248 insertions(+), 44 deletions(-) create mode 100644 hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerDirState.java diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java index 9f804086f4811..715a8171f6c47 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java @@ -40,7 +40,6 @@ import java.io.BufferedWriter; import java.io.IOException; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Serializable; import java.nio.charset.StandardCharsets; @@ -54,7 +53,6 @@ import java.util.stream.Stream; import static org.apache.hudi.common.util.MarkerUtils.MARKERS_FILENAME_PREFIX; -import static org.apache.hudi.io.util.FileIOUtils.closeQuietly; import static org.apache.hudi.timeline.service.RequestHandler.jsonifyResult; /** @@ -207,46 +205,62 @@ public void processMarkerCreationRequests( log.debug("timeMs={} markerDirPath={} numRequests={} fileIndex={}", System.currentTimeMillis(), markerDirPath, pendingMarkerCreationFutures.size(), fileIndex); boolean shouldFlushMarkers = false; - - synchronized (markerCreationProcessingLock) { - for (MarkerCreationFuture future : pendingMarkerCreationFutures) { - String markerName = future.getMarkerName(); - boolean exists = allMarkers.contains(markerName); - if (!exists) { - if (conflictDetectionStrategy.isPresent()) { - try { - conflictDetectionStrategy.get().detectAndResolveConflictIfNecessary(); - } catch (HoodieEarlyConflictDetectionException he) { - log.error("Detected the write conflict due to a concurrent writer, " - + "failing the marker creation as the early conflict detection is enabled", he); - future.setIsSuccessful(false); - continue; - } catch (Exception e) { - log.warn("Failed to execute early conflict detection. Marker creation will continue.", e); - // When early conflict detection fails to execute, we still allow the marker creation - // to continue - addMarkerToMap(fileIndex, markerName); - future.setIsSuccessful(true); - shouldFlushMarkers = true; - continue; + int fileMarkersLengthBeforeBatch = 0; + + try { + synchronized (markerCreationProcessingLock) { + StringBuilder fileMarkers = fileMarkersMap.get(fileIndex); + fileMarkersLengthBeforeBatch = fileMarkers == null ? 0 : fileMarkers.length(); + for (MarkerCreationFuture future : pendingMarkerCreationFutures) { + String markerName = future.getMarkerName(); + boolean exists = allMarkers.contains(markerName); + if (!exists) { + if (conflictDetectionStrategy.isPresent()) { + try { + conflictDetectionStrategy.get().detectAndResolveConflictIfNecessary(); + } catch (HoodieEarlyConflictDetectionException he) { + log.error("Detected the write conflict due to a concurrent writer, " + + "failing the marker creation as the early conflict detection is enabled", he); + future.setIsSuccessful(false); + continue; + } catch (Exception e) { + log.warn("Failed to execute early conflict detection. Marker creation will continue.", e); + // When early conflict detection fails to execute, we still allow the marker creation + // to continue + addMarkerToMap(fileIndex, markerName); + future.setIsSuccessful(true); + shouldFlushMarkers = true; + continue; + } } + addMarkerToMap(fileIndex, markerName); + shouldFlushMarkers = true; } - addMarkerToMap(fileIndex, markerName); - shouldFlushMarkers = true; + future.setIsSuccessful(!exists); } - future.setIsSuccessful(!exists); - } - if (!isMarkerTypeWritten) { - // Create marker directory and write marker type to MARKERS.type - writeMarkerTypeToFile(); - isMarkerTypeWritten = true; + if (!isMarkerTypeWritten) { + // Create marker directory and write marker type to MARKERS.type + writeMarkerTypeToFile(); + isMarkerTypeWritten = true; + } } + if (shouldFlushMarkers) { + flushMarkersToFile(fileIndex); + } + } catch (Exception e) { + log.error("Failed to persist markers to file index {} in {}", fileIndex, markerDirPath, e); + // The markers added by this batch are not durably persisted, so they are removed from + // the in-memory state and all pending requests fail, so that no write operation + // proceeds without a durable marker and a retried request can recreate the marker + removeMarkersOfPendingFutures(pendingMarkerCreationFutures, fileIndex, fileMarkersLengthBeforeBatch); + for (MarkerCreationFuture future : pendingMarkerCreationFutures) { + future.completeExceptionally(e); + } + return; + } finally { + markFileAsAvailable(fileIndex); } - if (shouldFlushMarkers) { - flushMarkersToFile(fileIndex); - } - markFileAsAvailable(fileIndex); for (MarkerCreationFuture future : pendingMarkerCreationFutures) { try { @@ -309,6 +323,29 @@ private void addMarkerToMap(int fileIndex, String markerName) { stringBuilder.append('\n'); } + /** + * Removes the markers added by the pending marker creation requests from the in-memory state, + * used when the markers cannot be persisted, so that a retried request can recreate the markers. + * + * @param pendingMarkerCreationFutures futures of pending marker creation requests + * @param fileIndex file index used by the batch of requests + * @param fileMarkersLengthBeforeBatch length of the buffered markers of the file index before the batch + */ + private void removeMarkersOfPendingFutures( + List pendingMarkerCreationFutures, int fileIndex, int fileMarkersLengthBeforeBatch) { + synchronized (markerCreationProcessingLock) { + for (MarkerCreationFuture future : pendingMarkerCreationFutures) { + if (future.isSuccessful()) { + allMarkers.remove(future.getMarkerName()); + } + } + StringBuilder fileMarkers = fileMarkersMap.get(fileIndex); + if (fileMarkers != null) { + fileMarkers.setLength(fileMarkersLengthBeforeBatch); + } + } + } + /** * Writes marker type, "TIMELINE_SERVER_BASED", to file. */ @@ -357,17 +394,14 @@ private void flushMarkersToFile(int markerFileIndex) { HoodieTimer timer = HoodieTimer.start(); StoragePath markersFilePath = new StoragePath( markerDirPath, MARKERS_FILENAME_PREFIX + markerFileIndex); - OutputStream outputStream = null; - BufferedWriter bufferedWriter = null; - try { - outputStream = storage.create(markersFilePath); - bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)); + // The writer must be closed within the try scope, so that a failure to persist the markers + // at close() time, e.g., when an object store uploads the file content in close(), is + // propagated to the caller instead of being swallowed + try (BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter( + storage.create(markersFilePath), StandardCharsets.UTF_8))) { bufferedWriter.write(fileMarkersMap.get(markerFileIndex).toString()); } catch (IOException e) { throw new HoodieIOException("Failed to overwrite marker file " + markersFilePath, e); - } finally { - closeQuietly(bufferedWriter); - closeQuietly(outputStream); } log.debug("{} written in {} ms", markersFilePath, timer.endTimer()); } diff --git a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerDirState.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerDirState.java new file mode 100644 index 0000000000000..76597609c2901 --- /dev/null +++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerDirState.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers.marker; + +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.io.util.FileIOUtils; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.hadoop.HoodieHadoopStorage; + +import io.javalin.http.Context; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.concurrent.ExecutionException; + +import static org.apache.hudi.common.util.MarkerUtils.MARKERS_FILENAME_PREFIX; +import static org.apache.hudi.common.util.MarkerUtils.MARKER_TYPE_FILENAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Tests {@link MarkerDirState}. + */ +public class TestMarkerDirState extends HoodieCommonTestHarness { + private static final String MARKER_NAME = + "2016/68b3ad4d-9d43-4e4c-8f4b-4c9b6d3e8b3a-0_1-0-1_00000000000001.parquet.marker.CREATE"; + + private CloseFailingHoodieStorage storage; + private String markerDir; + + @BeforeEach + void setUp() throws IOException { + initPath(); + storage = new CloseFailingHoodieStorage(basePath, new Configuration()); + markerDir = basePath + "/.hoodie/.temp/00000000000001"; + } + + @Test + void testMarkerCreationRequestProcessing() throws Exception { + MarkerDirState dirState = createMarkerDirState(); + MarkerCreationFuture future = createFuture(); + int fileIndex = dirState.getNextFileIndexToUse().get(); + + dirState.processMarkerCreationRequests(Collections.singletonList(future), fileIndex); + + assertTrue(future.isSuccessful()); + assertEquals("true", future.get()); + assertEquals(Collections.singleton(MARKER_NAME), dirState.getAllMarkers()); + assertEquals(MARKER_NAME + "\n", readMarkersFileContent(fileIndex)); + // The file index must be released after the batch is processed + assertEquals(Option.of(fileIndex), dirState.getNextFileIndexToUse()); + } + + @Test + void testMarkerCreationRequestsFailWhenFlushFails() { + MarkerDirState dirState = createMarkerDirState(); + storage.setShouldFailClose(true); + MarkerCreationFuture future = createFuture(); + int fileIndex = dirState.getNextFileIndexToUse().get(); + + dirState.processMarkerCreationRequests(Collections.singletonList(future), fileIndex); + + assertTrue(future.isCompletedExceptionally()); + ExecutionException exception = assertThrows(ExecutionException.class, future::get); + assertInstanceOf(HoodieIOException.class, exception.getCause()); + // The marker without durable persistence must not be acknowledged as existing + assertTrue(dirState.getAllMarkers().isEmpty()); + // The file index must be released even if the flush fails + assertEquals(Option.of(fileIndex), dirState.getNextFileIndexToUse()); + } + + @Test + void testMarkerRecreationAfterFlushFailure() throws Exception { + MarkerDirState dirState = createMarkerDirState(); + storage.setShouldFailClose(true); + MarkerCreationFuture failedFuture = createFuture(); + dirState.processMarkerCreationRequests(Collections.singletonList(failedFuture), 0); + assertTrue(failedFuture.isCompletedExceptionally()); + + // A retried request for the same marker must succeed once the storage recovers + storage.setShouldFailClose(false); + MarkerCreationFuture retriedFuture = createFuture(); + dirState.processMarkerCreationRequests(Collections.singletonList(retriedFuture), 0); + + assertTrue(retriedFuture.isSuccessful()); + assertEquals("true", retriedFuture.get()); + assertEquals(Collections.singleton(MARKER_NAME), dirState.getAllMarkers()); + assertEquals(MARKER_NAME + "\n", readMarkersFileContent(0)); + } + + private MarkerDirState createMarkerDirState() { + return new MarkerDirState( + markerDir, 1, Option.empty(), storage, Registry.getRegistry("TestMarkerDirState"), 1); + } + + private MarkerCreationFuture createFuture() { + return new MarkerCreationFuture(mock(Context.class), markerDir, MARKER_NAME); + } + + private String readMarkersFileContent(int fileIndex) throws IOException { + return FileIOUtils.readAsUTFString( + storage.open(new StoragePath(markerDir, MARKERS_FILENAME_PREFIX + fileIndex))); + } + + /** + * Storage whose created streams for {@code MARKERS} index files fail at close() time, + * simulating an object store that uploads the file content in close(). + */ + private static class CloseFailingHoodieStorage extends HoodieHadoopStorage { + private boolean shouldFailClose = false; + + CloseFailingHoodieStorage(String path, Configuration conf) { + super(path, conf); + } + + void setShouldFailClose(boolean shouldFailClose) { + this.shouldFailClose = shouldFailClose; + } + + @Override + public OutputStream create(StoragePath path) throws IOException { + OutputStream stream = super.create(path); + String fileName = path.getName(); + if (shouldFailClose + && fileName.startsWith(MARKERS_FILENAME_PREFIX) && !fileName.equals(MARKER_TYPE_FILENAME)) { + return new CloseFailingStream(stream); + } + return stream; + } + } + + private static class CloseFailingStream extends FilterOutputStream { + CloseFailingStream(OutputStream delegate) { + super(delegate); + } + + @Override + public void close() throws IOException { + super.close(); + throw new IOException("Simulated failure to persist the file at close() time"); + } + } +} From a96c592a49c26daf5c5bd9a7fdef75ba600c8d24 Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Fri, 24 Jul 2026 13:55:54 +0800 Subject: [PATCH 198/255] test(hudi-client): improve metadata table writer coverage (#19363) Adapted for release-1.2.1: - TestHoodieBackedTableMetadataWriter: dropped two of the eight new tests, exercisesNoOpAndUnsupportedBaseWriterPaths and fallsBackToConfiguredIndexersWhenTableConfigHasNoMetadataPartitions. Both drive the Indexer abstraction (org.apache.hudi.metadata.index.Indexer and the enabledIndexerMap field), introduced by a001ea87da7b (#18348, "feat(index): Add Indexer abstraction and refactor metadata table init"), which is not backported. The second does not compile here; the first sets enabledIndexerMap reflectively and would fail at runtime. The other six tests and the setField helper apply unchanged, with Lazy imported from this branch's org.apache.hudi.util rather than master's common.util. - TestHoodieMetadataWriteUtils: applied only this commit's two metrics-reporter tests. The conflict region also carried master's pre-existing testCreateEmptyNativeLogFile, which belongs to the native log format work (#19067) and is not part of #19363; taking the region wholesale would have imported it. HoodieMetricsConfig is imported from this branch's org.apache.hudi.config.metrics rather than master's common.config.metrics. - TestSparkHoodieBackedTableMetadataWriter: placed under org/apache/hudi/metadata/ to match its package declaration and the package of the class under test. Git suggested org/apache/hudi/ because that test directory is empty on this branch after 4d143a884fb0, which made it infer a directory rename. - TestMetadataTableWithSparkSQL: InProcessLockProvider imported from this branch's org.apache.hudi.client.transaction.lock rather than master's core.transaction.lock, and ENABLE_METADATA_INDEX_PARTITION_STATS used as the deprecated String constant it is here rather than a ConfigProperty. (cherry picked from commit a4373f75bad4e8175ab00b121706417b1cfaed7c) --- .../TestHoodieBackedTableMetadataWriter.java | 191 +++++++++++ ...kedTableMetadataWriterTableVersionSix.java | 153 +++++++++ .../TestHoodieMetadataWriteUtils.java | 46 +++ ...stSecondaryIndexRecordGenerationUtils.java | 73 ++++ ...tSparkHoodieBackedTableMetadataWriter.java | 183 ++++++++++ .../TestMetadataTableWithSparkSQL.scala | 313 ++++++++++++++++++ 6 files changed, 959 insertions(+) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestSecondaryIndexRecordGenerationUtils.java create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestSparkHoodieBackedTableMetadataWriter.java create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkSQL.scala diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java index 449578f3968d3..7a6cb14e0f872 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java @@ -18,6 +18,7 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.model.HoodieRestoreMetadata; import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieTableServiceManagerConfig; @@ -28,25 +29,34 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.v2.ActiveTimelineV2; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.util.Lazy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.MockedStatic; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -58,6 +68,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -578,4 +589,184 @@ void testPerformTableServicesWithFailureHandling( // Verify metrics are incremented when there's a failure verify(metrics, times(1)).incrementMetric(HoodieMetadataMetrics.PENDING_COMPACTIONS_FAILURES, 1); } + + @Test + void wrapsMetadataReaderAndFileSliceReadFailures() throws Exception { + // Reader setup and lazy file listing must preserve the public exception contract. + HoodieBackedTableMetadataWriter, List> writer = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + writer.dataWriteConfig = HoodieWriteConfig.newBuilder().withPath("/tmp/missing-table").build(); + writer.dataMetaClient = mock(HoodieTableMetaClient.class); + Method maybeReinitializeReader = + HoodieBackedTableMetadataWriter.class.getDeclaredMethod("mayBeReinitMetadataReader"); + maybeReinitializeReader.setAccessible(true); + InvocationTargetException readerFailure = assertThrows( + InvocationTargetException.class, () -> maybeReinitializeReader.invoke(writer)); + assertTrue(readerFailure.getCause() instanceof HoodieException); + + HoodieBackedTableMetadata metadata = mock(HoodieBackedTableMetadata.class); + HoodieTableFileSystemView metadataView = mock(HoodieTableFileSystemView.class); + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS); + when(dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()) + .thenReturn(Option.empty()); + when(metadata.getMetadataFileSystemView()).thenReturn(metadataView); + when(metadata.getAllPartitionPaths()).thenThrow(new IOException("listing failed")); + writer.metadata = metadata; + writer.dataMetaClient = dataMetaClient; + setField(writer, "metadataView", metadataView); + Method getLazyMergedFileSlices = + HoodieBackedTableMetadataWriter.class.getDeclaredMethod("getLazyMergedFileSlices"); + getLazyMergedFileSlices.setAccessible(true); + Lazy lazyFileSlices = (Lazy) getLazyMergedFileSlices.invoke(writer); + assertThrows(HoodieIOException.class, lazyFileSlices::get); + } + + @Test + void detectsEmptyMetadataTimelineAndHandlesMissingMetadataTable(@TempDir Path tempDir) throws Exception { + // Missing MDT state requires bootstrap without trusting stale table config. + HoodieBackedTableMetadataWriter, List> writer = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + Method isBootstrapNeeded = HoodieBackedTableMetadataWriter.class + .getDeclaredMethod("isBootstrapNeeded", Option.class); + isBootstrapNeeded.setAccessible(true); + assertTrue((boolean) isBootstrapNeeded.invoke(writer, Option.empty())); + + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(dataMetaClient.getTableConfig()).thenReturn(tableConfig); + when(tableConfig.isMetadataTableAvailable()).thenReturn(true); + writer.storageConf = org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf(); + writer.dataWriteConfig = HoodieWriteConfig.newBuilder() + .withPath(tempDir.resolve("data-table").toString()) + .build(); + writer.metadataWriteConfig = HoodieWriteConfig.newBuilder() + .withPath(tempDir.resolve("missing-metadata-table").toString()) + .build(); + Method metadataTableExists = HoodieBackedTableMetadataWriter.class + .getDeclaredMethod("metadataTableExists", HoodieTableMetaClient.class); + metadataTableExists.setAccessible(true); + assertFalse((boolean) metadataTableExists.invoke(writer, dataMetaClient)); + } + + @Test + void ignoresIOExceptionWhileRemovingPendingIndexInstant() throws Exception { + // A corrupt pending index plan must not block partition cleanup. + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class); + HoodieInstant pendingIndex = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.INDEXING_ACTION, "001"); + when(metaClient.getInstantGenerator()).thenReturn(INSTANT_GENERATOR); + when(metaClient.reloadActiveTimeline()).thenReturn(timeline); + when(metaClient.getActiveTimeline()).thenReturn(timeline); + when(timeline.filterPendingIndexTimeline()).thenReturn(timeline); + when(timeline.getInstantsAsStream()).thenReturn(Stream.of(pendingIndex)); + when(timeline.readIndexPlan(pendingIndex)).thenThrow(new IOException("cannot read plan")); + Method deletePendingIndexingInstant = HoodieBackedTableMetadataWriter.class + .getDeclaredMethod("deletePendingIndexingInstant", HoodieTableMetaClient.class, String.class); + deletePendingIndexingInstant.setAccessible(true); + + assertDoesNotThrow(() -> deletePendingIndexingInstant.invoke(null, metaClient, "column_stats")); + } + + @Test + void wrapsRestorePlanReadFailure() throws Exception { + // Restore-plan I/O failures must surface as HoodieIOException. + HoodieBackedTableMetadataWriter, List> writer = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + HoodieBackedTableMetadata metadata = mock(HoodieBackedTableMetadata.class); + HoodieTableFileSystemView metadataView = mock(HoodieTableFileSystemView.class); + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class); + when(metadata.getMetadataFileSystemView()).thenReturn(metadataView); + when(dataMetaClient.getInstantGenerator()).thenReturn(INSTANT_GENERATOR); + when(dataMetaClient.getActiveTimeline()).thenReturn(timeline); + when(timeline.readRestorePlan(any())).thenThrow(new IOException("cannot read restore plan")); + writer.metadata = metadata; + writer.metadataMetaClient = metadataMetaClient; + writer.dataMetaClient = dataMetaClient; + + assertThrows(HoodieIOException.class, + () -> writer.update(mock(HoodieRestoreMetadata.class), "001")); + } + + @Test + void rejectsPendingMetadataCompactionAndWrapsCloseFailures() { + // Pending compaction blocks scheduling, while close errors remain visible. + HoodieBackedTableMetadataWriter, List> writer = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + HoodieWriteConfig metadataWriteConfig = mock(HoodieWriteConfig.class); + when(metadataWriteConfig.isLogCompactionEnabled()).thenReturn(true); + writer.metadataWriteConfig = metadataWriteConfig; + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline metadataTimeline = mock(HoodieActiveTimeline.class, RETURNS_DEEP_STUBS); + HoodieInstant pendingCompaction = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "001"); + when(metadataMetaClient.getActiveTimeline()).thenReturn(metadataTimeline); + when(metadataTimeline.filterPendingLogCompactionTimeline().firstInstant()).thenReturn(Option.empty()); + when(metadataTimeline.filterPendingCompactionTimeline().firstInstant()).thenReturn(Option.of(pendingCompaction)); + writer.metadataMetaClient = metadataMetaClient; + + assertThrows(HoodieException.class, () -> { + doThrow(new HoodieException("close failed")).when(writer).close(); + writer.closeInternal(); + }); + assertFalse(writer.validateCompactionScheduling(Option.empty(), "002")); + } + + @Test + void compactIfNecessaryHandlesSkipDelegationAndFailures() { + // Exercise skip, delegation, and failure propagation for both compaction types. + Properties tableServiceManagerProperties = new Properties(); + tableServiceManagerProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ENABLED.key(), "true"); + tableServiceManagerProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ACTIONS.key(), "compaction,logcompaction"); + HoodieTableServiceManagerConfig tableServiceManagerConfig = + HoodieTableServiceManagerConfig.newBuilder().fromProperties(tableServiceManagerProperties).build(); + HoodieWriteConfig metadataWriteConfig = mock(HoodieWriteConfig.class); + when(metadataWriteConfig.getTableServiceManagerConfig()).thenReturn(tableServiceManagerConfig); + when(metadataWriteConfig.isLogCompactionEnabled()).thenReturn(true); + + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS); + when(dataMetaClient.reloadActiveTimeline().filterInflightsAndRequested() + .filter(any()).firstInstant()).thenReturn(Option.empty()); + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline metadataTimeline = mock(HoodieActiveTimeline.class); + HoodieTimeline completedTimeline = mock(HoodieTimeline.class); + when(metadataMetaClient.getActiveTimeline()).thenReturn(metadataTimeline); + when(metadataTimeline.filterCompletedInstants()).thenReturn(completedTimeline); + when(completedTimeline.containsInstant(any(String.class))) + .thenAnswer(invocation -> "100".equals(invocation.getArgument(0))); + + HoodieBackedTableMetadataWriter, List> writer = + mock(HoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + writer.dataMetaClient = dataMetaClient; + writer.metadataMetaClient = metadataMetaClient; + writer.metadataWriteConfig = metadataWriteConfig; + writer.metrics = Option.empty(); + + BaseHoodieWriteClient writeClient = mock(BaseHoodieWriteClient.class); + when(writeClient.createNewInstantTime(false)).thenReturn("100", "200", "300", "400"); + when(writeClient.scheduleCompactionAtInstant("200", Option.empty())).thenReturn(true); + when(writeClient.scheduleCompactionAtInstant("300", Option.empty())) + .thenThrow(new HoodieException("compaction failed")); + when(writeClient.scheduleCompactionAtInstant("400", Option.empty())).thenReturn(false); + when(writeClient.scheduleLogCompaction(Option.empty())) + .thenReturn(Option.of("201")) + .thenThrow(new HoodieException("log compaction failed")); + + writer.compactIfNecessary(writeClient, Option.empty()); + writer.compactIfNecessary(writeClient, Option.empty()); + assertThrows(HoodieException.class, () -> writer.compactIfNecessary(writeClient, Option.empty())); + assertThrows(HoodieException.class, () -> writer.compactIfNecessary(writeClient, Option.empty())); + } + + private static void setField(Object target, String name, Object value) throws Exception { + // Exercise private failure paths without changing production visibility. + java.lang.reflect.Field field = HoodieBackedTableMetadataWriter.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriterTableVersionSix.java index 804acc5f2ef3e..184003552ca39 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriterTableVersionSix.java @@ -18,6 +18,10 @@ package org.apache.hudi.metadata; +import org.apache.hudi.client.BaseHoodieWriteClient; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.HoodieTableServiceManagerConfig; +import org.apache.hudi.common.model.ActionType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -26,19 +30,28 @@ import org.apache.hudi.common.table.timeline.versioning.v1.ActiveTimelineV1; import org.apache.hudi.common.table.timeline.versioning.v1.InstantGeneratorV1; import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieMetadataException; import org.junit.jupiter.api.Test; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -225,6 +238,146 @@ void testShouldInitializeFromFilesystem_completedRollbacksDoNotCount() throws Ex assertFalse(result, "Should block initialization when rollbacks are completed, not pending"); } + @Test + void testGenerateUniqueInstantTimePreservesIndexingInstant() throws Exception { + // An indexing instant is already globally unique and must be reused. + String indexingInstant = "20250101120000000"; + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline timeline = createMockTimeline(Collections.singletonList( + INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.INDEXING_ACTION, indexingInstant))); + when(dataMetaClient.getActiveTimeline()).thenReturn(timeline); + HoodieBackedTableMetadataWriterTableVersionSix writer = createMockWriter(dataMetaClient); + + assertTrue(indexingInstant.equals(writer.generateUniqueInstantTime(indexingInstant))); + } + + @Test + void testValidateCompactionSchedulingRejectsPendingMetadataTableService() throws Exception { + // Do not schedule compaction while another metadata table service is pending. + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline dataTimeline = createMockTimeline(Collections.emptyList()); + when(dataMetaClient.reloadActiveTimeline()).thenReturn(dataTimeline); + HoodieBackedTableMetadataWriterTableVersionSix writer = createMockWriter(dataMetaClient); + + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline metadataTimeline = mock(HoodieActiveTimeline.class, RETURNS_DEEP_STUBS); + HoodieInstant pendingCompaction = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "20250101120000001"); + when(metadataMetaClient.getActiveTimeline()).thenReturn(metadataTimeline); + when(metadataTimeline.filterPendingLogCompactionTimeline().firstInstant()).thenReturn(Option.empty()); + when(metadataTimeline.filterPendingCompactionTimeline().firstInstant()).thenReturn(Option.of(pendingCompaction)); + writer.metadataMetaClient = metadataMetaClient; + + assertFalse(writer.validateCompactionScheduling(Option.empty(), "20250101120000002")); + } + + @Test + void testValidateCompactionSchedulingRejectsExcessiveDeltaCommits() throws Exception { + // Protect pending data commits from unbounded metadata delta commits. + HoodieTableMetaClient dataMetaClient = mock(HoodieTableMetaClient.class); + HoodieInstant pendingCommit = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "20250101120000000"); + HoodieActiveTimeline dataTimeline = createMockTimeline(Collections.singletonList(pendingCommit)); + when(dataMetaClient.reloadActiveTimeline()).thenReturn(dataTimeline); + HoodieBackedTableMetadataWriterTableVersionSix writer = createMockWriter(dataMetaClient); + + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline metadataTimeline = mock(HoodieActiveTimeline.class, RETURNS_DEEP_STUBS); + when(metadataMetaClient.reloadActiveTimeline()).thenReturn(metadataTimeline); + when(metadataTimeline.filterCompletedInstants().filter(any()).lastInstant()).thenReturn(Option.empty()); + when(metadataTimeline.getDeltaCommitTimeline().countInstants()).thenReturn(2); + writer.metadataMetaClient = metadataMetaClient; + writer.dataWriteConfig = HoodieWriteConfig.newBuilder() + .withPath("/tmp/table") + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .withMaxNumDeltacommitsWhenPending(1) + .build()) + .build(); + + assertThrows(HoodieMetadataException.class, + () -> writer.validateCompactionScheduling(Option.empty(), "20250101120000001")); + } + + @Test + void testCompactIfNecessaryCoversExistingDelegatedAndLogCompactionPaths() { + // Exercise completed, delegated, and log-compaction fallback paths. + Properties tableServiceManagerProperties = new Properties(); + tableServiceManagerProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ENABLED.key(), "true"); + tableServiceManagerProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ACTIONS.key(), ActionType.compaction.name()); + HoodieTableServiceManagerConfig tableServiceManagerConfig = + HoodieTableServiceManagerConfig.newBuilder() + .fromProperties(tableServiceManagerProperties) + .build(); + HoodieWriteConfig metadataWriteConfig = mock(HoodieWriteConfig.class); + when(metadataWriteConfig.getTableServiceManagerConfig()).thenReturn(tableServiceManagerConfig); + when(metadataWriteConfig.isLogCompactionEnabled()).thenReturn(true); + + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class, RETURNS_DEEP_STUBS); + when(metadataMetaClient.getActiveTimeline()).thenReturn(timeline); + when(timeline.filterCompletedInstants().containsInstant("100001")).thenReturn(true); + when(timeline.filterCompletedInstants().containsInstant("200001")).thenReturn(false); + when(timeline.filterCompletedInstants().containsInstant("300001")).thenReturn(false); + when(timeline.filterCompletedInstants().containsInstant("300005")).thenReturn(false); + when(timeline.filterCompletedInstants().containsInstant("400001")).thenReturn(false); + when(timeline.filterCompletedInstants().containsInstant("400005")).thenReturn(false); + + HoodieBackedTableMetadataWriterTableVersionSix writer = + mock(HoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + writer.metadataMetaClient = metadataMetaClient; + writer.metadataWriteConfig = metadataWriteConfig; + BaseHoodieWriteClient writeClient = mock(BaseHoodieWriteClient.class); + when(writeClient.scheduleCompactionAtInstant("200001", Option.empty())).thenReturn(true); + when(writeClient.scheduleCompactionAtInstant("300001", Option.empty())).thenReturn(false); + when(writeClient.scheduleLogCompactionAtInstant("300005", Option.empty())).thenReturn(true); + + // Version 6 derives compaction and log-compaction instants with fixed suffixes. + writer.compactIfNecessary(writeClient, Option.of("100")); + writer.compactIfNecessary(writeClient, Option.of("200")); + writer.compactIfNecessary(writeClient, Option.of("300")); + + Properties allTableServicesProperties = new Properties(); + allTableServicesProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ENABLED.key(), "true"); + allTableServicesProperties.put( + HoodieTableServiceManagerConfig.TABLE_SERVICE_MANAGER_ACTIONS.key(), "compaction,logcompaction"); + when(metadataWriteConfig.getTableServiceManagerConfig()).thenReturn( + HoodieTableServiceManagerConfig.newBuilder() + .fromProperties(allTableServicesProperties) + .build()); + when(writeClient.scheduleCompactionAtInstant("400001", Option.empty())).thenReturn(false); + when(writeClient.scheduleLogCompactionAtInstant("400005", Option.empty())).thenReturn(true); + writer.compactIfNecessary(writeClient, Option.of("400")); + + verify(writeClient).scheduleCompactionAtInstant("200001", Option.empty()); + verify(writeClient).scheduleLogCompactionAtInstant("300005", Option.empty()); + verify(writeClient).logCompact("300005", true); + } + + @Test + void testValidateRollbackRejectsCommitBeforeLatestCompaction() throws Exception { + // Version 6 cannot roll back beyond the latest compaction boundary. + HoodieBackedTableMetadataWriterTableVersionSix writer = + mock(HoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + HoodieInstant compactionInstant = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "200", "201"); + HoodieTimeline deltaCommits = mock(HoodieTimeline.class); + when(deltaCommits.countInstants()).thenReturn(2); + when(deltaCommits.getInstants()).thenReturn(Collections.emptyList()); + Method validateRollback = HoodieBackedTableMetadataWriterTableVersionSix.class + .getDeclaredMethod( + "validateRollbackVersionSix", String.class, HoodieInstant.class, HoodieTimeline.class); + validateRollback.setAccessible(true); + + InvocationTargetException exception = assertThrows( + InvocationTargetException.class, + () -> validateRollback.invoke(writer, "100", compactionInstant, deltaCommits)); + assertTrue(exception.getCause() instanceof HoodieMetadataException); + } + private HoodieBackedTableMetadataWriterTableVersionSix createMockWriter(HoodieTableMetaClient dataMetaClient) throws Exception { // Use CALLS_REAL_METHODS so that shouldInitializeFromFilesystem executes the real logic HoodieBackedTableMetadataWriterTableVersionSix writer = mock(HoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataWriteUtils.java index 73d10ef228d23..8845cbe8912ad 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataWriteUtils.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataWriteUtils.java @@ -32,10 +32,16 @@ import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.hudi.metrics.MetricsReporterType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import java.util.Collections; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -47,6 +53,46 @@ public class TestHoodieMetadataWriteUtils { + @ParameterizedTest + @EnumSource(value = MetricsReporterType.class, names = { + "GRAPHITE", "JMX", "PROMETHEUS_PUSHGATEWAY", "M3", "PROMETHEUS" + }) + void testCreateMetadataWriteConfigPropagatesSupportedMetricsReporter(MetricsReporterType reporterType) { + // Metadata writes must retain every reporter supported by the writer path. + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath("/tmp/base_path/") + .withProps(Collections.singletonMap( + "hoodie.metrics.graphite.metric.prefix", "metadata-test")) + .withMetricsConfig(HoodieMetricsConfig.newBuilder() + .on(true) + .withReporterType(reporterType.name()) + .build()) + .build(); + + HoodieWriteConfig metadataWriteConfig = HoodieMetadataWriteUtils.createMetadataWriteConfig( + writeConfig, HoodieFailedWritesCleaningPolicy.EAGER, HoodieTableVersion.EIGHT); + + assertTrue(metadataWriteConfig.isMetricsOn()); + assertEquals(reporterType, metadataWriteConfig.getMetricsReporterType()); + } + + @Test + void testCreateMetadataWriteConfigRejectsUnsupportedMetricsReporter() { + // Reject unsupported reporters before the metadata writer starts. + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath("/tmp/base_path/") + .withMetricsConfig(HoodieMetricsConfig.newBuilder() + .on(true) + .withReporterType(MetricsReporterType.SLF4J.name()) + .build()) + .build(); + + HoodieMetadataException exception = assertThrows(HoodieMetadataException.class, + () -> HoodieMetadataWriteUtils.createMetadataWriteConfig( + writeConfig, HoodieFailedWritesCleaningPolicy.EAGER, HoodieTableVersion.EIGHT)); + assertTrue(exception.getMessage().contains("Unsupported Metrics Reporter type SLF4J")); + } + @Test public void testCreateMetadataWriteConfigForCleaner() { HoodieWriteConfig writeConfig1 = HoodieWriteConfig.newBuilder() diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestSecondaryIndexRecordGenerationUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestSecondaryIndexRecordGenerationUtils.java new file mode 100644 index 0000000000000..3db50783cc6ca --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestSecondaryIndexRecordGenerationUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.metadata; + +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +/** + * Tests validation and error propagation before secondary-index record generation. + */ +class TestSecondaryIndexRecordGenerationUtils { + + @Test + void rejectsLogFileInsertsBeforeReadingFileSlices() { + // Log-file inserts cannot be reconstructed without a base-file slice. + HoodieWriteStat writeStat = new HoodieWriteStat(); + writeStat.setPartitionPath("p1"); + writeStat.setPath("p1/.fileid-1_014.log.1_1-0-1"); + writeStat.setNumInserts(1); + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp/table").build(); + + assertThrows(HoodieIOException.class, + () -> SecondaryIndexRecordGenerationUtils.convertWriteStatsToSecondaryIndexRecords( + Collections.singletonList(writeStat), "001", null, null, null, null, writeConfig)); + } + + @Test + void wrapsTableSchemaResolutionFailure() { + // Wrap schema lookup failures in the metadata utility's exception type. + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp/table").build(); + + try (MockedStatic metadataUtil = + mockStatic(HoodieTableMetadataUtil.class, CALLS_REAL_METHODS)) { + metadataUtil.when(() -> HoodieTableMetadataUtil.tryResolveSchemaForTable(metaClient)) + .thenThrow(new IllegalStateException("no schema")); + + assertThrows(HoodieException.class, + () -> SecondaryIndexRecordGenerationUtils.convertWriteStatsToSecondaryIndexRecords( + Collections.emptyList(), "001", null, null, metaClient, null, writeConfig)); + } + } +} diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestSparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestSparkHoodieBackedTableMetadataWriter.java new file mode 100644 index 0000000000000..370d715125360 --- /dev/null +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestSparkHoodieBackedTableMetadataWriter.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.metadata; + +import org.apache.hudi.client.BaseHoodieWriteClient; +import org.apache.hudi.client.HoodieWriteResult; +import org.apache.hudi.client.SparkRDDMetadataWriteClient; +import org.apache.hudi.client.SparkRDDWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.index.HoodieSparkIndexClient; + +import org.apache.spark.api.java.JavaRDD; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; + +import java.util.Collections; + +import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION; +import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests Spark-specific behavior shared by current and table-version-six metadata writers. + */ +class TestSparkHoodieBackedTableMetadataWriter { + + @Test + void exposesSparkEngineAndRejectsV6StreamingConversion() { + // Both implementations use Spark, but version 6 has no streaming conversion. + SparkHoodieBackedTableMetadataWriter currentWriter = + mock(SparkHoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + SparkHoodieBackedTableMetadataWriterTableVersionSix versionSixWriter = + mock(SparkHoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + + assertEquals(EngineType.SPARK, currentWriter.getEngineType()); + assertEquals(EngineType.SPARK, versionSixWriter.getEngineType()); + assertThrows(HoodieNotSupportedException.class, + () -> versionSixWriter.convertEngineSpecificDataToHoodieData(null)); + } + + @Test + void versionSixFileGroupUpsertCommitsWriteStatuses() { + // Version 6 file-group writes use a prepped-record upsert and delta commit. + SparkHoodieBackedTableMetadataWriterTableVersionSix writer = + mock(SparkHoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + BaseHoodieWriteClient, ?, JavaRDD> writeClient = + mock(BaseHoodieWriteClient.class); + JavaRDD records = mock(JavaRDD.class); + JavaRDD writeStatuses = mock(JavaRDD.class); + when(writeClient.upsertPreppedRecords(records, "001")).thenReturn(writeStatuses); + + writer.upsertAndCommit(writeClient, "001", records, Collections.emptyList()); + + verify(writeClient).commit( + "001", writeStatuses, Option.empty(), DELTA_COMMIT_ACTION, Collections.emptyMap()); + } + + @Test + void currentFileGroupUpsertCommitsFirstUpsertStatuses() { + // Current writer uses first-upsert semantics when file groups are supplied. + SparkHoodieBackedTableMetadataWriter writer = + mock(SparkHoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + SparkRDDMetadataWriteClient writeClient = mock(SparkRDDMetadataWriteClient.class); + JavaRDD records = mock(JavaRDD.class); + JavaRDD writeStatuses = mock(JavaRDD.class); + java.util.List fileGroups = Collections.emptyList(); + // The writer resolves its internal client even when one is passed in. + doReturn(writeClient).when(writer).getWriteClient(); + when(writeClient.firstUpsertPreppedRecords(records, "003", fileGroups)).thenReturn(writeStatuses); + + writer.upsertAndCommit(writeClient, "003", records, fileGroups); + + verify(writeClient).commit( + "003", writeStatuses, Option.empty(), DELTA_COMMIT_ACTION, Collections.emptyMap()); + } + + @Test + void currentUpsertCoalescesRecordPreparationInput() { + // Record preparation honors its configured parallelism before writing. + SparkHoodieBackedTableMetadataWriter writer = + mock(SparkHoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + HoodieMetadataConfig metadataConfig = mock(HoodieMetadataConfig.class); + HoodieWriteConfig dataWriteConfig = mock(HoodieWriteConfig.class); + when(dataWriteConfig.getMetadataConfig()).thenReturn(metadataConfig); + when(metadataConfig.getRecordPreparationParallelism()).thenReturn(1); + writer.dataWriteConfig = dataWriteConfig; + + BaseHoodieWriteClient, ?, JavaRDD> writeClient = + mock(BaseHoodieWriteClient.class); + JavaRDD records = mock(JavaRDD.class); + JavaRDD coalescedRecords = mock(JavaRDD.class); + JavaRDD writeStatuses = mock(JavaRDD.class); + when(records.getNumPartitions()).thenReturn(2); + when(records.coalesce(1)).thenReturn(coalescedRecords); + when(writeClient.upsertPreppedRecords(coalescedRecords, "004")).thenReturn(writeStatuses); + + writer.upsertAndCommit(writeClient, "004", records); + + verify(writeClient).commit( + "004", writeStatuses, Option.empty(), DELTA_COMMIT_ACTION, Collections.emptyMap()); + } + + @Test + void bothSparkWritersUpdateColumnStatsDefinition() { + // Both writer versions delegate column-stat definitions to the Spark index client. + SparkHoodieBackedTableMetadataWriter currentWriter = + mock(SparkHoodieBackedTableMetadataWriter.class, CALLS_REAL_METHODS); + SparkHoodieBackedTableMetadataWriterTableVersionSix versionSixWriter = + mock(SparkHoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + java.util.List columns = Collections.singletonList("rider"); + + // Capture index clients created internally by both writer versions. + try (MockedConstruction construction = + mockConstruction(HoodieSparkIndexClient.class)) { + currentWriter.updateColumnsToIndexWithColStats(columns); + versionSixWriter.updateColumnsToIndexWithColStats(columns); + + assertEquals(2, construction.constructed().size()); + verify(construction.constructed().get(0)) + .createOrUpdateColumnStatsIndexDefinition(null, columns); + verify(construction.constructed().get(1)) + .createOrUpdateColumnStatsIndexDefinition(null, columns); + } + } + + @Test + void versionSixDeletePartitionsCommitsReplaceCommit() { + // Deleting an MDT partition is committed as a replace commit. + SparkHoodieBackedTableMetadataWriterTableVersionSix writer = + mock(SparkHoodieBackedTableMetadataWriterTableVersionSix.class, CALLS_REAL_METHODS); + SparkRDDWriteClient writeClient = mock(SparkRDDWriteClient.class); + HoodieWriteResult writeResult = mock(HoodieWriteResult.class); + JavaRDD writeStatuses = mock(JavaRDD.class); + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + writer.metadataMetaClient = metadataMetaClient; + doReturn(writeClient).when(writer).getWriteClient(); + when(writeResult.getWriteStatuses()).thenReturn(writeStatuses); + when(writeResult.getPartitionToReplaceFileIds()).thenReturn(Collections.emptyMap()); + when(writeClient.deletePartitions( + Collections.singletonList(MetadataPartitionType.RECORD_INDEX.getPartitionPath()), "002")) + .thenReturn(writeResult); + + writer.deletePartitions("002", Collections.singletonList(MetadataPartitionType.RECORD_INDEX)); + + verify(writeClient).startCommitForMetadataTable(eq(metadataMetaClient), eq("002"), any()); + verify(writeClient).commit( + "002", writeStatuses, Option.empty(), REPLACE_COMMIT_ACTION, Collections.emptyMap()); + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkSQL.scala new file mode 100644 index 0000000000000..9b8f9cb5bed3f --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkSQL.scala @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.client.SparkRDDWriteClient +import org.apache.hudi.client.common.HoodieSparkEngineContext +import org.apache.hudi.client.transaction.lock.InProcessLockProvider +import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} +import org.apache.hudi.common.model.HoodieTableType +import org.apache.hudi.common.table.{HoodieTableMetaClient, HoodieTableVersion, TableSchemaResolver} +import org.apache.hudi.common.testutils.HoodieTestUtils +import org.apache.hudi.config.{HoodieLockConfig, HoodieWriteConfig} +import org.apache.hudi.metadata.HoodieMetadataPayload.SECONDARY_INDEX_RECORD_KEY_SEPARATOR +import org.apache.hudi.metadata.MetadataPartitionType +import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf +import org.apache.hudi.testutils.SparkClientFunctionalTestHarnessScala + +import org.apache.spark.SparkConf +import org.junit.jupiter.api.{BeforeEach, Tag, Test} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} + +import scala.collection.JavaConverters.mapAsJavaMapConverter + +/** + * Exercises {@code SparkHoodieBackedTableMetadataWriter} and its table-version-six + * implementation through real Spark writes. + */ +@Tag("functional-c") +class TestMetadataTableWithSparkSQL extends SparkClientFunctionalTestHarnessScala { + + override def conf: SparkConf = conf(getSparkSqlConf) + + @BeforeEach + override def runBeforeEach(): Unit = { + super.runBeforeEach() + spark.sql(s"set ${HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key()} = ${classOf[InProcessLockProvider].getName}") + } + + @Test + def testAllMetadataIndexesAcrossUpsertAndRollback(): Unit = { + // Validate the full MDT lifecycle against a current-version COW table. + val tableName = "metadata_writer_all_indexes" + val tablePath = s"$basePath/$tableName" + val writeOptions = metadataWriteOptions(tableName, streamingWrites = true) + + // Bootstrap all supported MDT partitions through real writes. + createTable(tableName, tablePath, tableVersion = None) + spark.sql( + s"""insert into $tableName values + | (1, 'row1', 'alpha', 'p1'), + | (2, 'row2', 'beta', 'p1'), + | (3, 'row3', 'gamma', 'p2') + |""".stripMargin) + spark.sql(s"create index idx_rider on $tableName (rider)") + + var metaClient = createMetaClient(tablePath) + assertMetadataPartitions(metaClient, includePartitionStats = true, includeSecondaryIndex = true) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 3, includePartitionStats = true) + // Metadata payload type 7 stores secondary-index mappings. + checkAnswer(s"select key from hudi_metadata('$tablePath') where type=7")( + Seq(s"alpha${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1"), + Seq(s"beta${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2"), + Seq(s"gamma${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3") + ) + + // Rebuild the secondary index and verify its records are removed and restored. + spark.sql(s"drop index idx_rider on $tableName") + metaClient = HoodieTableMetaClient.reload(metaClient) + assertFalse(metaClient.getTableConfig.getMetadataPartitions.contains("secondary_index_idx_rider")) + assertEquals(0L, spark.sql(s"select key from hudi_metadata('$tablePath') where type=7").count()) + + spark.sql(s"create index idx_rider on $tableName (rider)") + metaClient = HoodieTableMetaClient.reload(metaClient) + assertMetadataPartitions(metaClient, includePartitionStats = true, includeSecondaryIndex = true) + checkAnswer(s"select key from hudi_metadata('$tablePath') where type=7")( + Seq(s"alpha${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1"), + Seq(s"beta${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2"), + Seq(s"gamma${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3") + ) + + // Upsert changes both data and the secondary-index key. + spark.sql(s"update $tableName set rider = 'delta', ts = 4 where id = 'row1'") + metaClient = HoodieTableMetaClient.reload(metaClient) + val upsertInstant = metaClient.getActiveTimeline.getCommitsTimeline + .filterCompletedInstants.lastInstant().get().requestedTime() + + checkAnswer(s"select id, rider, part from $tableName order by id")( + Seq("row1", "delta", "p1"), + Seq("row2", "beta", "p1"), + Seq("row3", "gamma", "p2") + ) + checkAnswer(s"select key from hudi_metadata('$tablePath') where type=7")( + Seq(s"delta${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1"), + Seq(s"beta${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2"), + Seq(s"gamma${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3") + ) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 3, includePartitionStats = true) + + // Rollback must restore data and all MDT indexes. + rollback(metaClient, writeOptions, upsertInstant) + spark.catalog.refreshTable(tableName) + metaClient = HoodieTableMetaClient.reload(metaClient) + + checkAnswer(s"select id, rider, part from $tableName order by id")( + Seq("row1", "alpha", "p1"), + Seq("row2", "beta", "p1"), + Seq("row3", "gamma", "p2") + ) + assertMetadataPartitions(metaClient, includePartitionStats = true, includeSecondaryIndex = true) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 3, includePartitionStats = true) + checkAnswer(s"select key from hudi_metadata('$tablePath') where type=7")( + Seq(s"alpha${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1"), + Seq(s"beta${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2"), + Seq(s"gamma${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3") + ) + } + + @Test + def testTableVersionSixMetadataWritesAndRollback(): Unit = { + // Validate the legacy writer with the indexes supported by table version 6. + val tableName = "metadata_writer_v6" + val tablePath = s"$basePath/$tableName" + val writeOptions = metadataWriteOptions(tableName, streamingWrites = false) + + (HoodieWriteConfig.WRITE_TABLE_VERSION.key() -> HoodieTableVersion.SIX.versionCode().toString) + + // Version 6 uses the legacy writer and supports a smaller index set. + createTable(tableName, tablePath, tableVersion = Some(HoodieTableVersion.SIX.versionCode())) + spark.sql( + s"""insert into $tableName values + | (1, 'row1', 'alpha', 'p1'), + | (2, 'row2', 'beta', 'p2') + |""".stripMargin) + + var metaClient = createMetaClient(tablePath) + val metadataMetaClient = createMetaClient(metaClient.getMetaPath + "/metadata") + assertEquals(HoodieTableVersion.SIX, metaClient.getTableConfig.getTableVersion) + assertEquals(HoodieTableVersion.SIX, metadataMetaClient.getTableConfig.getTableVersion) + assertMetadataPartitions(metaClient, includePartitionStats = false, includeSecondaryIndex = false) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 2, includePartitionStats = false) + + spark.sql(s"update $tableName set rider = 'updated', ts = 3 where id = 'row1'") + metaClient = HoodieTableMetaClient.reload(metaClient) + val upsertInstant = metaClient.getActiveTimeline.getCommitsTimeline + .filterCompletedInstants.lastInstant().get().requestedTime() + checkAnswer(s"select id, rider from $tableName order by id")( + Seq("row1", "updated"), + Seq("row2", "beta") + ) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 2, includePartitionStats = false) + + rollback(metaClient, writeOptions, upsertInstant) + spark.catalog.refreshTable(tableName) + metaClient = HoodieTableMetaClient.reload(metaClient) + + checkAnswer(s"select id, rider from $tableName order by id")( + Seq("row1", "alpha"), + Seq("row2", "beta") + ) + assertEquals(HoodieTableVersion.SIX, metaClient.getTableConfig.getTableVersion) + assertMetadataPartitions(metaClient, includePartitionStats = false, includeSecondaryIndex = false) + assertCommonMetadataRecords(tablePath, expectedRecordIndexCount = 2, includePartitionStats = false) + + // Verify record-index deletion and bootstrap on the legacy table. + spark.sql(s"drop index record_index on $tableName") + metaClient = HoodieTableMetaClient.reload(metaClient) + assertFalse(metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)) + assertEquals(0L, spark.sql(s"select key from hudi_metadata('$tablePath') where type=5").count()) + + spark.sql(s"create index record_index on $tableName (id)") + metaClient = HoodieTableMetaClient.reload(metaClient) + assertTrue(metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)) + assertEquals(2L, spark.sql(s"select key from hudi_metadata('$tablePath') where type=5").count()) + } + + private def createTable( + tableName: String, + tablePath: String, + tableVersion: Option[Int]): Unit = { + val tableVersionOption = tableVersion + .map(version => s"hoodie.write.table.version = '$version',") + .getOrElse("") + val streamingWrites = tableVersion.isEmpty + spark.sql( + s""" + |create table $tableName ( + | ts bigint, + | id string, + | rider string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | type = 'cow', + | $tableVersionOption + | hoodie.metadata.enable = 'true', + | hoodie.metadata.index.column.stats.enable = 'true', + | hoodie.metadata.index.partition.stats.enable = 'true', + | hoodie.metadata.record.index.enable = 'true', + | hoodie.metadata.streaming.write.enabled = '$streamingWrites', + | hoodie.metadata.record.preparation.parallelism = '1', + | hoodie.metrics.on = 'true', + | hoodie.metrics.reporter.type = 'CONSOLE', + | hoodie.metrics.executor.enable = 'true', + | hoodie.datasource.write.recordkey.field = 'id', + | hoodie.datasource.write.partitionpath.field = 'part', + | hoodie.datasource.write.payload.class = + | 'org.apache.hudi.common.model.OverwriteWithLatestAvroPayload' + | ) + | partitioned by (part) + | location '$tablePath' + |""".stripMargin) + } + + private def metadataWriteOptions(tableName: String, streamingWrites: Boolean): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key() -> tableName, + TABLE_TYPE.key() -> HoodieTableType.COPY_ON_WRITE.name(), + RECORDKEY_FIELD.key() -> "id", + PARTITIONPATH_FIELD.key() -> "part", + PRECOMBINE_FIELD.key() -> "ts", + HoodieMetadataConfig.ENABLE.key() -> "true", + HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key() -> "true", + // On release-1.2.1 this key is a deprecated String constant, not a ConfigProperty; + // partition stats are driven by the column stats config above. + HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true", + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key() -> streamingWrites.toString, + HoodieMetadataConfig.RECORD_PREPARATION_PARALLELISM.key() -> "1", + "hoodie.metrics.on" -> "true", + "hoodie.metrics.reporter.type" -> "CONSOLE", + "hoodie.metrics.executor.enable" -> "true", + HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key() -> classOf[InProcessLockProvider].getName + ) + + private def createMetaClient(tablePath: String): HoodieTableMetaClient = + HoodieTableMetaClient.builder() + .setBasePath(tablePath) + .setConf(HoodieTestUtils.getDefaultStorageConf) + .build() + + private def rollback( + metaClient: HoodieTableMetaClient, + writeOptions: Map[String, String], + instantTime: String): Unit = { + val props = TypedProperties.fromMap(writeOptions.asJava) + val writeConfig = HoodieWriteConfig.newBuilder() + .withPath(metaClient.getBasePath) + .withSchema(new TableSchemaResolver(metaClient).getTableSchema(false).toString) + .withProps(props) + .withEmbeddedTimelineServerEnabled(false) + .build() + val writeClient = new SparkRDDWriteClient(new HoodieSparkEngineContext(jsc), writeConfig) + try { + assertTrue(writeClient.rollback(instantTime)) + } finally { + writeClient.close() + } + } + + private def assertMetadataPartitions( + metaClient: HoodieTableMetaClient, + includePartitionStats: Boolean, + includeSecondaryIndex: Boolean): Unit = { + val partitions = metaClient.getTableConfig.getMetadataPartitions + assertTrue(partitions.contains(MetadataPartitionType.FILES.getPartitionPath)) + assertTrue(partitions.contains(MetadataPartitionType.COLUMN_STATS.getPartitionPath)) + assertTrue(partitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)) + if (includePartitionStats) { + assertTrue(partitions.contains(MetadataPartitionType.PARTITION_STATS.getPartitionPath)) + } + if (includeSecondaryIndex) { + assertTrue(partitions.contains("secondary_index_idx_rider")) + } + } + + private def assertCommonMetadataRecords( + tablePath: String, + expectedRecordIndexCount: Long, + includePartitionStats: Boolean): Unit = { + // Payload types 3, 6, and 5 represent column stats, partition stats, and record index. + assertTrue(spark.sql(s"select key from hudi_metadata('$tablePath') where type=3").count() > 0) + if (includePartitionStats) { + assertTrue(spark.sql(s"select key from hudi_metadata('$tablePath') where type=6").count() > 0) + } + assertEquals( + expectedRecordIndexCount, + spark.sql(s"select key from hudi_metadata('$tablePath') where type=5").count()) + } + + private def checkAnswer(query: String)(expected: Seq[Any]*): Unit = { + val expectedRows = expected.map(_.mkString("|")).sorted.toList + val actualRows = spark.sql(query).collect().map(_.toSeq.mkString("|")).sorted.toList + assertEquals(expectedRows, actualRows) + } +} From 3c3a76db4805b95433ae30b97683ca63972d49ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:44:26 +0000 Subject: [PATCH 199/255] chore(deps): bump io.airlift:aircompressor from 0.27 to 2.0.3 (#18247) Bumps [io.airlift:aircompressor](https://github.com/airlift/aircompressor) from 0.27 to 2.0.3. - [Release notes](https://github.com/airlift/aircompressor/releases) - [Commits](https://github.com/airlift/aircompressor/compare/0.27...2.0.3) --- updated-dependencies: - dependency-name: io.airlift:aircompressor dependency-version: 2.0.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit dfb763ec2ef5c09ee04dda5e9aa5d332b52ff4c9) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 57e29a16f5fa6..2b354fa4437bd 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ 1.6.0 1.5.6 0.9.47 - 0.27 + 2.0.3 0.13.0 0.16.0 4.5.13 From 8199b6b798ef640306491310f081663aea260d6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:15:08 +0000 Subject: [PATCH 200/255] chore(deps): bump org.apache.thrift:libthrift (#18910) Bumps [org.apache.thrift:libthrift](https://github.com/apache/thrift) from 0.14.0 to 0.23.0. - [Release notes](https://github.com/apache/thrift/releases) - [Changelog](https://github.com/apache/thrift/blob/master/CHANGES.md) - [Commits](https://github.com/apache/thrift/compare/v0.14.0...v0.23.0) --- updated-dependencies: - dependency-name: org.apache.thrift:libthrift dependency-version: 0.23.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit d4d8bfa08a270fa9377fdf6c0c6c84ca46b59c09) --- packaging/hudi-integ-test-bundle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/hudi-integ-test-bundle/pom.xml b/packaging/hudi-integ-test-bundle/pom.xml index e1e15b7d24d9b..870a9ecadc2fd 100644 --- a/packaging/hudi-integ-test-bundle/pom.xml +++ b/packaging/hudi-integ-test-bundle/pom.xml @@ -679,7 +679,7 @@ org.apache.thrift libthrift - 0.14.0 + 0.23.0 From 278fc83d33baa5d33ceb61345a74f39e3891c297 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:23:47 +0000 Subject: [PATCH 201/255] chore(deps): bump org.apache.commons:commons-configuration2 (#18801) Bumps org.apache.commons:commons-configuration2 from 2.11.0 to 2.15.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-configuration2 dependency-version: 2.15.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 503963910d84224b094d7ecfb8ca59d6881f8dc8) --- packaging/hudi-cli-bundle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/hudi-cli-bundle/pom.xml b/packaging/hudi-cli-bundle/pom.xml index f63b9edbc0739..e65c89e24d205 100644 --- a/packaging/hudi-cli-bundle/pom.xml +++ b/packaging/hudi-cli-bundle/pom.xml @@ -36,7 +36,7 @@ 2.0.2 3.21.0 2.11.0 - 2.11.0 + 2.15.0 From 4ac33ddcaa74b5490263b585cb62d97c6332a816 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Fri, 24 Jul 2026 19:50:07 +0700 Subject: [PATCH 202/255] fix(hive-sync): stop the HMS lock heartbeat once the metastore drops the lock (#19370) * fix(hive-sync): stop the HMS lock heartbeat once the metastore drops the lock * addressed review comments: scope the lock-loss callback to its lock id * addressed review comments: document the unlock throw, snapshot lock reads and extend lock-loss coverage * addressed review comments: skip the heartbeat for a lock that was never granted --------- Co-authored-by: Vova Kolmakov (cherry picked from commit 48fe2c6bd0a44fa1ef5693aa7f7460c32f0cfd61) --- .../hudi/hive/transaction/lock/Heartbeat.java | 22 +- .../lock/HiveMetastoreBasedLockProvider.java | 117 +++++- ...iveMetastoreBasedLockProviderTestBase.java | 95 +++++ .../hive/transaction/lock/TestHeartbeat.java | 73 +++- ...stHiveMetastoreBasedLockProviderClose.java | 50 +-- ...iveMetastoreBasedLockProviderLockLoss.java | 362 ++++++++++++++++++ 6 files changed, 652 insertions(+), 67 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProviderTestBase.java create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderLockLoss.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java index bf0037faf6df1..0766419d677ab 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java @@ -21,21 +21,41 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchLockException; +import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; +import org.apache.hadoop.hive.metastore.api.TxnAbortedException; + +import java.util.function.Consumer; @Slf4j class Heartbeat implements Runnable { private final IMetaStoreClient client; private final long lockId; + private final Consumer onLockLost; + // Latches the terminal failure so that a tick already queued when the schedule was cancelled + // does not issue another doomed heartbeat or report the loss a second time. + private volatile boolean lockLost = false; - Heartbeat(IMetaStoreClient client, long lockId) { + Heartbeat(IMetaStoreClient client, long lockId, Consumer onLockLost) { this.client = client; this.lockId = lockId; + this.onLockLost = onLockLost; } @Override public void run() { + if (lockLost) { + return; + } try { client.heartbeat(0, lockId); + } catch (NoSuchLockException | NoSuchTxnException | TxnAbortedException e) { + // Terminal. The metastore has already expired or aborted this lock, so no later tick can + // renew it. Retrying would only log once per interval while the writer keeps believing it + // holds exclusivity, so report the loss to the owner, which stops the schedule and drops + // the lock. + lockLost = true; + onLockLost.accept(e); } catch (Exception e) { // Do not rethrow. This task is scheduled via ScheduledExecutorService.scheduleAtFixedRate, // where a thrown exception permanently cancels all subsequent executions and is only diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java index c83267be5319c..91d4062ecc3bf 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java @@ -87,7 +87,11 @@ public class HiveMetastoreBasedLockProvider implements LockProvider future = null; + // Assigned by the acquiring thread, read and cancelled by the heartbeat thread. + private transient volatile ScheduledFuture future = null; + // Set when the metastore reports the lock as expired or aborted, so that a later unlock() can + // tell the caller its exclusivity was lost instead of silently doing nothing. + private volatile boolean lockLostRemotely = false; private final transient ScheduledExecutorService executor = Executors.newScheduledThreadPool(2); public HiveMetastoreBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration conf) { @@ -123,21 +127,35 @@ public boolean tryLock(long time, TimeUnit unit) { } catch (ExecutionException | InterruptedException | TimeoutException | TException e) { throw new HoodieLockException(generateLogStatement(FAILED_TO_ACQUIRE, generateLogSuffixString()), e); } - return this.lock != null && this.lock.getState() == LockState.ACQUIRED; + return isLockAcquired(); } + /** + * Releases the lock held at the metastore, if any. + * + *

    Unlike most providers, which stay silent when they do not believe they hold a lock, this one + * deliberately fails when the metastore had already expired or aborted the lock: the writer went + * on committing without exclusivity and has to hear about it. Note that {@code LockManager.unlock()} + * skips its metrics and its {@code close()} when the provider throws. + * + * @throws HoodieLockException if the metastore took the lock away, or if releasing it fails. + */ @Override public void unlock() { try { log.info(generateLogStatement(RELEASING, generateLogSuffixString())); LockResponse lockResponseLocal = lock; if (lockResponseLocal == null) { + if (lockLostRemotely) { + // The heartbeat already dropped the lock. Unlocking it would fail with a bare + // NoSuchLockException anyway, so fail with the actual reason instead. + throw new HoodieLockException(generateLogStatement(FAILED_TO_RELEASE, generateLogSuffixString()) + + ", the metastore had already expired or aborted it"); + } return; } lock = null; - if (future != null) { - future.cancel(false); - } + cancelHeartbeat(); hiveClient.unlock(lockResponseLocal.getLockid()); log.info(generateLogStatement(RELEASED, generateLogSuffixString())); } catch (TException e) { @@ -159,12 +177,14 @@ public void acquireLock(long time, TimeUnit unit) throws InterruptedException, E @Override public void close() { try { - if (lock != null) { - hiveClient.unlock(lock.getLockid()); - lock = null; - } - if (future != null) { - future.cancel(false); + // Snapshot the lock, then stop claiming it before releasing it, exactly as unlock() does: + // the release itself makes an in-flight heartbeat fail, and that must not be mistaken for + // the metastore taking the lock away. + LockResponse lockResponseLocal = lock; + lock = null; + cancelHeartbeat(); + if (lockResponseLocal != null) { + hiveClient.unlock(lockResponseLocal.getLockid()); } Hive.closeCurrent(); } catch (Exception e) { @@ -181,12 +201,22 @@ public boolean acquireLock(long time, TimeUnit unit, final LockComponent compone throws InterruptedException, ExecutionException, TimeoutException, TException { ValidationUtils.checkArgument(this.lock == null, ALREADY_ACQUIRED.name()); acquireLockInternal(time, unit, component); - return this.lock != null && this.lock.getState() == LockState.ACQUIRED; + return isLockAcquired(); + } + + /** + * Whether the lock is held right now. Reads {@link #lock} once: the heartbeat thread clears it + * as soon as the metastore reports the lock as gone, so re-reading the field can mix two states. + */ + private boolean isLockAcquired() { + LockResponse lockResponseLocal = this.lock; + return lockResponseLocal != null && lockResponseLocal.getState() == LockState.ACQUIRED; } private void acquireLockInternal(long time, TimeUnit unit, LockComponent lockComponent) throws InterruptedException, ExecutionException, TimeoutException, TException { LockRequest lockRequest = null; + lockLostRemotely = false; try { // TODO : FIX:Using the parameterized constructor throws MethodNotFound final LockRequestBuilder builder = new LockRequestBuilder(); @@ -210,27 +240,76 @@ private void acquireLockInternal(long time, TimeUnit unit, LockComponent lockCom } } finally { // it is better to release WAITING lock, otherwise hive lock will hang forever - if (this.lock != null && this.lock.getState() != LockState.ACQUIRED) { - hiveClient.unlock(this.lock.getLockid()); + // Snapshot the lock: the heartbeat thread clears it as soon as the metastore reports it gone. + LockResponse lockResponseLocal = this.lock; + if (lockResponseLocal != null && lockResponseLocal.getState() != LockState.ACQUIRED) { + hiveClient.unlock(lockResponseLocal.getLockid()); lock = null; - if (future != null) { - future.cancel(false); - } + cancelHeartbeat(); } } } /** * Schedules a periodic {@link Heartbeat} to refresh the currently held lock in case a commit - * takes a long time. Must be called only after {@link #lock} has been set. + * takes a long time. Does nothing unless {@link #lock} is held right now, so that callers may + * invoke it without checking the state of the response they just got. */ private void scheduleHeartbeat() { - Heartbeat heartbeat = new Heartbeat(hiveClient, lock.getLockid()); + LockResponse lockResponseLocal = lock; + if (lockResponseLocal == null) { + // Released while the acquisition was still completing, so there is nothing left to renew. + return; + } + if (lockResponseLocal.getState() != LockState.ACQUIRED) { + // The metastore only queued the request. The caller releases such a lock right away, and + // renewing one that was never granted can only latch a loss that never happened. + return; + } + // Bind the id into the task and its callback: cancelling a schedule does not stop a tick that + // is already inside the heartbeat RPC, so a tick can outlive the lock it was scheduled for. + long lockId = lockResponseLocal.getLockid(); + Heartbeat heartbeat = new Heartbeat(hiveClient, lockId, cause -> onLockLost(lockId, cause)); long heartbeatIntervalMs = lockConfiguration.getConfig() .getLong(LOCK_HEARTBEAT_INTERVAL_MS_KEY, DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS); future = executor.scheduleAtFixedRate(heartbeat, heartbeatIntervalMs / 2, heartbeatIntervalMs, TimeUnit.MILLISECONDS); } + /** + * Invoked from the heartbeat thread once the metastore reports the lock as expired or aborted. + * The lock cannot be renewed anymore, so stop heartbeating it and stop claiming it is held. + * + * @param lockId the lock this heartbeat was scheduled for, which is not necessarily the one held + * now: releasing a lock is itself a reason for an in-flight heartbeat to fail. + */ + private void onLockLost(long lockId, Exception cause) { + LockResponse lockResponseLocal = lock; + if (lockResponseLocal == null || lockResponseLocal.getLockid() != lockId) { + // We released this lock ourselves while the tick was in flight, which is why the metastore + // no longer knows about it. Nothing was lost, and any lock held now is a different one that + // keeps its own heartbeat. + log.debug("Ignoring a heartbeat failure for the already released lock {}", lockId, cause); + return; + } + // Order matters: the flag is set first, so an unlock() racing with this can never find the + // lock gone without a reason for it; and the schedule is cancelled before the lock is dropped, + // so whoever observes the drop is guaranteed to see the renewal already stopped. + lockLostRemotely = true; + cancelHeartbeat(); + lock = null; + log.error("The metastore expired or aborted the lock at{}, heartbeat stopped and exclusivity is lost", + generateLogSuffixString(), cause); + } + + private void cancelHeartbeat() { + ScheduledFuture futureLocal = future; + if (futureLocal != null) { + // Never interrupt: this can run on the heartbeat thread itself, and the current tick is + // harmless. Cancelling only prevents further executions. + futureLocal.cancel(false); + } + } + private void checkRequiredProps(final LockConfiguration lockConfiguration) { ValidationUtils.checkArgument(lockConfiguration.getConfig().getString(HIVE_DATABASE_NAME_PROP_KEY) != null); ValidationUtils.checkArgument(lockConfiguration.getConfig().getString(HIVE_TABLE_NAME_PROP_KEY) != null); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProviderTestBase.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProviderTestBase.java new file mode 100644 index 0000000000000..0e329e3b2f459 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProviderTestBase.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.transaction.lock; + +import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.config.TypedProperties; + +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockLevel; +import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.LockType; +import org.junit.jupiter.api.BeforeEach; + +import java.lang.reflect.Field; + +import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS; +import static org.apache.hudi.common.config.LockConfiguration.HIVE_DATABASE_NAME_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.HIVE_TABLE_NAME_PROP_KEY; +import static org.apache.hudi.common.config.LockConfiguration.LOCK_HEARTBEAT_INTERVAL_MS_KEY; + +/** + * Shared fixture for the {@link HiveMetastoreBasedLockProvider} unit tests that drive the provider + * against a mocked {@code IMetaStoreClient}, without a live metastore or ZooKeeper. + */ +abstract class HiveMetastoreBasedLockProviderTestBase { + + protected static final String DB = "testdb"; + protected static final String TABLE = "testtable"; + + protected LockConfiguration lockConfiguration; + protected LockComponent lockComponent; + + @BeforeEach + void setUpLockFixture() { + TypedProperties props = new TypedProperties(); + props.setProperty(HIVE_DATABASE_NAME_PROP_KEY, DB); + props.setProperty(HIVE_TABLE_NAME_PROP_KEY, TABLE); + props.setProperty(LOCK_HEARTBEAT_INTERVAL_MS_KEY, String.valueOf(heartbeatIntervalMs())); + lockConfiguration = new LockConfiguration(props); + lockComponent = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, DB); + lockComponent.setTablename(TABLE); + } + + /** + * The heartbeat interval the provider is configured with, overridden by tests that need the + * scheduled heartbeat to actually fire while the test runs. + */ + protected long heartbeatIntervalMs() { + return DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS; + } + + protected static LockResponse acquiredLock(long lockId) { + return lockResponse(lockId, LockState.ACQUIRED); + } + + protected static LockResponse waitingLock(long lockId) { + return lockResponse(lockId, LockState.WAITING); + } + + private static LockResponse lockResponse(long lockId, LockState state) { + LockResponse response = new LockResponse(); + response.setLockid(lockId); + response.setState(state); + return response; + } + + /** + * Reads a private field of the provider, for the state it does not expose: the heartbeat + * schedule and the thread pool running it. + */ + @SuppressWarnings("unchecked") + protected static T readField(HiveMetastoreBasedLockProvider provider, String name) throws Exception { + Field field = HiveMetastoreBasedLockProvider.class.getDeclaredField(name); + field.setAccessible(true); + return (T) field.get(provider); + } +} diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java index ca2386f7a28f5..2dbdf827de87f 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHeartbeat.java @@ -20,10 +20,22 @@ package org.apache.hudi.hive.transaction.lock; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchLockException; +import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; +import org.apache.hadoop.hive.metastore.api.TxnAbortedException; import org.apache.thrift.TException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -32,24 +44,81 @@ class TestHeartbeat { + private final List lockLostCauses = new ArrayList<>(); + + /** + * Failures that mean the metastore has already expired or aborted the lock. They are declared by + * {@code IMetaStoreClient.heartbeat(long, long)} and cannot be recovered from by retrying. + */ + private static Stream terminalFailures() { + return Stream.of( + new NoSuchLockException("lock does not exist"), + new NoSuchTxnException("txn does not exist"), + new TxnAbortedException("txn was aborted")); + } + @Test void runDoesNotRethrowWhenHeartbeatFails() throws TException { IMetaStoreClient client = mock(IMetaStoreClient.class); doThrow(new TException("transient failure")).when(client).heartbeat(anyLong(), anyLong()); - Heartbeat heartbeat = new Heartbeat(client, 7L); + Heartbeat heartbeat = new Heartbeat(client, 7L, lockLostCauses::add); // Rethrowing here would cancel every subsequent execution of a scheduleAtFixedRate task, // silently stopping lock renewal. The fix must swallow the failure so the next tick retries. assertDoesNotThrow(heartbeat::run); + assertTrue(lockLostCauses.isEmpty(), "a transient failure must not be reported as a lost lock"); + } + + @Test + void runKeepsRetryingAfterTransientFailure() throws TException { + IMetaStoreClient client = mock(IMetaStoreClient.class); + doThrow(new TException("transient failure")).when(client).heartbeat(anyLong(), anyLong()); + + Heartbeat heartbeat = new Heartbeat(client, 7L, lockLostCauses::add); + heartbeat.run(); + heartbeat.run(); + + verify(client, times(2)).heartbeat(0L, 7L); + assertTrue(lockLostCauses.isEmpty()); } @Test void runHeartbeatsTheLockOnSuccess() throws TException { IMetaStoreClient client = mock(IMetaStoreClient.class); - new Heartbeat(client, 99L).run(); + new Heartbeat(client, 99L, lockLostCauses::add).run(); verify(client, times(1)).heartbeat(0L, 99L); + assertTrue(lockLostCauses.isEmpty()); + } + + @ParameterizedTest + @MethodSource("terminalFailures") + void runReportsLockLossOnTerminalFailure(Exception terminalFailure) throws TException { + IMetaStoreClient client = mock(IMetaStoreClient.class); + doThrow(terminalFailure).when(client).heartbeat(anyLong(), anyLong()); + + Heartbeat heartbeat = new Heartbeat(client, 11L, lockLostCauses::add); + + assertDoesNotThrow(heartbeat::run); + assertEquals(1, lockLostCauses.size()); + assertSame(terminalFailure, lockLostCauses.get(0)); + } + + @ParameterizedTest + @MethodSource("terminalFailures") + void runStopsHeartbeatingAfterTerminalFailure(Exception terminalFailure) throws TException { + IMetaStoreClient client = mock(IMetaStoreClient.class); + doThrow(terminalFailure).when(client).heartbeat(anyLong(), anyLong()); + + Heartbeat heartbeat = new Heartbeat(client, 11L, lockLostCauses::add); + heartbeat.run(); + // A tick already queued when the schedule was cancelled must not renew a lock that is gone, + // nor report the loss a second time. + heartbeat.run(); + + verify(client, times(1)).heartbeat(0L, 11L); + assertEquals(1, lockLostCauses.size()); } } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java index 817c1407f3b9e..3d3a8c26c4e89 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderClose.java @@ -19,25 +19,13 @@ package org.apache.hudi.hive.transaction.lock; -import org.apache.hudi.common.config.LockConfiguration; -import org.apache.hudi.common.config.TypedProperties; - import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.LockComponent; -import org.apache.hadoop.hive.metastore.api.LockLevel; -import org.apache.hadoop.hive.metastore.api.LockResponse; -import org.apache.hadoop.hive.metastore.api.LockState; -import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.thrift.TException; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.lang.reflect.Field; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import static org.apache.hudi.common.config.LockConfiguration.HIVE_DATABASE_NAME_PROP_KEY; -import static org.apache.hudi.common.config.LockConfiguration.HIVE_TABLE_NAME_PROP_KEY; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -51,23 +39,7 @@ * Unit tests for {@link HiveMetastoreBasedLockProvider#close()} that exercise the thread-pool * shutdown path with a mocked {@link IMetaStoreClient}, without a live metastore or ZooKeeper. */ -class TestHiveMetastoreBasedLockProviderClose { - - private static final String DB = "testdb"; - private static final String TABLE = "testtable"; - - private LockConfiguration lockConfiguration; - private LockComponent lockComponent; - - @BeforeEach - void setUp() { - TypedProperties props = new TypedProperties(); - props.setProperty(HIVE_DATABASE_NAME_PROP_KEY, DB); - props.setProperty(HIVE_TABLE_NAME_PROP_KEY, TABLE); - lockConfiguration = new LockConfiguration(props); - lockComponent = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, DB); - lockComponent.setTablename(TABLE); - } +class TestHiveMetastoreBasedLockProviderClose extends HiveMetastoreBasedLockProviderTestBase { @Test void closeShutsDownExecutorEvenWhenUnlockThrows() throws Exception { @@ -80,8 +52,8 @@ void closeShutsDownExecutorEvenWhenUnlockThrows() throws Exception { // A failing unlock() must not prevent the heartbeat thread pool from being shut down. assertDoesNotThrow(provider::close); - assertTrue(executorOf(provider).isShutdown(), - "executor must be shut down even when unlock() throws"); + ScheduledExecutorService executor = readField(provider, "executor"); + assertTrue(executor.isShutdown(), "executor must be shut down even when unlock() throws"); } @Test @@ -95,19 +67,7 @@ void closeShutsDownExecutorOnNormalPath() throws Exception { provider.close(); verify(client).unlock(1L); - assertTrue(executorOf(provider).isShutdown()); - } - - private static LockResponse acquiredLock(long lockId) { - LockResponse response = new LockResponse(); - response.setLockid(lockId); - response.setState(LockState.ACQUIRED); - return response; - } - - private static ScheduledExecutorService executorOf(HiveMetastoreBasedLockProvider provider) throws Exception { - Field field = HiveMetastoreBasedLockProvider.class.getDeclaredField("executor"); - field.setAccessible(true); - return (ScheduledExecutorService) field.get(provider); + ScheduledExecutorService executor = readField(provider, "executor"); + assertTrue(executor.isShutdown()); } } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderLockLoss.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderLockLoss.java new file mode 100644 index 0000000000000..b485f4b9cc391 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderLockLoss.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.transaction.lock; + +import org.apache.hudi.exception.HoodieLockException; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchLockException; +import org.apache.thrift.TException; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for how {@link HiveMetastoreBasedLockProvider} reacts to the metastore reporting its + * lock as gone, with a mocked {@link IMetaStoreClient} and no live metastore or ZooKeeper. + */ +class TestHiveMetastoreBasedLockProviderLockLoss extends HiveMetastoreBasedLockProviderTestBase { + + private static final long LOCK_ID = 42L; + private static final long OTHER_LOCK_ID = 43L; + private static final long HEARTBEAT_INTERVAL_MS = 100L; + private static final long AWAIT_TIMEOUT_MS = 30_000L; + + @Override + protected long heartbeatIntervalMs() { + // Keep the ticks short so the scheduled heartbeat fires within the test. + return HEARTBEAT_INTERVAL_MS; + } + + @Test + void terminalHeartbeatFailureStopsRenewalAndDropsTheLock() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID)); + doThrow(new NoSuchLockException("lock " + LOCK_ID + " does not exist")) + .when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + // The metastore has expired the lock, so the provider must stop claiming to hold it. + awaitUntil(() -> provider.getLock() == null, "the lost lock must be dropped by the heartbeat"); + + // The heartbeat task latches the failure on its own, so the count below would stay at one + // even if the schedule were left running. Assert the cancellation itself as well. + assertTrue(heartbeatFutureOf(provider).isCancelled(), "the heartbeat schedule must be cancelled"); + + // Give the scheduler several more intervals: no further heartbeat may be attempted, since + // both the schedule and the heartbeat task itself are stopped after a terminal failure. + Thread.sleep(HEARTBEAT_INTERVAL_MS * 5); + verify(client, times(1)).heartbeat(0L, LOCK_ID); + + // The writer must learn that it no longer holds exclusivity, and the provider must not send + // a doomed unlock for a lock the metastore has already dropped. + assertThrows(HoodieLockException.class, provider::unlock); + verify(client, never()).unlock(anyLong()); + } finally { + provider.close(); + } + } + + @Test + void lockLostAfterTryLockIsReportedOnUnlock() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID)); + doThrow(new NoSuchLockException("lock " + LOCK_ID + " does not exist")) + .when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + // Same loss, but driven through the entry point the LockManager actually calls rather than + // the test-only acquireLock overload. + assertTrue(provider.tryLock(1000L, TimeUnit.MILLISECONDS)); + awaitUntil(() -> provider.getLock() == null, "the lost lock must be dropped by the heartbeat"); + + assertThrows(HoodieLockException.class, provider::unlock); + verify(client, never()).unlock(anyLong()); + } finally { + provider.close(); + } + } + + @Test + void closeAfterALostLockSendsNoUnlockAndShutsDownTheExecutor() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID)); + doThrow(new NoSuchLockException("lock " + LOCK_ID + " does not exist")) + .when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + awaitUntil(() -> provider.getLock() == null, "the lost lock must be dropped by the heartbeat"); + } finally { + provider.close(); + } + + // There is nothing left to release at the metastore, but the heartbeat pool must still go away. + verify(client, never()).unlock(anyLong()); + ScheduledExecutorService executor = readField(provider, "executor"); + assertTrue(executor.isShutdown(), "the heartbeat pool must be shut down after a lost lock"); + } + + @Test + void transientHeartbeatFailureKeepsRenewingTheLock() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID)); + CountDownLatch heartbeats = new CountDownLatch(2); + doAnswer(invocation -> { + heartbeats.countDown(); + throw new TException("transient failure"); + }).when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + assertTrue(heartbeats.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS), + "a transient failure must not stop the heartbeat schedule"); + assertNotNull(provider.getLock(), "a transient failure must not drop the lock"); + + provider.unlock(); + verify(client).unlock(LOCK_ID); + assertNull(provider.getLock()); + } finally { + provider.close(); + } + } + + @Test + void lockCanBeAcquiredAgainAfterItWasLost() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID), acquiredLock(OTHER_LOCK_ID)); + // Only the first lock is expired by the metastore; heartbeating the second one succeeds. + doThrow(new NoSuchLockException("lock " + LOCK_ID + " does not exist")) + .doNothing() + .when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + awaitUntil(() -> provider.getLock() == null, "the lost lock must be dropped by the heartbeat"); + + // Acquiring again must clear the lost-lock state, otherwise the provider would keep failing + // to release locks it holds perfectly well. + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + provider.unlock(); + + verify(client).unlock(OTHER_LOCK_ID); + assertNull(provider.getLock()); + } finally { + provider.close(); + } + } + + @Test + void lostLockStateDoesNotLeakIntoTheNextAcquire() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID), waitingLock(OTHER_LOCK_ID)); + doThrow(new NoSuchLockException("lock " + LOCK_ID + " does not exist")) + .doNothing() + .when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + awaitUntil(() -> provider.getLock() == null, "the lost lock must be dropped by the heartbeat"); + + // The second attempt only got queued, so it leaves no lock behind and nothing was expired + // by the metastore this time round. + assertFalse(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + verify(client).unlock(OTHER_LOCK_ID); + + // Releasing must not report the loss that belonged to the previous lock. + assertDoesNotThrow(provider::unlock); + } finally { + provider.close(); + } + } + + @Test + void lockThatWasOnlyQueuedIsNeverHeartbeated() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(waitingLock(LOCK_ID)); + doThrow(new NoSuchLockException("lock " + LOCK_ID + " was never granted")) + .when(client).heartbeat(anyLong(), anyLong()); + // Releasing the queued lock is the provider's very next step, and it is an RPC: parking inside + // it holds open exactly the window in which a heartbeat scheduled for that lock would tick. + doAnswer(invocation -> { + Thread.sleep(HEARTBEAT_INTERVAL_MS * 5); + return null; + }).when(client).unlock(anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertFalse(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + // A lock that was never granted must never be renewed: a tick failing for it would be read + // as the metastore taking away exclusivity that the writer never had in the first place. + verify(client, never()).heartbeat(anyLong(), anyLong()); + assertNull(heartbeatFutureOf(provider), "a queued lock must not be given a heartbeat schedule"); + assertDoesNotThrow(provider::unlock); + } finally { + provider.close(); + } + } + + @Test + void unlockStaysSilentWhenNoLockWasEverHeld() { + IMetaStoreClient client = mock(IMetaStoreClient.class); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + // Releasing a lock that was never acquired is still a no-op: only a lock the metastore took + // away is reported as a failure. + assertDoesNotThrow(provider::unlock); + } finally { + provider.close(); + } + } + + @Test + void staleHeartbeatFromAReleasedLockDoesNotDropTheNextLock() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID), acquiredLock(OTHER_LOCK_ID)); + CountDownLatch staleTickStarted = new CountDownLatch(1); + CountDownLatch releaseStaleTick = new CountDownLatch(1); + CountDownLatch newLockHeartbeats = new CountDownLatch(3); + doAnswer(invocation -> { + long heartbeatedLockId = invocation.getArgument(1); + if (heartbeatedLockId == LOCK_ID) { + // Park this tick inside the RPC until the lock it renews has been released and another one + // taken, then fail it the way the metastore fails a heartbeat for a lock it no longer has. + staleTickStarted.countDown(); + releaseStaleTick.await(); + throw new NoSuchLockException("lock " + LOCK_ID + " does not exist"); + } + newLockHeartbeats.countDown(); + return null; + }).when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + assertTrue(staleTickStarted.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS), + "the heartbeat for the first lock must be in flight before it is released"); + + provider.unlock(); + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + + // Only now does the tick scheduled for the first lock come back, failing because that lock + // was released here. Cancelling its schedule could never have stopped it. + releaseStaleTick.countDown(); + + assertTrue(newLockHeartbeats.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS), + "the stale tick must not stop the renewal of the lock held now, which would let the " + + "metastore expire a perfectly healthy lock mid-commit"); + assertNotNull(provider.getLock(), "the stale tick must not drop the lock held now"); + assertFalse(heartbeatFutureOf(provider).isCancelled(), + "the stale tick must not cancel the schedule of the lock held now"); + + assertDoesNotThrow(provider::unlock); + verify(client).unlock(OTHER_LOCK_ID); + } finally { + // Free the parked tick even when an assertion above failed: close() only shuts the executor + // down, which neither interrupts the await nor lets the non-daemon thread exit, so leaving + // it parked would turn a failing test into a hanging JVM. + releaseStaleTick.countDown(); + provider.close(); + } + } + + @Test + void releasingALockNormallyIsNotReportedAsLost() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + when(client.lock(any())).thenReturn(acquiredLock(LOCK_ID)); + CountDownLatch tickStarted = new CountDownLatch(1); + CountDownLatch releaseTick = new CountDownLatch(1); + doAnswer(invocation -> { + tickStarted.countDown(); + releaseTick.await(); + throw new NoSuchLockException("lock " + LOCK_ID + " does not exist"); + }).when(client).heartbeat(anyLong(), anyLong()); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + assertTrue(provider.acquireLock(1000L, TimeUnit.MILLISECONDS, lockComponent)); + assertTrue(tickStarted.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS), + "the heartbeat must be in flight before the lock is released"); + + provider.unlock(); + verify(client).unlock(LOCK_ID); + releaseTick.countDown(); + + // Let the released tick finish and run whatever it makes of the failure. + Thread.sleep(HEARTBEAT_INTERVAL_MS * 5); + + // The heartbeat failed only because the lock was released here, so releasing again stays the + // no-op it has always been instead of reporting a loss that never happened. + assertDoesNotThrow(provider::unlock); + } finally { + // See the note in staleHeartbeatFromAReleasedLockDoesNotDropTheNextLock: a parked tick must + // never outlive a failed assertion. + releaseTick.countDown(); + provider.close(); + } + } + + private static ScheduledFuture heartbeatFutureOf(HiveMetastoreBasedLockProvider provider) throws Exception { + return readField(provider, "future"); + } + + private static void awaitUntil(BooleanSupplier condition, String message) throws InterruptedException { + long deadline = System.currentTimeMillis() + AWAIT_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return; + } + Thread.sleep(20L); + } + fail(message); + } +} From 64aac8b974ea411669ca1b4371cbb09d4b2b47fe Mon Sep 17 00:00:00 2001 From: Nada Date: Fri, 24 Jul 2026 10:10:19 -0400 Subject: [PATCH 203/255] fix(record-index-bootstrap): sort record index keys by UTF-8 bytes to match HFile sorting (#18941) Adapted for release-1.2.1: - The HoodieInlineLogAppendHandle change was applied to HoodieAppendHandle instead. Master split that handle for native logs (#19067) into HoodieNativeLogAppendHandle / HoodieInlineLogAppendHandle; this branch has no native log format and keeps the single HoodieAppendHandle, which carried the same UTF-16 sort on the HFILE_DATA_BLOCK path. Dropping the file would have left that bug in place, since it is exactly what this fix targets. - Skipped the HoodieRecordUtils#sortRecordsByRecordKey change and its test. That method does not exist here; it arrives with 5e1b89fd490c (#19079, "feat: sort input for lsm write"), which is not backported, and nothing on this branch calls it. Adding it would have introduced dead code. HoodieNativeAvroHFileReader was relocated automatically to this branch's org.apache.hudi.io.storage package (master moved it to core.io.storage in #19195). Every other file applied unchanged. (cherry picked from commit 26b1ebc4e9384d15c5599a4d97d522f781063b35) --- .../org/apache/hudi/io/BaseCreateHandle.java | 7 +- .../apache/hudi/io/HoodieAppendHandle.java | 5 +- .../hudi/io/HoodieSortedMergeHandle.java | 5 +- .../hudi/client/HoodieFlinkWriteClient.java | 5 +- ...vaHoodieMetadataBulkInsertPartitioner.java | 3 +- ...vaHoodieMetadataBulkInsertPartitioner.java | 100 ++++++++++ ...rkHoodieMetadataBulkInsertPartitioner.java | 2 +- .../commit/BaseSparkCommitActionExecutor.java | 5 +- ...rkHoodieMetadataBulkInsertPartitioner.java | 61 ++++++ .../SortedKeyBasedFileGroupRecordBuffer.java | 12 +- .../storage/HoodieNativeAvroHFileReader.java | 8 +- .../metadata/HoodieBackedTableMetadata.java | 15 +- ...stSortedKeyBasedFileGroupRecordBuffer.java | 35 ++++ .../apache/hudi/common/util/StringUtils.java | 36 ++++ .../hudi/common/util/TestStringUtils.java | 64 +++++++ .../functional/TestHoodieBackedMetadata.java | 176 ++++++++++++++++++ .../org/apache/hudi/io/TestMergeHandle.java | 59 ++++++ .../TestSecondaryIndexPruning.scala | 72 +++++++ 18 files changed, 652 insertions(+), 18 deletions(-) create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/metadata/TestJavaHoodieMetadataBulkInsertPartitioner.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java index e24da085a613a..6eb999d74b550 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.model.MetadataValues; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieInsertException; @@ -131,8 +132,10 @@ protected void doWrite(HoodieRecord record, HoodieSchema schema, TypedProperties public void write() { Iterator keyIterator; if (hoodieTable.requireSortedRecords()) { - // Sorting the keys limits the amount of extra memory required for writing sorted records - keyIterator = recordMap.keySet().stream().sorted().iterator(); + // Sorting the keys limits the amount of extra memory required for writing sorted records. + // requireSortedRecords() is true only for HFile base files, which order keys by UTF-8 bytes, + // not String (UTF-16) order, so sort with the matching comparator. + keyIterator = recordMap.keySet().stream().sorted(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR).iterator(); } else { keyIterator = recordMap.keySet().stream().iterator(); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index b179bd3ea5bf2..e76d3dbd77bcd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -54,6 +54,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieAppendException; @@ -727,7 +728,9 @@ protected HoodieLogBlock getDataBlock(HoodieWriteConfig writeConfig, case HFILE_DATA_BLOCK: // Not supporting positions in HFile data blocks header.remove(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS); - records.sort(Comparator.comparing(HoodieRecord::getRecordKey)); + // HFile orders keys by their raw UTF-8 bytes, so sort by UTF-8 bytes rather than + // String (UTF-16) order to keep non-ASCII / binary keys consistent with the writer. + records.sort(Comparator.comparing(HoodieRecord::getRecordKey, StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR)); return new HoodieHFileDataBlock( records, header, writeConfig.getHFileCompressionAlgorithm(), new StoragePath(writeConfig.getBasePath())); case PARQUET_DATA_BLOCK: diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java index 9456d5ce586bb..7cc74c40afeeb 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.keygen.BaseKeyGenerator; @@ -47,7 +48,7 @@ @NotThreadSafe public class HoodieSortedMergeHandle extends HoodieWriteMergeHandle { - private final Queue newRecordKeysSorted = new PriorityQueue<>(); + private final Queue newRecordKeysSorted = new PriorityQueue<>(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); public HoodieSortedMergeHandle(HoodieWriteConfig config, String instantTime, HoodieTable hoodieTable, Iterator> recordItr, String partitionPath, String fileId, TaskContextSupplier taskContextSupplier, @@ -78,7 +79,7 @@ public void write(HoodieRecord oldRecord) { // To maintain overall sorted order across updates and inserts, write any new inserts whose keys are less than // the oldRecord's key. - while (!newRecordKeysSorted.isEmpty() && newRecordKeysSorted.peek().compareTo(key) <= 0) { + while (!newRecordKeysSorted.isEmpty() && StringUtils.compareUtf8Bytes(newRecordKeysSorted.peek(), key) <= 0) { String keyToPreWrite = newRecordKeysSorted.remove(); if (keyToPreWrite.equals(key)) { // will be handled as an update later diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java index 51ef11b390131..df044ed129c6d 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java @@ -35,6 +35,7 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.FlinkHoodieIndexFactory; @@ -337,7 +338,9 @@ public List bulkInsertPreppedRecords(List> preppedR Map>> preppedRecordsByFileId = preppedRecords.stream().parallel() .collect(Collectors.groupingBy(r -> r.getCurrentLocation().getFileId())); return preppedRecordsByFileId.values().stream().parallel().map(records -> { - records.sort(Comparator.comparing(HoodieRecord::getRecordKey)); + // Only used for the metadata table, whose base files are HFiles ordered by raw UTF-8 bytes, + // so sort by UTF-8 bytes rather than String (UTF-16) order for non-ASCII / binary keys. + records.sort(Comparator.comparing(HoodieRecord::getRecordKey, StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR)); HoodieWriteMetadata> result; BucketInfo bucketInfo = new BucketInfo(BucketType.INSERT, records.get(0).getCurrentLocation().getFileId(), records.get(0).getPartitionPath()); try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(bucketInfo, records.iterator(), instantTime, table, true)) { diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieMetadataBulkInsertPartitioner.java index 0d81cc91fcffb..e2a2137a1252f 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieMetadataBulkInsertPartitioner.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieMetadataBulkInsertPartitioner.java @@ -19,6 +19,7 @@ package org.apache.hudi.metadata; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.table.BulkInsertPartitioner; import java.util.Comparator; @@ -39,7 +40,7 @@ public List> repartitionRecords(List> records, i if (records.isEmpty()) { return records; } - records.sort(Comparator.comparing(record -> record.getKey().getRecordKey())); + records.sort(Comparator.comparing(record -> record.getKey().getRecordKey(), StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR)); fileId = HoodieTableMetadataUtil.getFileGroupPrefix(records.get(0).getCurrentLocation().getFileId()); return records; } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/metadata/TestJavaHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/metadata/TestJavaHoodieMetadataBulkInsertPartitioner.java new file mode 100644 index 0000000000000..2f600bac0954f --- /dev/null +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/metadata/TestJavaHoodieMetadataBulkInsertPartitioner.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.metadata; + +import org.apache.hudi.common.model.EmptyHoodieRecordPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.util.StringUtils; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link JavaHoodieMetadataBulkInsertPartitioner}, which sorts MDT/HFile record keys by raw + * UTF-8 bytes rather than String (UTF-16) order. + */ +class TestJavaHoodieMetadataBulkInsertPartitioner { + + @Test + void repartitionRecordsSortsBinaryKeysByUtf8Bytes() { + // U+E000 (UTF-8 lead byte 0xEE) sorts BEFORE U+20000 (UTF-8 lead byte 0xF0) in raw UTF-8 byte + // order, but AFTER it under String.compareTo (UTF-16). This is the pathological pair the + // partitioner's UTF-8 comparator must get right so HFile forward-only seeks stay valid. + String bmpPrivateUse = new String(Character.toChars(0xE000)); + String supplementary = new String(Character.toChars(0x20000)); + // All records share one file group so the partitioner's single-group assumption holds. + String fileId = "files-0000"; + + // Shuffled input mixing both prefixes plus ascii suffixes. + List inputKeys = Arrays.asList( + supplementary + "-b", + "ascii-key", + bmpPrivateUse + "-a", + supplementary + "-a", + bmpPrivateUse + "-b"); + + List> records = new ArrayList<>(); + for (String key : inputKeys) { + HoodieRecord record = + new HoodieAvroRecord<>(new HoodieKey(key, ""), new EmptyHoodieRecordPayload()); + record.unseal(); + record.setCurrentLocation(new HoodieRecordLocation("001", fileId)); + record.seal(); + records.add(record); + } + + JavaHoodieMetadataBulkInsertPartitioner partitioner = + new JavaHoodieMetadataBulkInsertPartitioner<>(); + List> sorted = partitioner.repartitionRecords(records, 1); + + assertTrue(partitioner.arePartitionRecordsSorted(), "Records must be sorted"); + + List actualKeys = new ArrayList<>(); + for (HoodieRecord record : sorted) { + actualKeys.add(record.getRecordKey()); + } + List expectedKeys = new ArrayList<>(inputKeys); + expectedKeys.sort(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + assertEquals(expectedKeys, actualKeys, "Records must be sorted by UTF-8 byte order"); + + // The divergent pair: every U+E000-prefixed key precedes every U+20000-prefixed key in UTF-8 + // byte order, the opposite of String.compareTo (UTF-16) order. + int lastBmpIndex = -1; + int firstSupplementaryIndex = actualKeys.size(); + for (int i = 0; i < actualKeys.size(); i++) { + if (actualKeys.get(i).startsWith(bmpPrivateUse)) { + lastBmpIndex = i; + } else if (actualKeys.get(i).startsWith(supplementary) && firstSupplementaryIndex == actualKeys.size()) { + firstSupplementaryIndex = i; + } + } + assertTrue(lastBmpIndex < firstSupplementaryIndex, + "All U+E000-prefixed keys should sort before U+20000-prefixed keys in UTF-8 order"); + } +} diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java index 3fd5f346ce110..38ff2f4a77c14 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java @@ -68,7 +68,7 @@ public int numPartitions() { @Override public JavaRDD repartitionRecords(JavaRDD records, int outputSparkPartitions) { Comparator> keyComparator = - (Comparator> & Serializable)(t1, t2) -> t1._2.compareTo(t2._2); + (Comparator> & Serializable)(t1, t2) -> StringUtils.compareUtf8Bytes(t1._2, t2._2); // Partition the records by their file group JavaRDD partitionedRDD = records diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java index ed0dcaf321be2..1659c43a2bc19 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java @@ -40,6 +40,7 @@ import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.data.HoodieJavaPairRDD; @@ -326,10 +327,12 @@ protected HoodieData mapPartitionsAsRDD(HoodieData> if (table.requireSortedRecords()) { // Partition and sort within each partition as a single step. This is faster than partitioning first and then // applying a sort. + // requireSortedRecords() is true only for HFile base files, which order keys by UTF-8 bytes, + // not String (UTF-16) order, so sort with the matching comparator. Comparator>> comparator = (Comparator>> & Serializable) (t1, t2) -> { HoodieKey key1 = t1._1; HoodieKey key2 = t2._1; - return key1.getRecordKey().compareTo(key2.getRecordKey()); + return StringUtils.compareUtf8Bytes(key1.getRecordKey(), key2.getRecordKey()); }; partitionedRDD = mappedRDD.repartitionAndSortWithinPartitions(partitioner, comparator); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java index aa46a177ac4ea..0a8b7918d8503 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java @@ -19,8 +19,10 @@ package org.apache.hudi.client; +import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.metadata.DefaultMetadataTableFileGroupIndexParser; import org.apache.hudi.metadata.HoodieMetadataPayload; import org.apache.hudi.metadata.MetadataPartitionType; @@ -31,6 +33,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -108,4 +111,62 @@ public void testPartitioner() { Set fileIDPrefixes = IntStream.of(0, 1, 2, 4).mapToObj(partitioner::getFileIdPfx).collect(Collectors.toSet()); assertEquals(fileIDPrefixes, recordsPerFileGroup.keySet(), "fileIDPrefixes should match the name of the MDT fileGroups"); } + + @Test + public void testPartitionerSortsBinaryKeysByUtf8Bytes() { + // U+E000 (UTF-8 lead byte 0xEE) sorts BEFORE U+20000 (UTF-8 lead byte 0xF0) in raw UTF-8 byte + // order, but AFTER it under String.compareTo (UTF-16). All records target a single MDT file group + // so the partitioner's only job here is the within-partition UTF-8 sort. + String fileGroupId = MetadataPartitionType.FILES.getFileIdPrefix() + "000"; + String bmpPrivateUse = new String(Character.toChars(0xE000)); + String supplementary = new String(Character.toChars(0x20000)); + + // Shuffled input mixing both prefixes plus an ascii key. + List inputKeys = Arrays.asList( + supplementary + "-b", + "ascii-key", + bmpPrivateUse + "-a", + supplementary + "-a", + bmpPrivateUse + "-b"); + + List records = new ArrayList<>(); + for (String key : inputKeys) { + // createPartitionListRecord fixes the record key, so start from it (for a valid MDT payload) + // and rebind an explicitly chosen HoodieKey via newInstance. + HoodieRecord r = HoodieMetadataPayload.createPartitionListRecord(Collections.EMPTY_LIST) + .newInstance(new HoodieKey(key, "")); + r.unseal(); + r.setCurrentLocation(new HoodieRecordLocation("001", fileGroupId)); + r.seal(); + records.add(r); + } + + SparkHoodieMetadataBulkInsertPartitioner partitioner = + new SparkHoodieMetadataBulkInsertPartitioner(new DefaultMetadataTableFileGroupIndexParser(1)); + JavaRDD partitionedRecords = + partitioner.repartitionRecords(jsc().parallelize(records, records.size()), 0); + + // All records map to one file group, hence a single partition. + assertEquals(1, partitionedRecords.getNumPartitions(), "All records map to a single file group"); + assertTrue(partitioner.arePartitionRecordsSorted(), "Must be sorted"); + + List actualKeys = partitionedRecords.map(r -> r.getRecordKey()).collect(); + List expectedKeys = new ArrayList<>(inputKeys); + expectedKeys.sort(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + assertEquals(expectedKeys, actualKeys, "Records must be sorted by UTF-8 byte order within the file group"); + + // The divergent pair: every U+E000-prefixed key precedes every U+20000-prefixed key in UTF-8 + // byte order, the opposite of String.compareTo (UTF-16) order. + int lastBmpIndex = -1; + int firstSupplementaryIndex = actualKeys.size(); + for (int i = 0; i < actualKeys.size(); i++) { + if (actualKeys.get(i).startsWith(bmpPrivateUse)) { + lastBmpIndex = i; + } else if (actualKeys.get(i).startsWith(supplementary) && firstSupplementaryIndex == actualKeys.size()) { + firstSupplementaryIndex = i; + } + } + assertTrue(lastBmpIndex < firstSupplementaryIndex, + "All U+E000-prefixed keys should sort before U+20000-prefixed keys in UTF-8 order"); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java index 8fa9aa1dad912..083a7762bec27 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.table.read.BufferedRecord; import org.apache.hudi.common.table.read.UpdateProcessor; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import java.io.IOException; @@ -58,14 +59,18 @@ class SortedKeyBasedFileGroupRecordBuffer extends KeyBasedFileGroupRecordBuff @Override protected void initializeLogRecordIterator() { - logRecordIterator = records.values().stream().sorted(Comparator.comparing(BufferedRecord::getRecordKey)).iterator(); + // This buffer is only used when the base file format is HFile (requireSortedRecords()), which orders + // keys by UTF-8 bytes, not String (UTF-16) order, so sort with the matching comparator. + logRecordIterator = records.values().stream() + .sorted(Comparator.comparing(BufferedRecord::getRecordKey, StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR)) + .iterator(); } @Override protected boolean hasNextBaseRecord(T baseRecord) throws IOException { String recordKey = readerContext.getRecordContext().getRecordKey(baseRecord, readerSchema); int comparison = 0; - while (!getLogRecordKeysSorted().isEmpty() && (comparison = getLogRecordKeysSorted().peek().compareTo(recordKey)) <= 0) { + while (!getLogRecordKeysSorted().isEmpty() && (comparison = StringUtils.compareUtf8Bytes(getLogRecordKeysSorted().peek(), recordKey)) <= 0) { String nextLogRecordKey = getLogRecordKeysSorted().poll(); if (comparison == 0) { break; // Log record key matches the base record key, exit loop after removing the key from the queue of log record keys @@ -102,7 +107,8 @@ protected boolean doHasNext() throws IOException { private Queue getLogRecordKeysSorted() { if (logRecordKeysSorted == null) { - logRecordKeysSorted = records.keySet().stream().map(Object::toString).collect(Collectors.toCollection(PriorityQueue::new)); + logRecordKeysSorted = records.keySet().stream().map(Object::toString) + .collect(Collectors.toCollection(() -> new PriorityQueue<>(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR))); } return logRecordKeysSorted; } diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java index 1020d1a8c9f02..4865124c69e19 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.CloseableMappingIterator; @@ -134,8 +135,11 @@ public BloomFilter readBloomFilter() { public Set> filterRowKeys(Set candidateRowKeys) { try (HFileReader reader = readerFactory.createHFileReader()) { reader.seekTo(); - // candidateRowKeys must be sorted - return (candidateRowKeys instanceof TreeSet ? candidateRowKeys : new TreeSet<>(candidateRowKeys)) + // candidateRowKeys must be sorted by UTF-8 bytes to match HFile ordering because the reader + // only seeks forward. + TreeSet sortedRowKeys = new TreeSet<>(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + sortedRowKeys.addAll(candidateRowKeys); + return sortedRowKeys .stream() .filter(k -> { try { diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java index 70b136f2a646d..4f055baeb424f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java @@ -50,6 +50,7 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.ClosableSortedDedupingIterator; @@ -231,9 +232,9 @@ public HoodieData> getRecordsByKeyPrefixes( boolean shouldLoadInMemory) { // Apply key encoding List sortedKeyPrefixes = new ArrayList<>(rawKeys.map(key -> key.encode()).collectAsList()); - // Sort the prefixes so that keys are looked up in order - // Sort must come after encoding. - Collections.sort(sortedKeyPrefixes); + // Sort the prefixes so that keys are looked up in order. Sort must come after encoding. + // Sort by UTF-8 bytes to match the HFile order; the reader seeks forward without rewinding. + sortedKeyPrefixes.sort(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); // NOTE: Since we partition records to a particular file-group by full key, we will have // to scan all file-groups for all key-prefixes as each of these might contain some @@ -258,7 +259,10 @@ public HoodieData> getRecordsByKeyPrefixes( private static TreeSet getDistinctSortedKeysForSingleSlice(HoodieData keys) { List keysList = keys.collectAsList(); - return new TreeSet<>(keysList); + // Order by UTF-8 bytes to match the HFile order used for point lookups. + TreeSet sortedKeys = new TreeSet<>(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + sortedKeys.addAll(keysList); + return sortedKeys; } /** @@ -316,6 +320,9 @@ private HoodieData> lookupIndexRecords(Hoodi } distinctSortedKeyIter.forEachRemaining(keysList::add); } + // The shuffle above repartitions/sorts by String (UTF-16) order, but the HFile reader below + // does a forward-only seek in UTF-8 byte order. Re-sort so the two agree. + keysList.sort(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); FileSlice fileSlice = fileSlices.get(mappingFunction.apply(keysList.get(0), numFileSlices)); return lookupRecordsItr(partitionName, keysList, fileSlice, !isSecondaryIndex); }; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java index 611128a963098..599358c74fc89 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java @@ -56,6 +56,7 @@ import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY; import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; @@ -188,6 +189,40 @@ void readLogFiles() throws IOException { assertEquals(1, readStats.getNumDeletes()); } + @Test + void readBaseFileAndLogFileWithBinaryKeys() throws IOException { + // U+E000 (UTF-8 lead byte 0xEE) sorts BEFORE U+20000 (UTF-8 lead byte 0xF0) in raw UTF-8 byte + // order, but AFTER it under String.compareTo (UTF-16). The base-file record carries the + // U+E000-prefixed (UTF-8-smaller, UTF-16-larger) key and the log carries the U+20000-prefixed + // (UTF-8-larger, UTF-16-smaller) key, so a correct merge must emit them in UTF-8 byte order. + String bmpPrivateUse = new String(Character.toChars(0xE000)); + String supplementary = new String(Character.toChars(0x20000)); + TestRecord asciiA = new TestRecord("a", 0); + TestRecord asciiB = new TestRecord("b", 0); + TestRecord bmpRecord = new TestRecord(bmpPrivateUse + "-base", 0); + TestRecord supplementaryRecord = new TestRecord(supplementary + "-log", 0); + + HoodieReadStats readStats = new HoodieReadStats(); + HoodieReaderContext mockReaderContext = mock(HoodieReaderContext.class, RETURNS_DEEP_STUBS); + SortedKeyBasedFileGroupRecordBuffer fileGroupRecordBuffer = buildSortedKeyBasedFileGroupRecordBuffer(mockReaderContext, readStats); + + // Base-file records must already be in UTF-8 byte order: "a" (0x61) then the U+E000 key (0xEE...). + fileGroupRecordBuffer.setBaseFileIterator(ClosableIterator.wrap(Arrays.asList(asciiA, bmpRecord).iterator())); + + // Log records are supplied shuffled; the buffer sorts them by UTF-8 bytes before merging. + HoodieDataBlock dataBlock = mock(HoodieDataBlock.class); + when(dataBlock.getSchema()).thenReturn(HoodieTestDataGenerator.HOODIE_SCHEMA); + when(dataBlock.getEngineRecordIterator(mockReaderContext)).thenReturn( + ClosableIterator.wrap(Arrays.asList(supplementaryRecord, asciiB).iterator())); + fileGroupRecordBuffer.processDataBlock(dataBlock, Option.empty()); + + List actualRecords = getActualRecordsForSortedKeyBased(fileGroupRecordBuffer); + // Expected UTF-8 byte order: "a", "b", U+E000 key, U+20000 key; nothing is dropped. + assertEquals(Arrays.asList(asciiA, asciiB, bmpRecord, supplementaryRecord), actualRecords); + // The U+E000-prefixed base record precedes the U+20000-prefixed log record (reverse of UTF-16). + assertTrue(actualRecords.indexOf(bmpRecord) < actualRecords.indexOf(supplementaryRecord)); + } + private SortedKeyBasedFileGroupRecordBuffer buildSortedKeyBasedFileGroupRecordBuffer(HoodieReaderContext mockReaderContext, HoodieReadStats readStats) { when(mockReaderContext.getSchemaHandler().getRequiredSchema()).thenReturn(HoodieTestDataGenerator.HOODIE_SCHEMA); when(mockReaderContext.getSchemaHandler().getInternalSchema()).thenReturn(InternalSchema.getEmptyInternalSchema()); diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java index 3c5545a6bf71c..2fdefeadc7931 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java @@ -21,9 +21,11 @@ import javax.annotation.Nullable; +import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -119,6 +121,40 @@ public static byte[] getUTF8Bytes(String str) { return str.getBytes(StandardCharsets.UTF_8); } + /** + * Serializable comparator ordering strings by their unsigned UTF-8 byte representation. See + * {@link #compareUtf8Bytes(String, String)} for the rationale and null-handling contract. + */ + public static final Comparator UTF8_LEXICOGRAPHIC_COMPARATOR = + (Comparator & Serializable) StringUtils::compareUtf8Bytes; + + /** + * Compares two strings by their unsigned UTF-8 byte order, matching the ordering HFiles enforce + * (HBase's {@code CellComparatorImpl}). Unlike {@link String#compareTo(String)} (UTF-16 code unit + * order), this stays consistent with HFile ordering for non-ASCII / binary keys. + * + *

    Neither argument may be {@code null}; like {@link String#compareTo(String)}, a {@code null} + * argument throws {@link NullPointerException}. + * + *

    Assumes well-formed UTF-16 input: {@code String#getBytes(UTF_8)} replaces unpaired surrogates + * with {@code '?'}, so strings differing only in unpaired surrogates compare equal. + * + *

    Note: encodes both strings to UTF-8 on every call; for very large sorts consider + * pre-encoding keys to byte arrays once and comparing those. + */ + public static int compareUtf8Bytes(String s1, String s2) { + byte[] b1 = getUTF8Bytes(s1); + byte[] b2 = getUTF8Bytes(s2); + int len = Math.min(b1.length, b2.length); + for (int i = 0; i < len; i++) { + int cmp = (b1[i] & 0xFF) - (b2[i] & 0xFF); + if (cmp != 0) { + return cmp; + } + } + return b1.length - b2.length; + } + public static String fromUTF8Bytes(byte[] bytes) { return fromUTF8Bytes(bytes, 0, bytes.length); } diff --git a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java index 43a547744baa2..c5791222f1df2 100644 --- a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java +++ b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java @@ -21,12 +21,17 @@ import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -268,4 +273,63 @@ public void testStripEnd() { assertEquals("abc", StringUtils.stripEnd("abc", "")); assertEquals("abc", StringUtils.stripEnd("abcabab", "ab")); } + + @Test + public void testCompareUtf8BytesAsciiMatchesStringCompareTo() { + // Pure ASCII bytes equal their UTF-16 code unit values, so UTF-8 byte order and + // String.compareTo order coincide. + assertEquals(Integer.signum("apple".compareTo("banana")), Integer.signum(StringUtils.compareUtf8Bytes("apple", "banana"))); + assertEquals(Integer.signum("banana".compareTo("apple")), Integer.signum(StringUtils.compareUtf8Bytes("banana", "apple"))); + assertEquals(0, StringUtils.compareUtf8Bytes("apple", "apple")); + } + + @Test + public void testCompareUtf8BytesSupplementaryPairFlipsOrderVsStringCompareTo() { + // U+E000 (BMP private-use, UTF-8 lead byte 0xEE) vs U+20000 (supplementary plane, UTF-8 lead byte + // 0xF0). In UTF-16, U+20000 is encoded as a surrogate pair starting with 0xD840, which is < 0xE000, + // so String.compareTo orders U+20000 first. In UTF-8 byte order 0xF0 > 0xEE, flipping the order -- + // exactly the pathological shape that breaks HFile's forward-only seek under String.compareTo. + String bmpPrivateUse = new String(Character.toChars(0xE000)); + String supplementary = new String(Character.toChars(0x20000)); + + assertTrue(bmpPrivateUse.compareTo(supplementary) > 0, + "String.compareTo should order U+E000 after U+20000 (UTF-16 code unit order)"); + assertTrue(StringUtils.compareUtf8Bytes(bmpPrivateUse, supplementary) < 0, + "compareUtf8Bytes should order U+E000 before U+20000 (UTF-8 byte order)"); + } + + @Test + public void testCompareUtf8BytesEmptyPrefixAndIdenticalStrings() { + assertTrue(StringUtils.compareUtf8Bytes("", "a") < 0); + assertTrue(StringUtils.compareUtf8Bytes("a", "") > 0); + assertEquals(0, StringUtils.compareUtf8Bytes("", "")); + assertTrue(StringUtils.compareUtf8Bytes("ab", "abc") < 0); + assertTrue(StringUtils.compareUtf8Bytes("abc", "ab") > 0); + assertEquals(0, StringUtils.compareUtf8Bytes("abc", "abc")); + } + + @Test + @SuppressWarnings("unchecked") + public void testUtf8LexicographicComparatorSerializableAndRejectsNull() throws Exception { + // Like String.compareTo, a null argument is rejected. + assertThrows(NullPointerException.class, () -> StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR.compare(null, "a")); + + // The comparator is declared as (Comparator & Serializable) so Spark can capture it inside + // serialized closures. Round-trip it through Java serialization and confirm the deserialized + // instance still orders keys by UTF-8 bytes for the divergent U+E000 vs U+20000 pair (U+E000's + // UTF-8 lead byte 0xEE sorts before U+20000's 0xF0, the reverse of String.compareTo / UTF-16). + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + } + Comparator deserialized; + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) { + deserialized = (Comparator) ois.readObject(); + } + + String bmpPrivateUse = new String(Character.toChars(0xE000)); + String supplementary = new String(Character.toChars(0x20000)); + assertTrue(deserialized.compare(bmpPrivateUse, supplementary) < 0, + "Deserialized comparator should order U+E000 before U+20000 (UTF-8 byte order)"); + } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java index b0a4aab2e7899..1110fce8c9ca8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java @@ -40,6 +40,7 @@ import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -162,6 +163,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -1944,6 +1946,180 @@ public void testClusteringWithRecordIndex() throws Exception { testTableOperationsForMetaIndexImpl(writeConfig); } + /** + * Record index bootstrap over binary / non-ASCII record keys must succeed. The RI HFile orders + * keys by their raw UTF-8 bytes, so the bulk-insert partitioner must sort by UTF-8 bytes too; + * sorting by {@link String#compareTo(String)} (UTF-16) lays the HFile entries out of order + * relative to their UTF-8 bytes. + * + *

    The failure is read-side, not write-side: the native {@code HFileWriterImpl.append} does no + * key-order validation, so a mis-sorted HFile is still written successfully. The forward-only + * HFile reader ({@code HFileReaderImpl.seekTo}) then either throws {@code IllegalStateException} + * on a backward seek or silently misses keys, which the read-back count assertion at the end of + * this test catches. + * + *

    {@code riFileGroupCount == 1} covers the single-slice lookup path; {@code riFileGroupCount == 4} + * covers the multi-slice {@code mapGroupsByKey} lookup path in + * {@code HoodieBackedTableMetadata#lookupIndexRecords}, which repartitions keys in String/UTF-16 + * order before doing a forward-only HFile seek in UTF-8 order. + */ + @ParameterizedTest + @ValueSource(ints = {1, 4}) + public void testRecordIndexBootstrapWithBinaryRecordKeys(int riFileGroupCount) throws Exception { + init(COPY_ON_WRITE, true); + HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc); + + // First commit with the record index disabled: write base files with binary record keys. + List records = generateRecordsWithBinaryKeys(WriteClientTestUtils.createNewInstantTime(), 0, 200); + HoodieWriteConfig firstConfig = getWriteConfigBuilder(true, true, false).build(); + String firstCommitTime = WriteClientTestUtils.createNewInstantTime(); + try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, firstConfig)) { + WriteClientTestUtils.startCommitWithTime(client, firstCommitTime); + List writeStatuses = client.insert(jsc.parallelize(records, 1), firstCommitTime).collect(); + assertNoWriteErrors(writeStatuses); + client.commit(firstCommitTime, jsc.parallelize(writeStatuses)); + } + metaClient = HoodieTableMetaClient.reload(metaClient); + assertFalse(metaClient.getTableConfig().isMetadataPartitionAvailable(RECORD_INDEX)); + + // Enable the record index. The next commit triggers the bootstrap, reading the binary keys from + // the base files above. One file group puts all keys in a single HFile; more than one file group + // exercises the multi-slice lookup path on read-back. + HoodieWriteConfig riConfig = getWriteConfigBuilder(false, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withEnableGlobalRecordLevelIndex(true) + .withRecordIndexFileGroupCount(riFileGroupCount, riFileGroupCount) + .build()) + .build(); + + String secondCommitTime = WriteClientTestUtils.createNewInstantTime(); + // Disjoint key range so the bootstrapped keys are not mutated. + List secondBatch = generateRecordsWithBinaryKeys(secondCommitTime, 1000, 20); + try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, riConfig)) { + WriteClientTestUtils.startCommitWithTime(client, secondCommitTime); + // Without the fix the mis-sorted record-index HFile is still written; the failure surfaces on + // read-back below, so the write itself is expected to succeed here. + List writeStatuses = client.insert(jsc.parallelize(secondBatch, 1), secondCommitTime).collect(); + assertNoWriteErrors(writeStatuses); + client.commit(secondCommitTime, jsc.parallelize(writeStatuses)); + } + + // The record index partition should exist and resolve every key: the bootstrapped keys live in + // the record-index base HFiles, while the second-batch keys still sit in un-compacted metadata + // log files at this point, so the lookup covers the log-side seek path with binary keys too. + metaClient = HoodieTableMetaClient.reload(metaClient); + assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(RECORD_INDEX)); + HoodieTableMetadata metadataReader = metaClient.getTableFormat().getMetadataFactory().create( + context, storage, riConfig.getMetadataConfig(), riConfig.getBasePath()); + List allKeys = Stream.concat(records.stream(), secondBatch.stream()) + .map(HoodieRecord::getRecordKey).collect(Collectors.toList()); + // With more than one file group, readRecordIndexLocationsWithKeys triggers the mapGroupsByKey + // multi-slice path. + HoodiePairData recordIndexData = metadataReader + .readRecordIndexLocationsWithKeys(HoodieListData.eager(allKeys)); + try { + Map result = HoodieDataUtils.dedupeAndCollectAsMap(recordIndexData); + assertEquals(allKeys.size(), result.size(), + "Record index should resolve every binary key, bootstrapped or still in a metadata log file."); + } finally { + recordIndexData.unpersistWithDependencies(); + } + } + + /** + * Generates {@code count} records with binary record keys interleaving U+E000 (BMP) and U+20000 + * (supplementary) prefixes, whose UTF-16 char order is the reverse of their UTF-8 byte order. + */ + private List generateRecordsWithBinaryKeys(String commitTime, int startIndex, int count) { + List baseRecords = dataGen.generateInserts(commitTime, count); + String[] binaryPrefixes = {new String(Character.toChars(0xE000)), new String(Character.toChars(0x20000))}; + List binaryRecords = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + HoodieRecord baseRecord = baseRecords.get(i); + int index = startIndex + i; + String binaryKey = binaryPrefixes[index % binaryPrefixes.length] + String.format("%08d", index); + binaryRecords.add(new HoodieAvroIndexedRecord( + new HoodieKey(binaryKey, baseRecord.getPartitionPath()), + (IndexedRecord) baseRecord.getData())); + } + return binaryRecords; + } + + /** + * Same as {@link #testRecordIndexBootstrapWithBinaryRecordKeys(int)} but forces an MDT compaction after + * bootstrap, exercising the {@code BaseCreateHandle} / {@code SortedKeyBasedFileGroupRecordBuffer} + * sort-order paths hit when the record-index base HFile is rewritten. + */ + @Test + public void testRecordIndexBootstrapWithBinaryRecordKeysAfterCompaction() throws Exception { + init(COPY_ON_WRITE, true); + HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc); + + List records = generateRecordsWithBinaryKeys(WriteClientTestUtils.createNewInstantTime(), 0, 200); + HoodieWriteConfig firstConfig = getWriteConfigBuilder(true, true, false).build(); + String firstCommitTime = WriteClientTestUtils.createNewInstantTime(); + try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, firstConfig)) { + WriteClientTestUtils.startCommitWithTime(client, firstCommitTime); + List writeStatuses = client.insert(jsc.parallelize(records, 1), firstCommitTime).collect(); + assertNoWriteErrors(writeStatuses); + client.commit(firstCommitTime, jsc.parallelize(writeStatuses)); + } + metaClient = HoodieTableMetaClient.reload(metaClient); + assertFalse(metaClient.getTableConfig().isMetadataPartitionAvailable(RECORD_INDEX)); + + // A single delta commit is enough to trigger compaction on the very next delta commit. + HoodieWriteConfig riConfig = getWriteConfigBuilder(false, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withEnableGlobalRecordLevelIndex(true) + .withRecordIndexFileGroupCount(1, 1) + .withMaxNumDeltaCommitsBeforeCompaction(1) + .build()) + .build(); + + List allKeys = new ArrayList<>(records); + try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, riConfig)) { + // Bootstrap: writes the initial record-index HFile from the binary keys above. + String secondCommitTime = WriteClientTestUtils.createNewInstantTime(); + List secondBatch = generateRecordsWithBinaryKeys(secondCommitTime, 1000, 20); + WriteClientTestUtils.startCommitWithTime(client, secondCommitTime); + // The mis-sorted bootstrap write succeeds; key ordering is validated by the read-back below. + List secondWriteStatuses = client.insert(jsc.parallelize(secondBatch, 1), secondCommitTime).collect(); + assertNoWriteErrors(secondWriteStatuses); + client.commit(secondCommitTime, jsc.parallelize(secondWriteStatuses)); + allKeys.addAll(secondBatch); + + // The next delta commit on the record index partition triggers compaction, rewriting the base + // HFile via BaseCreateHandle / SortedKeyBasedFileGroupRecordBuffer. + String thirdCommitTime = WriteClientTestUtils.createNewInstantTime(); + List thirdBatch = generateRecordsWithBinaryKeys(thirdCommitTime, 2000, 20); + WriteClientTestUtils.startCommitWithTime(client, thirdCommitTime); + // The compaction rewrite of the base HFile also succeeds; ordering is validated on read-back. + List thirdWriteStatuses = client.insert(jsc.parallelize(thirdBatch, 1), thirdCommitTime).collect(); + assertNoWriteErrors(thirdWriteStatuses); + client.commit(thirdCommitTime, jsc.parallelize(thirdWriteStatuses)); + allKeys.addAll(thirdBatch); + } + + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieTableMetadata metadataReader = metaClient.getTableFormat().getMetadataFactory().create( + context, storage, riConfig.getMetadataConfig(), riConfig.getBasePath()); + assertTrue(metadataReader.getLatestCompactionTime().isPresent(), + "Record index partition should have been compacted by now."); + + List allRecordKeys = allKeys.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()); + HoodiePairData recordIndexData = metadataReader + .readRecordIndexLocationsWithKeys(HoodieListData.eager(allRecordKeys)); + try { + Map result = HoodieDataUtils.dedupeAndCollectAsMap(recordIndexData); + assertEquals(allRecordKeys.size(), result.size(), + "Record index should resolve every binary key after the record index partition has been compacted."); + } finally { + recordIndexData.unpersistWithDependencies(); + } + } + /** * First attempt at bootstrap failed but the file slices get created. The next bootstrap should continue successfully. */ diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java index 95a5d64cf0fb2..ee1c00323cfb2 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java @@ -58,6 +58,7 @@ import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; @@ -79,6 +80,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -203,6 +205,57 @@ public void testMergeHandleRLIAndSIStatsWithUpdatesAndDeletes(boolean useFileGro validateSecondaryIndexStatsContent(writeStatus, numUpdates, numDeletes); } + @Test + public void testSortedMergeHandleWritesBinaryKeysInUtf8Order() throws Exception { + // Drives HoodieSortedMergeHandle directly against a Parquet base file (requireSortedRecords() is + // false), validating comparator ordering only; does not cover the HoodieMergeHandleFactory + // selection path for HFILE base-format tables. + // delete and recreate + metaClient.getStorage().deleteDirectory(metaClient.getBasePath()); + Properties properties = new Properties(); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path"); + properties.put(HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key(), ORDERING_FIELD); + initMetaClient(getTableType(), properties); + + HoodieWriteConfig config = getHoodieWriteConfigBuilder().build(); + HoodieSparkTable.create(config, new HoodieLocalEngineContext(storageConf), metaClient); + + String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[0]; + HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator(new String[] {partitionPath}); + + // These two keys sort in opposite order under UTF-16 (String#compareTo) vs UTF-8 bytes (HFile/MDT order). + String supplementaryKey = "😀_record"; + String bmpHighKey = new String(Character.toChars(0xFFFD)) + "_record"; + assertTrue(supplementaryKey.compareTo(bmpHighKey) < 0); + assertTrue(StringUtils.compareUtf8Bytes(bmpHighKey, supplementaryKey) < 0); + + // Base file has the UTF-8-smaller key. + List baseRecords = withRowKey(dataGenerator.generateInserts("000", 1), bmpHighKey, partitionPath); + SparkRDDWriteClient client = getHoodieWriteClient(config); + String instantTime = client.startCommit(); + JavaRDD statuses = client.upsert(jsc.parallelize(baseRecords, 1), instantTime); + client.commit(instantTime, statuses, Option.empty(), COMMIT_ACTION, Collections.emptyMap(), Option.empty()); + + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieSparkCopyOnWriteTable table = (HoodieSparkCopyOnWriteTable) HoodieSparkCopyOnWriteTable.create(config, context, metaClient); + HoodieFileGroup fileGroup = table.getFileSystemView().getAllFileGroups(partitionPath).collect(Collectors.toList()).get(0); + String fileId = fileGroup.getFileGroupId().getFileId(); + + // Merge the UTF-8-larger key in directly via HoodieSortedMergeHandle. + List newRecords = withRowKey(dataGenerator.generateInserts("001", 1), supplementaryKey, partitionPath); + HoodieSortedMergeHandle mergeHandle = new HoodieSortedMergeHandle( + config, "001", table, newRecords.iterator(), partitionPath, fileId, new LocalTaskContextSupplier(), Option.empty()); + mergeHandle.doMerge(); + WriteStatus writeStatus = (WriteStatus) mergeHandle.close().get(0); + + String fullPath = metaClient.getBasePath() + "/" + writeStatus.getStat().getPath(); + List actualRecords = new ParquetUtils().readAvroRecords(metaClient.getStorage(), new StoragePath(fullPath)); + List actualKeysInOrder = actualRecords.stream().map(r -> r.get("_row_key").toString()).collect(Collectors.toList()); + // bmpHighKey must come first when sorted by UTF-8 bytes. + assertEquals(Arrays.asList(bmpHighKey, supplementaryKey), actualKeysInOrder); + } + @Test void testWriteFailures() throws Exception { // delete and recreate @@ -600,6 +653,12 @@ private List getHoodieRecords(String payloadClass, List withRowKey(List records, String rowKey, String partitionPath) { + GenericRecord genericRecord = (GenericRecord) ((SerializableIndexedRecord) records.get(0).getData()).getData(); + genericRecord.put("_row_key", rowKey); + return getHoodieRecords(OverwriteWithLatestAvroPayload.class.getName(), Collections.singletonList(genericRecord), partitionPath, false); + } + private void setCurLocation(List records, String fileId, String instantTime) { records.forEach(record -> record.setCurrentLocation(new HoodieRecordLocation(instantTime, fileId))); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala index 2f91b6e98b393..fb86709baa89f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala @@ -230,6 +230,78 @@ class TestSecondaryIndexPruning extends SparkClientFunctionalTestHarnessScala { } } + @Test + def testSecondaryIndexWithNonAsciiSecondaryKeyValues(): Unit = { + var hudiOpts = commonOpts + hudiOpts = hudiOpts ++ Map( + DataSourceWriteOptions.TABLE_TYPE.key -> COW_TABLE_TYPE_OPT_VAL, + DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true") + tableName += "test_secondary_index_non_ascii_partitioned_cow" + + // These two secondary values have UTF-16 order reversed vs their raw UTF-8 byte order: + // U+E000 encodes to bytes EE 80 80 while U+20000 encodes to F0 A0 80 80, so U+E000 sorts + // before U+20000 by UTF-8 bytes; but in UTF-16 the U+20000 surrogate pair (D840 DC00) + // sorts before the single U+E000 code unit. The fix orders metadata keys by UTF-8 bytes, + // matching HFile, so both lookups must still resolve to the correct row. + val bmpVal = new String(Character.toChars(0xE000)) + "acme" + val astralVal = new String(Character.toChars(0x20000)) + "acme" + val asciiVal = "acme" + + spark.sql( + s""" + |create table $tableName ( + | ts bigint, + | record_key_col string, + | not_record_key_col string, + | partition_key_col string + |) using hudi + | options ( + | primaryKey ='record_key_col', + | type = 'cow', + | hoodie.metadata.enable = 'true', + | hoodie.metadata.record.index.enable = 'true', + | hoodie.datasource.write.recordkey.field = 'record_key_col', + | hoodie.enable.data.skipping = 'true', + | hoodie.datasource.write.payload.class = "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload" + | ) + | partitioned by(partition_key_col) + | location '$basePath' + """.stripMargin) + // small file limit 0 so each insert lands in its own file, giving data skipping something to prune + withSQLConf("hoodie.parquet.small.file.limit" -> "0") { + spark.sql(s"insert into $tableName values(1, 'row1', '$bmpVal', 'p1')") + spark.sql(s"insert into $tableName values(2, 'row2', '$astralVal', 'p2')") + spark.sql(s"insert into $tableName values(3, 'row3', '$asciiVal', 'p3')") + // create secondary index on the column holding the non-ascii values + spark.sql(s"create index idx_not_record_key_col on $tableName (not_record_key_col)") + metaClient = HoodieTableMetaClient.builder() + .setBasePath(basePath) + .setConf(HoodieTestUtils.getDefaultStorageConf) + .build() + assert(metaClient.getTableConfig.getMetadataPartitions.contains("secondary_index_idx_not_record_key_col")) + // non-ascii secondary values must be preserved verbatim in the secondary index records + checkAnswer(s"select key from hudi_metadata('$basePath') where type=7")( + Seq(bmpVal + SECONDARY_INDEX_RECORD_KEY_SEPARATOR + "row1"), + Seq(astralVal + SECONDARY_INDEX_RECORD_KEY_SEPARATOR + "row2"), + Seq(asciiVal + SECONDARY_INDEX_RECORD_KEY_SEPARATOR + "row3") + ) + withSQLConf("hoodie.metadata.enable" -> "true", + "hoodie.enable.data.skipping" -> "true", + "hoodie.fileIndex.dataSkippingFailureMode" -> "strict") { + // each non-ascii equality predicate must resolve to exactly its own row via the SI prefix lookup + checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName where not_record_key_col = '$bmpVal'")( + Seq(1, "row1", bmpVal, "p1") + ) + checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName where not_record_key_col = '$astralVal'")( + Seq(2, "row2", astralVal, "p2") + ) + // data skipping must prune files using the non-ascii secondary keys + verifyFilePruning(hudiOpts, EqualTo(attribute("not_record_key_col"), Literal(bmpVal))) + verifyFilePruning(hudiOpts, EqualTo(attribute("not_record_key_col"), Literal(astralVal))) + } + } + } + @Test def testCreateAndDropSecondaryIndex(): Unit = { var hudiOpts = commonOpts From ef9e70ea77d7ac130d6195ed4ab7e8c05aed838a Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 17 Jun 2026 11:21:45 +0800 Subject: [PATCH 204/255] perf(flink): Parse bucket index hash-field config once instead of per record (#18993) Adapted for release-1.2.1: - Dropped BucketIndexPartitionerFactory, BucketIndexRemotePartitioner and their two tests. They arrive with 5182bf851569 (#18897, "feat(flink): support remote partitioner for simple bucket index"), which is not backported. - Pipelines: kept this branch's direct `new BucketIndexPartitioner<>(...)` construction instead of the absent factory, and applied the hoisting the commit is actually about (getIndexKeyFields parsed once, NumBucketsFunction built once and captured by the per-record map closure). - Also converted the second BucketIndexPartitioner call site (the SIMPLE bucket-engine branch in the append pipeline) to pass List. #18993 does not touch that site because on master both sites go through the factory, so only the factory needed updating. Here both construct the partitioner directly, and this commit changes its constructor from String to List, so the second site would otherwise not compile. Same parse, same value, different type. (cherry picked from commit 059649c51fbbfeca809825548efa486d5ef80bdb) --- .../hudi/configuration/OptionsResolver.java | 7 +++++++ .../sink/bucket/BucketBulkInsertWriterHelper.java | 15 +++++++-------- .../sink/bucket/BucketStreamWriteFunction.java | 9 ++++++--- .../sink/partitioner/BucketIndexPartitioner.java | 12 ++++++++---- .../org/apache/hudi/sink/utils/Pipelines.java | 14 ++++++++++---- .../sink/utils/BulkInsertFunctionWrapper.java | 7 +++++-- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java index 42cdad7032c02..5956c185f24e0 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java @@ -542,6 +542,13 @@ public static String getIndexKeyField(Configuration conf) { return conf.getString(FlinkOptions.INDEX_KEY_FIELD.key(), getRecordKeyStr(conf)); } + /** + * Returns the index key fields as a list, parsing the comma-separated config value once. + */ + public static List getIndexKeyFields(Configuration conf) { + return KeyGenUtils.getIndexKeyFields(getIndexKeyField(conf)); + } + /** * Returns the conflict resolution strategy. */ diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java index 37d8b8294c09d..34b4afd85457d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketBulkInsertWriterHelper.java @@ -19,7 +19,6 @@ package org.apache.hudi.sink.bucket; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.partition.NumBucketsFunction; import org.apache.hudi.io.storage.row.HoodieRowDataCreateHandle; @@ -38,6 +37,7 @@ import org.apache.flink.table.types.logical.RowType; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -93,20 +93,19 @@ public static SortOperatorGen getFileIdSorterGen(RowType rowType) { return new SortOperatorGen(rowType, new String[] {FILE_GROUP_META_FIELD}); } - private static String getFileId(Map bucketIdToFileId, RowDataKeyGen keyGen, RowData record, String indexKeys, Configuration conf, boolean needFixedFileIdSuffix) { + private static String getFileId(Map bucketIdToFileId, RowDataKeyGen keyGen, RowData record, List indexKeyFields, + NumBucketsFunction numBucketsFunction, boolean needFixedFileIdSuffix) { String recordKey = keyGen.getRecordKey(record); String partition = keyGen.getPartitionPath(record); - NumBucketsFunction numBucketsFunction = new NumBucketsFunction(conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS), conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_RULE), - conf.get(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS)); - final int numBuckets = numBucketsFunction.getNumBuckets(partition); - final int bucketNum = BucketIdentifier.getBucketId(recordKey, indexKeys, numBuckets); + final int bucketNum = BucketIdentifier.getBucketId(recordKey, indexKeyFields, numBuckets); String bucketId = partition + bucketNum; return bucketIdToFileId.computeIfAbsent(bucketId, k -> needFixedFileIdSuffix ? BucketIdentifier.newBucketFileIdForNBCC(bucketNum) : BucketIdentifier.newBucketFileIdPrefix(bucketNum)); } - public static RowData rowWithFileId(Map bucketIdToFileId, RowDataKeyGen keyGen, RowData record, String indexKeys, Configuration conf, boolean needFixedFileIdSuffix) { - final String fileId = getFileId(bucketIdToFileId, keyGen, record, indexKeys, conf, needFixedFileIdSuffix); + public static RowData rowWithFileId(Map bucketIdToFileId, RowDataKeyGen keyGen, RowData record, List indexKeyFields, + NumBucketsFunction numBucketsFunction, boolean needFixedFileIdSuffix) { + final String fileId = getFileId(bucketIdToFileId, keyGen, record, indexKeyFields, numBucketsFunction, needFixedFileIdSuffix); return GenericRowData.of(StringData.fromString(fileId), record); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java index 6763400b02c39..0c732f6507a31 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -54,7 +55,9 @@ public class BucketStreamWriteFunction extends StreamWriteFunction { private int parallelism; - private String indexKeyFields; + // parsed once in open(); the per-record defineRecordLocation path uses the List overload of + // getBucketId so the comma-separated config string is not re-split per record + private List indexKeyFieldList; private boolean isNonBlockingConcurrencyControl; @@ -97,7 +100,7 @@ public BucketStreamWriteFunction(Configuration config, RowType rowType) { @Override public void open(Configuration parameters) throws IOException { super.open(parameters); - this.indexKeyFields = OptionsResolver.getIndexKeyField(config); + this.indexKeyFieldList = OptionsResolver.getIndexKeyFields(config); this.isNonBlockingConcurrencyControl = OptionsResolver.isNonBlockingConcurrencyControl(config); this.taskID = RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()); this.parallelism = RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()); @@ -135,7 +138,7 @@ private void defineRecordLocation(HoodieFlinkInternalRow record) { bootstrapIndexIfNeed(partition); } Map bucketToFileId = bucketIndex.computeIfAbsent(partition, p -> new HashMap<>()); - final int bucketNum = BucketIdentifier.getBucketId(record.getRecordKey(), indexKeyFields, numBucketsFunction.getNumBuckets(record.getPartitionPath())); + final int bucketNum = BucketIdentifier.getBucketId(record.getRecordKey(), indexKeyFieldList, numBucketsFunction.getNumBuckets(record.getPartitionPath())); final String bucketId = partition + "/" + bucketNum; if (incBucketIndex.contains(bucketId)) { diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java index fb5b3fdb6f0f6..d63ebbbed86ab 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java @@ -28,6 +28,8 @@ import org.apache.flink.api.common.functions.Partitioner; import org.apache.flink.configuration.Configuration; +import java.util.List; + /** * Bucket index input partitioner. * The fields to hash can be a subset of the primary key fields. @@ -36,13 +38,15 @@ */ public class BucketIndexPartitioner implements Partitioner { - private final String indexKeyFields; + // Index key fields, pre-parsed by the caller. The per-record partition() path uses the List + // overload of getBucketId so the comma-separated config string is never re-split per record. + private final List indexKeyFieldList; private final NumBucketsFunction numBucketsFunction; private Functions.Function3 partitionIndexFunc; - public BucketIndexPartitioner(Configuration conf, String indexKeyFields) { - this.indexKeyFields = indexKeyFields; + public BucketIndexPartitioner(Configuration conf, List indexKeyFieldList) { + this.indexKeyFieldList = indexKeyFieldList; this.numBucketsFunction = new NumBucketsFunction(conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS), conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_RULE), conf.get(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS)); } @@ -53,7 +57,7 @@ public int partition(HoodieKey key, int numPartitions) { this.partitionIndexFunc = BucketIndexUtil.getPartitionIndexFunc(numPartitions); } int numBuckets = numBucketsFunction.getNumBuckets(key.getPartitionPath()); - int curBucket = BucketIdentifier.getBucketId(key.getRecordKey(), indexKeyFields, numBuckets); + int curBucket = BucketIdentifier.getBucketId(key.getRecordKey(), indexKeyFieldList, numBuckets); return this.partitionIndexFunc.apply(numBuckets, key.getPartitionPath(), curBucket); } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java index 8d39fae75092d..2e9da8acf2aa4 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/Pipelines.java @@ -28,6 +28,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.index.bucket.partition.NumBucketsFunction; import org.apache.hudi.sink.CleanFunction; import org.apache.hudi.sink.StreamWriteOperator; import org.apache.hudi.sink.append.AppendWriteFunctions; @@ -84,6 +85,7 @@ import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; @@ -138,8 +140,12 @@ public static DataStream bulkInsert(Configuration conf, RowType rowType throw new HoodieException( "Consistent hashing bucket index does not work with bulk insert using FLINK engine. Use simple bucket index or Spark engine."); } - String indexKeys = OptionsResolver.getIndexKeyField(conf); - BucketIndexPartitioner partitioner = new BucketIndexPartitioner<>(conf, indexKeys); + List indexKeyFieldList = OptionsResolver.getIndexKeyFields(conf); + // built once and captured by the per-record map closure (NumBucketsFunction is Serializable), + // avoiding a per-record rebuild from conf inside BucketBulkInsertWriterHelper + NumBucketsFunction numBucketsFunction = new NumBucketsFunction(conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS), + conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_RULE), conf.get(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS)); + BucketIndexPartitioner partitioner = new BucketIndexPartitioner<>(conf, indexKeyFieldList); RowDataKeyGen keyGen = RowDataKeyGens.instance(conf, rowType); RowType rowTypeWithFileId = BucketBulkInsertWriterHelper.rowTypeWithFileId(rowType); InternalTypeInfo typeInfo = InternalTypeInfo.of(rowTypeWithFileId); @@ -147,7 +153,7 @@ public static DataStream bulkInsert(Configuration conf, RowType rowType Map bucketIdToFileId = new HashMap<>(); dataStream = dataStream.partitionCustom(partitioner, keyGen::getHoodieKey) - .map(record -> BucketBulkInsertWriterHelper.rowWithFileId(bucketIdToFileId, keyGen, record, indexKeys, conf, needFixedFileIdSuffix), typeInfo) + .map(record -> BucketBulkInsertWriterHelper.rowWithFileId(bucketIdToFileId, keyGen, record, indexKeyFieldList, numBucketsFunction, needFixedFileIdSuffix), typeInfo) .setParallelism(PARALLELISM_VALUE); if (conf.get(FlinkOptions.WRITE_BULK_INSERT_SORT_INPUT)) { SortOperatorGen sortOperatorGen = BucketBulkInsertWriterHelper.getFileIdSorterGen(rowTypeWithFileId); @@ -369,7 +375,7 @@ public static DataStream hoodieStreamWrite(Configuration conf, HoodieIndex.BucketIndexEngineType bucketIndexEngineType = OptionsResolver.getBucketEngineType(conf); switch (bucketIndexEngineType) { case SIMPLE: - String indexKeyFields = OptionsResolver.getIndexKeyField(conf); + List indexKeyFields = OptionsResolver.getIndexKeyFields(conf); // [HUDI-9036] BucketIndexPartitioner is also used in bulk insert mode, // keep use of HoodieKey here in partitionCustom for now BucketIndexPartitioner partitioner = new BucketIndexPartitioner<>(conf, indexKeyFields); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java index 159ead016fb5d..1d5cc2fc9a97e 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BulkInsertFunctionWrapper.java @@ -24,6 +24,7 @@ import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.configuration.OptionsResolver; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.index.bucket.partition.NumBucketsFunction; import org.apache.hudi.sink.StreamWriteOperatorCoordinator; import org.apache.hudi.sink.bucket.BucketBulkInsertWriterHelper; import org.apache.hudi.sink.bulk.BulkInsertWriteFunction; @@ -213,10 +214,12 @@ private void setupWriteFunction() throws Exception { private void setupMapFunction() { RowDataKeyGen keyGen = RowDataKeyGens.instance(conf, rowType); - String indexKeys = OptionsResolver.getIndexKeyField(conf); + List indexKeyFieldList = OptionsResolver.getIndexKeyFields(conf); + NumBucketsFunction numBucketsFunction = new NumBucketsFunction(conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS), + conf.get(FlinkOptions.BUCKET_INDEX_PARTITION_RULE), conf.get(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS)); boolean needFixedFileIdSuffix = OptionsResolver.isNonBlockingConcurrencyControl(conf); this.bucketIdToFileId = new HashMap<>(); - this.mapFunction = r -> BucketBulkInsertWriterHelper.rowWithFileId(bucketIdToFileId, keyGen, r, indexKeys, conf, needFixedFileIdSuffix); + this.mapFunction = r -> BucketBulkInsertWriterHelper.rowWithFileId(bucketIdToFileId, keyGen, r, indexKeyFieldList, numBucketsFunction, needFixedFileIdSuffix); } private void setupSortOperator() throws Exception { From b5a36f7c58878d6f192876ff36a5034aa2265612 Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 16:03:36 +0800 Subject: [PATCH 205/255] fix(build): use log4j-slf4j2-impl in hudi-flink1.17.x 12d25eb2bef8 (#18177, "align log4j2 and slf4j versions") moved the root pom's dependencyManagement from log4j-slf4j-impl to log4j-slf4j2-impl and bumped slf4j 1.7.36 -> 2.0.7. Every module upstream carries was updated with it. hudi-flink1.17.x does not exist on master, so it kept the old artifactId, which is no longer managed. Building with -Dflink1.17 failed at model resolution with "'dependencies.dependency.version' for org.apache.logging.log4j:log4j-slf4j-impl:jar is missing". Match the sibling version modules (1.18.x and later all declare log4j-slf4j2-impl at the same position). --- hudi-flink-datasource/hudi-flink1.17.x/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-flink-datasource/hudi-flink1.17.x/pom.xml b/hudi-flink-datasource/hudi-flink1.17.x/pom.xml index d7c8eef3d7c91..a4d2ae96f2977 100644 --- a/hudi-flink-datasource/hudi-flink1.17.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.17.x/pom.xml @@ -40,7 +40,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j From 36fc3a27b48e2342e7725c7de0b31774d8afbb22 Mon Sep 17 00:00:00 2001 From: Lin Liu <141371752+linliu-code@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:50:50 -0700 Subject: [PATCH 206/255] fix: Fix lock expiration metric (#18492) * Fix metric * Address review: read the clock once when reporting the renewed lock deadline The SUCCESS branch of renewLock called getCurrentEpochMs() three times, so the value fed to updateLockExpirationDeadlineMetric and the two values in the log line could each be computed against a different clock read. Capture the clock once and derive both the metric and the log arguments from it. --------- Co-authored-by: voon (cherry picked from commit 42e938dbd2874086c07234faffe278b3be3354c7) --- .../lock/StorageBasedLockProvider.java | 20 ++++++--- .../lock/TestStorageBasedLockProvider.java | 45 ++++++++++++++++++- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageBasedLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageBasedLockProvider.java index 2ba31e36897da..a2c6aec8afa79 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageBasedLockProvider.java @@ -637,16 +637,22 @@ protected synchronized boolean renewLock() { hoodieLockMetrics.ifPresent(HoodieLockMetrics::updateLockThrottledMetric); // Let heartbeat retry later. return true; - case SUCCESS: - // Only positive outcome - this.setLock(currentLock.getRight().get()); - hoodieLockMetrics.ifPresent(metrics -> metrics.updateLockExpirationDeadlineMetric( - (int) (oldExpirationMs - getCurrentEpochMs()))); - logger.info("Owner {}: Lock renewal successful. The renewal completes {} ms before expiration for lock {}.", - ownerId, oldExpirationMs - getCurrentEpochMs(), lockFilePath); + case SUCCESS: { + // Only positive outcome. Source the deadline metric and log from the renewed lock file + // returned by the storage client (same as the acquisition path), not the locally + // computed expiration, so both callers agree on where the deadline comes from. + StorageLockFile renewedLock = currentLock.getRight().get(); + this.setLock(renewedLock); + // Read the clock once so the metric and the log line below report the same deadline. + long renewalCompletionMs = getCurrentEpochMs(); + long remainingLeaseMs = renewedLock.getValidUntilMs() - renewalCompletionMs; + hoodieLockMetrics.ifPresent(metrics -> metrics.updateLockExpirationDeadlineMetric((int) remainingLeaseMs)); + logger.info("Owner {}: Lock renewal successful. The renewal completes {} ms before old expiration. The lock will expire in {} ms for lock {}.", + ownerId, oldExpirationMs - renewalCompletionMs, remainingLeaseMs, lockFilePath); recordAuditOperation(AuditOperationState.RENEW, acquisitionTimestamp); // Let heartbeat continue to renew lock lease again later. return true; + } default: throw new HoodieLockException("Unexpected lock update result: " + currentLock.getLeft()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestStorageBasedLockProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestStorageBasedLockProvider.java index 11ea99df36aa3..26db59f5dc6ac 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestStorageBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/TestStorageBasedLockProvider.java @@ -625,8 +625,49 @@ void testRenewLockSucceeds() { assertTrue(lockProvider.renewLock()); verify(mockLogger).info( - eq("Owner {}: Lock renewal successful. The renewal completes {} ms before expiration for lock {}."), - eq(this.ownerId), anyLong(), eq("gs://bucket/lake/db/tbl-default/.hoodie/.locks/table_lock.json")); + eq("Owner {}: Lock renewal successful. The renewal completes {} ms before old expiration. The lock will expire in {} ms for lock {}."), + eq(this.ownerId), anyLong(), anyLong(), eq("gs://bucket/lake/db/tbl-default/.hoodie/.locks/table_lock.json")); + } + + @Test + void testRenewLockMetricUsesNewLeaseExpiration() { + // Regression test for https://github.com/apache/hudi/issues/18493: after a successful + // renewal the lock.expiration.deadline metric must reflect the remaining time on the newly + // renewed lease, not on the previous (about-to-expire) lease. + TypedProperties props = new TypedProperties(); + props.put(StorageBasedLockConfig.VALIDITY_TIMEOUT_SECONDS.key(), "10"); + props.put(StorageBasedLockConfig.RENEW_INTERVAL_SECS.key(), "1"); + props.put(BASE_PATH.key(), "gs://bucket/lake/db/tbl-default"); + + HoodieLockMetrics mockMetrics = mock(HoodieLockMetrics.class); + try (StorageBasedLockProvider providerWithMetrics = spy(new StorageBasedLockProvider( + ownerId, + props, + (a, b, c) -> mockHeartbeatManager, + (a, b, c) -> mockLockService, + mockLogger, + mockMetrics))) { + + long t0 = 100_000L; + when(providerWithMetrics.getCurrentEpochMs()).thenReturn(t0); + + // The currently held lease is about to expire (only 100 ms left) -- this is what makes the + // old vs new distinction observable: the old lease would have reported ~100 ms. + StorageLockData oldData = new StorageLockData(false, t0 + 100, ownerId); + StorageLockFile oldLockFile = new StorageLockFile(oldData, "v1"); + doReturn(oldLockFile).when(providerWithMetrics).getLock(); + + StorageLockData renewedLockData = new StorageLockData(false, t0 + DEFAULT_LOCK_VALIDITY_MS, ownerId); + StorageLockFile renewedLockFile = new StorageLockFile(renewedLockData, "v2"); + when(mockLockService.tryUpsertLockFile(any(), eq(Option.of(oldLockFile)))) + .thenReturn(Pair.of(LockUpsertResult.SUCCESS, Option.of(renewedLockFile))); + + assertTrue(providerWithMetrics.renewLock()); + + // New lease = t0 + 10000ms validity, evaluated at t0 -> 10000 ms remaining (the fix), + // not the ~100 ms remaining on the old lease (the bug). + verify(mockMetrics).updateLockExpirationDeadlineMetric(DEFAULT_LOCK_VALIDITY_MS); + } } @Test From 74d1097335d46860ceeed6e2b6e06b0f968dfee8 Mon Sep 17 00:00:00 2001 From: voonhous Date: Mon, 27 Jul 2026 00:02:04 +0800 Subject: [PATCH 207/255] feat(trino): Migrate the Trino-Hudi connector into the Hudi repo (RFC-105) (#18837) Adapted for release-1.2.1: - validate_staged_bundles.sh: dropped hudi-trino-bundle as upstream does, but did not add hudi-spark4.2-bundle_2.13; this branch has no Spark 4.2 module. - The docker trinobase/trinocoordinator/trinoworker poms and packaging/hudi-trino-bundle differed from master's pre-image only by 1.2.0 vs 1.3.0-SNAPSHOT, so their deletion was accepted as upstream intends. - bot.yml: removed the test-hudi-trino-plugin job. This branch's copy lacked the `needs: changes` guard master had, which is why it conflicted; upstream adds no replacement job because hudi-trino is opt-in behind the hudi-trino profile and stays out of the default CI matrix. The dedicated hudi_trino_ci.yml workflow comes in with this commit. hudi-trino-plugin was byte-identical to master's pre-image (all 143 files), so the migration applied as a clean rename. The resulting hudi-trino tree matches the upstream commit exactly (152/152 files). NOTE: this removes the published hudi-trino-bundle artifact and the docker trino images from the release. See the release notes discussion before tagging. (cherry picked from commit c3c9367907273aeae19ec072871bb9a99dc7e3d8) --- .github/workflows/bot.yml | 22 - .github/workflows/hudi_trino_ci.yml | 147 ++++ .github/workflows/hudi_trino_compat.yml | 105 +++ azure-pipelines-20230430.yml | 1 - docker/README.md | 18 +- docker/hoodie/hadoop/pom.xml | 3 - docker/hoodie/hadoop/sparkadhoc/adhoc.sh | 2 - docker/hoodie/hadoop/trinobase/Dockerfile | 67 -- docker/hoodie/hadoop/trinobase/pom.xml | 116 --- .../hoodie/hadoop/trinobase/scripts/trino.sh | 25 - .../hoodie/hadoop/trinocoordinator/Dockerfile | 29 - .../etc/catalog/hive.properties | 22 - .../trinocoordinator/etc/config.properties | 26 - .../hadoop/trinocoordinator/etc/jvm.config | 27 - .../trinocoordinator/etc/log.properties | 19 - .../trinocoordinator/etc/node.properties | 21 - docker/hoodie/hadoop/trinocoordinator/pom.xml | 96 --- docker/hoodie/hadoop/trinoworker/Dockerfile | 29 - .../trinoworker/etc/catalog/hive.properties | 22 - .../hadoop/trinoworker/etc/config.properties | 24 - .../hoodie/hadoop/trinoworker/etc/jvm.config | 27 - .../hadoop/trinoworker/etc/log.properties | 19 - .../hadoop/trinoworker/etc/node.properties | 21 - docker/hoodie/hadoop/trinoworker/pom.xml | 96 --- .../src/hudi_agent_gateway.egg-info/PKG-INFO | 163 ++++ .../hudi_agent_gateway.egg-info/SOURCES.txt | 43 ++ .../dependency_links.txt | 1 + .../entry_points.txt | 2 + .../hudi_agent_gateway.egg-info/requires.txt | 20 + .../hudi_agent_gateway.egg-info/top_level.txt | 1 + hudi-trino-plugin/pom.xml | 482 ------------ .../hudi/reader/HudiTrinoReaderContext.java | 227 ------ .../plugin/hudi/reader/HudiTrinoRecord.java | 183 ----- .../violations-production-code-only.xml | 0 .../.mvn/modernizer/violations.xml | 0 hudi-trino/README.md | 57 ++ hudi-trino/pom.xml | 718 ++++++++++++++++++ .../plugin/hudi/ForHudiSplitManager.java | 0 .../trino/plugin/hudi/ForHudiSplitSource.java | 0 .../hudi/HudiBaseFileOnlyPageSource.java | 12 +- .../java/io/trino/plugin/hudi/HudiConfig.java | 18 +- .../io/trino/plugin/hudi/HudiConnector.java | 0 .../plugin/hudi/HudiConnectorFactory.java | 28 +- .../io/trino/plugin/hudi/HudiErrorCode.java | 0 .../io/trino/plugin/hudi/HudiFileStatus.java | 0 .../io/trino/plugin/hudi/HudiMetadata.java | 30 +- .../plugin/hudi/HudiMetadataFactory.java | 0 .../java/io/trino/plugin/hudi/HudiModule.java | 4 +- .../io/trino/plugin/hudi/HudiPageSource.java | 74 +- .../plugin/hudi/HudiPageSourceProvider.java | 159 ++-- .../java/io/trino/plugin/hudi/HudiPlugin.java | 0 .../io/trino/plugin/hudi/HudiPredicates.java | 0 .../plugin/hudi/HudiSessionProperties.java | 19 + .../java/io/trino/plugin/hudi/HudiSplit.java | 1 - .../trino/plugin/hudi/HudiSplitManager.java | 11 +- .../io/trino/plugin/hudi/HudiSplitSource.java | 16 +- .../io/trino/plugin/hudi/HudiTableHandle.java | 75 +- .../io/trino/plugin/hudi/HudiTableInfo.java | 0 .../io/trino/plugin/hudi/HudiTableName.java | 0 .../plugin/hudi/HudiTableProperties.java | 0 .../plugin/hudi/HudiTransactionManager.java | 0 .../java/io/trino/plugin/hudi/HudiUtil.java | 175 ++++- .../java/io/trino/plugin/hudi/TableType.java | 0 .../io/trino/plugin/hudi/TimelineTable.java | 0 .../hudi/cache/HudiCacheKeyProvider.java | 0 .../trino/plugin/hudi/file/HudiBaseFile.java | 0 .../io/trino/plugin/hudi/file/HudiFile.java | 0 .../trino/plugin/hudi/file/HudiLogFile.java | 0 .../hudi/io/HudiTrinoFileReaderFactory.java | 39 +- .../plugin/hudi/io/HudiTrinoIOFactory.java | 12 +- .../io/HudiTrinoParquetFileFormatUtils.java | 210 +++++ .../io/InlineSeekableDataInputStream.java | 2 +- .../hudi/io/TrinoSeekableDataInputStream.java | 0 .../hudi/partition/HiveHudiPartitionInfo.java | 0 .../hudi/partition/HudiPartitionInfo.java | 0 .../partition/HudiPartitionInfoLoader.java | 0 .../hudi/query/HudiDirectoryLister.java | 0 .../query/HudiSnapshotDirectoryLister.java | 2 +- .../query/index/HudiBaseIndexSupport.java | 2 +- .../index/HudiColumnStatsIndexSupport.java | 12 +- .../hudi/query/index/HudiIndexSupport.java | 0 .../query/index/HudiNoOpIndexSupport.java | 2 +- .../index/HudiPartitionStatsIndexSupport.java | 11 +- .../index/HudiRecordLevelIndexSupport.java | 6 +- .../index/HudiSecondaryIndexSupport.java | 6 +- .../hudi/query/index/IndexSupportFactory.java | 4 +- .../hudi/reader/HudiTrinoReaderContext.java | 288 +++++++ .../hudi/split/HudiBackgroundSplitLoader.java | 6 +- .../plugin/hudi/split/HudiSplitFactory.java | 28 +- .../hudi/split/HudiSplitWeightProvider.java | 0 .../split/SizeBasedSplitWeightProvider.java | 0 .../hudi/stats/ForHudiTableStatistics.java | 0 .../hudi/stats/HudiTableStatistics.java | 0 .../hudi/stats/TableMetadataReader.java | 56 +- .../hudi/stats/TableStatisticsReader.java | 4 +- .../hudi/storage/HudiTrinoInlineStorage.java | 0 .../plugin/hudi/storage/HudiTrinoStorage.java | 0 .../storage/TrinoStorageConfiguration.java | 0 .../plugin/hudi/util/HudiAvroSerializer.java | 2 +- .../plugin/hudi/util/HudiTableTypeUtils.java | 0 .../hudi/util/SynthesizedColumnHandler.java | 0 .../hudi/util/SynthesizedColumnStrategy.java | 0 .../plugin/hudi/util/TupleDomainUtils.java | 0 .../hudi/BaseHudiConnectorSmokeTest.java | 0 .../io/trino/plugin/hudi/HudiQueryRunner.java | 8 +- .../io/trino/plugin/hudi/HudiUtilTest.java | 0 .../io/trino/plugin/hudi/SessionBuilder.java | 21 + .../TestHudiAlluxioCacheFileOperations.java | 10 - .../hudi/TestHudiAlluxioCachingSmokeTest.java | 0 .../io/trino/plugin/hudi/TestHudiConfig.java | 9 +- .../plugin/hudi/TestHudiConnectorFactory.java | 0 ...stHudiConnectorParquetColumnNamesTest.java | 0 .../plugin/hudi/TestHudiConnectorTest.java | 5 + .../plugin/hudi/TestHudiCustomMerger.java | 104 +++ .../hudi/TestHudiCustomMergerEndToEnd.java | 128 ++++ .../TestHudiMemoryCacheFileOperations.java | 10 - .../hudi/TestHudiMinioConnectorSmokeTest.java | 0 .../hudi/TestHudiNoCacheFileOperations.java | 10 - .../trino/plugin/hudi/TestHudiPageSource.java | 2 +- .../hudi/TestHudiPageSourceProviderTest.java | 0 .../io/trino/plugin/hudi/TestHudiPlugin.java | 0 .../hudi/TestHudiSessionProperties.java | 14 + .../plugin/hudi/TestHudiSharedMetastore.java | 13 +- .../trino/plugin/hudi/TestHudiSmokeTest.java | 12 +- .../plugin/hudi/TestHudiSystemTables.java | 0 .../hudi/TestingHudiConnectorFactory.java | 0 .../trino/plugin/hudi/TestingHudiPlugin.java | 0 .../io/TestInlineSeekableDataInputStream.java | 0 .../TestHudiPartitionInfoLoader.java | 6 +- .../HudiRecordLevelIndexSupportTest.java | 0 .../hudi/query/index/TestingColumnHandle.java | 0 .../hudi/split/TestHudiSplitFactory.java | 7 +- .../CustomMergerHudiTablesInitializer.java | 239 ++++++ .../hudi/testing/HudiTableUnzipper.java | 0 .../hudi/testing/HudiTablesInitializer.java | 0 .../plugin/hudi/testing/HudiTestUtils.java | 0 ...ntalCustomMergerHudiTablesInitializer.java | 481 ++++++++++++ .../testing/KeyBasedTestRecordMerger.java | 84 ++ .../hudi/testing/MaxRankRecordMerger.java | 108 +++ ...nProjectionCompatibleTestRecordMerger.java | 34 + .../ResourceHudiTablesInitializer.java | 0 .../testing/TpchHudiTablesInitializer.java | 21 +- .../plugin/hudi/testing/TypeInfoHelper.java | 0 .../hudi/util/FileOperationAssertions.java | 176 +++++ .../plugin/hudi/util/FileOperationUtils.java | 0 ...upleDomainUtilsExtendedNullFilterTest.java | 0 .../hudi/util/TestTupleDomainUtilsTest.java | 0 .../hudi_comprehensive_types_v6_mor.md | 17 + .../hudi_comprehensive_types_v6_mor.zip | Bin .../hudi_comprehensive_types_v8_mor.md | 17 + .../hudi_comprehensive_types_v8_mor.zip | Bin ...i_cow_pt_table_with_field_names_in_caps.md | 17 + ..._cow_pt_table_with_field_names_in_caps.zip | Bin .../hudi-testing-data/hudi_cow_pt_tbl.zip | Bin 11750 -> 11752 bytes ...hudi_cow_table_with_field_names_in_caps.md | 17 + ...udi_cow_table_with_field_names_in_caps.zip | Bin ...with_multi_keys_and_field_names_in_caps.md | 17 + ...ith_multi_keys_and_field_names_in_caps.zip | Bin .../hudi_custom_keygen_pt_v8_mor.md | 17 + .../hudi_custom_keygen_pt_v8_mor.zip | Bin ...hudi_mor_table_with_field_names_in_caps.md | 17 + ...udi_mor_table_with_field_names_in_caps.zip | Bin .../hudi_multi_fg_pt_v6_mor.md | 17 + .../hudi_multi_fg_pt_v6_mor.zip | Bin .../hudi_multi_fg_pt_v8_mor.md | 17 + .../hudi_multi_fg_pt_v8_mor.zip | Bin .../hudi-testing-data/hudi_multi_pt_v8_mor.md | 17 + .../hudi_multi_pt_v8_mor.zip | Bin .../hudi_non_extractable_partition_path.md | 17 + .../hudi_non_extractable_partition_path.zip | Bin .../hudi-testing-data/hudi_non_part_cow.md | 17 + .../hudi-testing-data/hudi_non_part_cow.zip | Bin 5980 -> 5966 bytes .../hudi-testing-data/hudi_non_part_mor.md | 17 + .../hudi-testing-data/hudi_non_part_mor.zip | Bin .../hudi_stock_ticks_cow.zip | Bin .../hudi_stock_ticks_mor.zip | Bin ...keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md | 17 + ...eygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip | Bin ...eygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md | 17 + ...ygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip | Bin .../hudi-testing-data/hudi_trips_cow_v8.md | 17 + .../hudi-testing-data/hudi_trips_cow_v8.zip | Bin .../hudi-testing-data/stock_ticks_cow.zip | Bin .../hudi-testing-data/stock_ticks_mor.zip | Bin .../src/test/resources/long_timestamp.parquet | Bin packaging/hudi-trino-bundle/pom.xml | 236 ------ .../org/apache/hudi/trino/bundle/Main.java | 36 - pom.xml | 9 +- scripts/release/validate_source_copyright.sh | 4 +- scripts/release/validate_staged_bundles.sh | 2 +- style/checkstyle-suppressions.xml | 4 +- 191 files changed, 4016 insertions(+), 2244 deletions(-) create mode 100644 .github/workflows/hudi_trino_ci.yml create mode 100644 .github/workflows/hudi_trino_compat.yml delete mode 100644 docker/hoodie/hadoop/trinobase/Dockerfile delete mode 100644 docker/hoodie/hadoop/trinobase/pom.xml delete mode 100644 docker/hoodie/hadoop/trinobase/scripts/trino.sh delete mode 100644 docker/hoodie/hadoop/trinocoordinator/Dockerfile delete mode 100644 docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties delete mode 100644 docker/hoodie/hadoop/trinocoordinator/etc/config.properties delete mode 100644 docker/hoodie/hadoop/trinocoordinator/etc/jvm.config delete mode 100644 docker/hoodie/hadoop/trinocoordinator/etc/log.properties delete mode 100644 docker/hoodie/hadoop/trinocoordinator/etc/node.properties delete mode 100644 docker/hoodie/hadoop/trinocoordinator/pom.xml delete mode 100644 docker/hoodie/hadoop/trinoworker/Dockerfile delete mode 100644 docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties delete mode 100644 docker/hoodie/hadoop/trinoworker/etc/config.properties delete mode 100644 docker/hoodie/hadoop/trinoworker/etc/jvm.config delete mode 100644 docker/hoodie/hadoop/trinoworker/etc/log.properties delete mode 100644 docker/hoodie/hadoop/trinoworker/etc/node.properties delete mode 100644 docker/hoodie/hadoop/trinoworker/pom.xml create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/PKG-INFO create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/SOURCES.txt create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/dependency_links.txt create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/entry_points.txt create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/requires.txt create mode 100644 hudi-agent-gateway/src/hudi_agent_gateway.egg-info/top_level.txt delete mode 100644 hudi-trino-plugin/pom.xml delete mode 100644 hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java delete mode 100644 hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java rename {hudi-trino-plugin => hudi-trino}/.mvn/modernizer/violations-production-code-only.xml (100%) rename {hudi-trino-plugin => hudi-trino}/.mvn/modernizer/violations.xml (100%) create mode 100644 hudi-trino/README.md create mode 100644 hudi-trino/pom.xml rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java (91%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiConfig.java (95%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiConnector.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java (79%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiMetadata.java (94%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiModule.java (95%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiPageSource.java (60%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java (78%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiPlugin.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiPredicates.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java (95%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiSplit.java (99%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java (93%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java (94%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java (77%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiTableName.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/HudiUtil.java (65%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/TableType.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/TimelineTable.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/cache/HudiCacheKeyProvider.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/file/HudiFile.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java (55%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java (80%) create mode 100644 hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoParquetFileFormatUtils.java rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java (98%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java (99%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java (98%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java (97%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java (96%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java (94%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java (98%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java (97%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java (98%) create mode 100644 hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java (98%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java (85%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java (50%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java (97%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java (99%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/main/java/io/trino/plugin/hudi/util/TupleDomainUtils.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorSmokeTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java (97%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/HudiUtilTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/SessionBuilder.java (86%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java (91%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCachingSmokeTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java (94%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiConnectorFactory.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java (89%) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMergerEndToEnd.java rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java (89%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java (88%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java (91%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java (68%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java (94%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java (99%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestHudiSystemTables.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestingHudiConnectorFactory.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/TestingHudiPlugin.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/io/TestInlineSeekableDataInputStream.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java (98%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupportTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/query/index/TestingColumnHandle.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java (98%) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/HudiTableUnzipper.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/HudiTablesInitializer.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/HudiTestUtils.java (100%) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/KeyBasedTestRecordMerger.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java (95%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/testing/TypeInfoHelper.java (100%) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationAssertions.java rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/util/FileOperationUtils.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsExtendedNullFilterTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsTest.java (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md (91%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md (91%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md (72%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_pt_tbl.zip (74%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.md (59%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md (60%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md (78%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md (59%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md (72%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md (75%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md (78%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md (66%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_part_cow.md (65%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_part_cow.zip (62%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_part_mor.md (74%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_stock_ticks_cow.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_stock_ticks_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md (77%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md (78%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md (59%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/stock_ticks_cow.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/hudi-testing-data/stock_ticks_mor.zip (100%) rename {hudi-trino-plugin => hudi-trino}/src/test/resources/long_timestamp.parquet (100%) delete mode 100644 packaging/hudi-trino-bundle/pom.xml delete mode 100644 packaging/hudi-trino-bundle/src/main/java/org/apache/hudi/trino/bundle/Main.java diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 4d180e94d6f24..a996e9adc1368 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1428,25 +1428,3 @@ jobs: run: mvn test -Punit-tests -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS - test-hudi-trino-plugin: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v5 - - name: Set up JDK 23 - uses: actions/setup-java@v5 - with: - # Note: We are not caching here again, as we want to use the .m2 repository populated by - # the previous step - java-version: '23' - distribution: 'temurin' - architecture: x64 - cache: maven - - name: Build hudi-trino-plugin with JDK 23 - working-directory: ./hudi-trino-plugin - run: - mvn clean install -DskipTests - - name: Test hudi-trino-plugin with JDK 23 - working-directory: ./hudi-trino-plugin - run: - mvn test -Dapi.version=1.44 diff --git a/.github/workflows/hudi_trino_ci.yml b/.github/workflows/hudi_trino_ci.yml new file mode 100644 index 0000000000000..0cfd1684d817d --- /dev/null +++ b/.github/workflows/hudi_trino_ci.yml @@ -0,0 +1,147 @@ +name: Hudi Trino Connector CI + +on: + push: + branches: + - master + - 'release-*' + paths: + - 'hudi-trino/**' + - '.github/workflows/hudi_trino_ci.yml' + # Upstream modules the connector build installs (the -am closure of the JDK 17 + # install step) plus the poms that own trino.version and the dependency pins. + # Keep in sync with the case patterns in detect-trino-changes below. + - 'pom.xml' + - 'hudi-tests-common/**' + - 'hudi-io/**' + - 'hudi-common/**' + - 'hudi-hadoop-common/**' + - 'hudi-timeline-service/**' + - 'hudi-hadoop-mr/**' + - 'hudi-client/pom.xml' + - 'hudi-client/hudi-client-common/**' + - 'hudi-client/hudi-java-client/**' + - 'hudi-sync/hudi-sync-common/**' + - 'hudi-sync/hudi-hive-sync/**' + # No `paths:` filter here on purpose. test-hudi-trino-plugin is a required status check + # in .asf.yaml, and a path-filtered workflow is never instantiated on PRs that miss the + # filter, leaving the required context permanently pending. Run on every PR instead and + # skip the expensive steps via the detect-trino-changes job below. + pull_request: + branches: + - master + - 'release-*' + workflow_dispatch: + +concurrency: + group: hudi-trino-ci-${{ github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master') && !contains(github.ref, 'release-') }} + +env: + MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log + +jobs: + changes: + name: detect-trino-changes + runs-on: ubuntu-latest + outputs: + trino: ${{ steps.filter.outputs.trino }} + steps: + - name: Detect hudi-trino changes + id: filter + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + EVENT: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BEFORE_SHA: ${{ github.event.before }} + AFTER_SHA: ${{ github.sha }} + run: | + set -euo pipefail + TRINO=false + if [ "$EVENT" = "pull_request" ]; then + FILES=$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename') + elif [ "$EVENT" = "push" ]; then + FILES=$(gh api "repos/$REPO/compare/$BEFORE_SHA...$AFTER_SHA" --jq '.files[].filename') + else + # workflow_dispatch and anything else: always run the full build. + FILES="" + TRINO=true + fi + echo "Changed files:" + printf '%s\n' "$FILES" + while IFS= read -r f; do + [ -z "$f" ] && continue + case "$f" in + hudi-trino/*) TRINO=true ;; + .github/workflows/hudi_trino_ci.yml) TRINO=true ;; + # Upstream modules the connector build installs (the -am closure of the JDK 17 + # install step) plus the poms that own trino.version and the dependency pins. + # Keep in sync with the push paths above. + pom.xml) TRINO=true ;; + hudi-tests-common/*) TRINO=true ;; + hudi-io/*) TRINO=true ;; + hudi-common/*) TRINO=true ;; + hudi-hadoop-common/*) TRINO=true ;; + hudi-timeline-service/*) TRINO=true ;; + hudi-hadoop-mr/*) TRINO=true ;; + hudi-client/pom.xml) TRINO=true ;; + hudi-client/hudi-client-common/*) TRINO=true ;; + hudi-client/hudi-java-client/*) TRINO=true ;; + hudi-sync/hudi-sync-common/*) TRINO=true ;; + hudi-sync/hudi-hive-sync/*) TRINO=true ;; + esac + done <<< "$FILES" + echo "trino=$TRINO" + echo "trino=$TRINO" >> "$GITHUB_OUTPUT" + + build-and-test: + name: test-hudi-trino-plugin + runs-on: ubuntu-latest + needs: changes + steps: + - name: Checkout repository + if: needs.changes.outputs.trino == 'true' + uses: actions/checkout@v5 + # Hudi targets Java 11 and uses Lombok 1.18.36, which does not run on JDK 25. + # Build the upstream modules hudi-trino depends on under JDK 17 first, + # install them into the local m2, then build the connector itself under JDK 25. + - name: Set up JDK 17 + if: needs.changes.outputs.trino == 'true' + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: Install upstream Hudi modules (JDK 17) + if: needs.changes.outputs.trino == 'true' + # hudi-client-common and hudi-java-client are pulled in here for the test stage; + # they live behind the hudi-trino-tests profile but the test step needs them. + run: mvn $MVN_ARGS install -pl :hudi-common,:hudi-hive-sync,:hudi-io,:hudi-sync-common,:hudi-client-common,:hudi-java-client -am -Dmaven.test.skip=true -Drat.skip -Dcheckstyle.skip + - name: Set up JDK 25 + if: needs.changes.outputs.trino == 'true' + uses: actions/setup-java@v5 + with: + java-version: '25' + distribution: 'temurin' + cache: maven + # Trino does not publish trino-spi / trino-filesystem / trino-hive test-jars to + # Maven Central. Check out the matching release tag and install just the modules + # whose test classifiers we need into the local m2. + - name: Checkout trinodb/trino at 481 + if: needs.changes.outputs.trino == 'true' + uses: actions/checkout@v5 + with: + repository: trinodb/trino + ref: '481' + path: trino-src + - name: Install Trino test-jars (JDK 25) + if: needs.changes.outputs.trino == 'true' + working-directory: trino-src + run: mvn $MVN_ARGS install -pl :trino-spi,:trino-filesystem,:trino-hive,:trino-main -am -DskipTests -Dair.check.skip-all=true + - name: Build connector (JDK 25) + if: needs.changes.outputs.trino == 'true' + run: mvn $MVN_ARGS -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + - name: Test connector (JDK 25) + if: needs.changes.outputs.trino == 'true' + run: mvn $MVN_ARGS -Phudi-trino,hudi-trino-tests -pl hudi-trino test diff --git a/.github/workflows/hudi_trino_compat.yml b/.github/workflows/hudi_trino_compat.yml new file mode 100644 index 0000000000000..701f721155109 --- /dev/null +++ b/.github/workflows/hudi_trino_compat.yml @@ -0,0 +1,105 @@ +name: Hudi Trino SPI Compatibility + +on: + schedule: + - cron: '17 4 * * *' + workflow_dispatch: + +# The failure handler files/updates a drift report issue. +permissions: + contents: read + issues: write + +env: + MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip + +jobs: + compile-against-trino-master: + name: Compile hudi-trino against trinodb/trino master + runs-on: ubuntu-latest + steps: + - name: Checkout Hudi + uses: actions/checkout@v5 + with: + path: hudi + - name: Checkout trinodb/trino master + uses: actions/checkout@v5 + with: + repository: trinodb/trino + ref: master + path: trino + # Hudi targets Java 11 and uses Lombok 1.18.36, which does not run on JDK 25. + # Install the upstream Hudi modules under JDK 17 first, then compile the connector + # under JDK 25. + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: Install upstream Hudi modules (JDK 17) + working-directory: hudi + run: mvn $MVN_ARGS install -pl :hudi-common,:hudi-hive-sync,:hudi-io,:hudi-sync-common -am -Dmaven.test.skip=true -Drat.skip -Dcheckstyle.skip + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + java-version: '25' + distribution: 'temurin' + cache: maven + - name: Read Trino version + id: trino-version + working-directory: trino + run: | + set -euo pipefail + # Ask Maven for the project version rather than grepping the pom: the first in + # trinodb/trino's root pom belongs to the (io.airlift:airbase), not to Trino. + # Keep the -SNAPSHOT suffix -- master's version is unreleased, so it only resolves against + # the artifacts installed from source in the next step. + VERSION=$(mvn -q -N help:evaluate -Dexpression=project.version -DforceStdout) + echo "trino_version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Detected Trino version: $VERSION" + - name: Install Trino modules from master (JDK 25) + working-directory: trino + # hudi-trino compiles against these plus their transitive modules (spi, cache, metastore, + # hive-formats, memory-context). They must come from the master checkout -- resolving from + # Maven Central would defeat the point of the drift check. + run: mvn $MVN_ARGS install -pl :trino-hive,:trino-filesystem-manager,:trino-parquet,:trino-plugin-toolkit -am -DskipTests -Dair.check.skip-all=true + - name: Compile hudi-trino against current Trino SPI (JDK 25) + id: compile + working-directory: hudi + run: | + mvn $MVN_ARGS -Phudi-trino \ + -Dtrino.version=${{ steps.trino-version.outputs.trino_version }} \ + -pl hudi-trino compile + - name: Open issue on failure + # Only a connector compile failure is SPI drift. A bare failure() would also file + # the issue for a failed checkout or a broken trinodb/trino master build, with an + # empty version in the title when the failure is before Read Trino version. + if: failure() && steps.compile.outcome == 'failure' + uses: actions/github-script@v7 + with: + script: | + const marker = ''; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const version = '${{ steps.trino-version.outputs.trino_version }}'; + // Drift usually persists for days until someone fixes it. Comment on the existing report + // instead of filing a fresh issue every night. + const existing = await github.rest.search.issuesAndPullRequests({ + q: `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:body "${marker}"`, + }); + if (existing.data.total_count > 0) { + const number = existing.data.items[0].number; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + body: `Still failing against Trino ${version}. See ${runUrl}.`, + }); + return; + } + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `hudi-trino SPI drift detected against Trino ${version}`, + body: `${marker}\nNightly compatibility build failed against Trino ${version}. See ${runUrl}.`, + }); diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index ce003a13b11f3..3a00589a83db7 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -97,7 +97,6 @@ parameters: - '!packaging/hudi-presto-bundle' - '!packaging/hudi-spark-bundle' - '!packaging/hudi-timeline-server-bundle' - - '!packaging/hudi-trino-bundle' - '!packaging/hudi-utilities-slim-bundle' variables: diff --git a/docker/README.md b/docker/README.md index f445a87fb4d4b..1563ce76aeb11 100644 --- a/docker/README.md +++ b/docker/README.md @@ -25,7 +25,7 @@ docker demo environment. ### Configs for assembling docker images - `/hoodie` The `/hoodie` folder contains all the configs for assembling necessary docker images. The name and repository of each -docker image, e.g., `apachehudi/hudi-hadoop_2.8.4-trinobase_368`, is defined in the maven configuration file `pom.xml`. +docker image, e.g., `apachehudi/hudi-hadoop_2.8.4-prestobase_0.232`, is defined in the maven configuration file `pom.xml`. ### Base images by Java version @@ -39,9 +39,9 @@ docker image, e.g., `apachehudi/hudi-hadoop_2.8.4-trinobase_368`, is defined in The legacy Java 8 `base` module under `/hoodie/hadoop/base` is retained for historical reference only; Spark 2.x is no longer supported and `build_docker_images.sh` never selects it. -Downstream Dockerfiles (`datanode`, `historyserver`, `hive_base`, `namenode`, `prestobase`, `trinobase`) pick the base -via the `BASE_IMAGE_TAG` build arg (default `java11`). `build_docker_images.sh` sets it automatically; bare `docker -build` invocations targeting the Java 17 base must pass `--build-arg BASE_IMAGE_TAG=java17`. +Downstream Dockerfiles (`datanode`, `historyserver`, `hive_base`, `namenode`, `prestobase`) pick the base via the +`BASE_IMAGE_TAG` build arg (default `java11`). `build_docker_images.sh` sets it automatically; bare `docker build` +invocations targeting the Java 17 base must pass `--build-arg BASE_IMAGE_TAG=java17`. ### Docker compose config for the Demo - `/compose` @@ -94,8 +94,8 @@ To build a single image target, you can run ```shell mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.build.skip=false -pl : -am -# For example, to build hudi-hadoop-trinobase-docker -mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.build.skip=false -pl :hudi-hadoop-trinobase-docker -am +# For example, to build hudi-hadoop-prestobase-docker +mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.build.skip=false -pl :hudi-hadoop-prestobase-docker -am ``` Alternatively, you can use `docker` cli directly under `hoodie/hadoop` to build images in a faster way. If you use this @@ -115,8 +115,8 @@ steps in the next section). ```shell # Run under hoodie/hadoop, the is optional, "latest" by default docker build -t /[:] -# For example, to build trinobase -docker build trinobase -t apachehudi/hudi-hadoop_2.8.4-trinobase_368 +# For example, to build prestobase +docker build prestobase -t apachehudi/hudi-hadoop_2.8.4-prestobase_0.232 ``` After new images are built, you can run the following script to bring up docker demo with your local images: @@ -133,7 +133,7 @@ Hud registry designated by its name or tag: ```shell docker push /: # For example -docker push apachehudi/hudi-hadoop_2.8.4-trinobase_368 +docker push apachehudi/hudi-hadoop_2.8.4-prestobase_0.232 ``` You can also easily push the image to the Docker Hub using Docker Desktop app: go to `Images`, search for the image by diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml index 459818eeb0ae2..3fe8093e99e5f 100644 --- a/docker/hoodie/hadoop/pom.xml +++ b/docker/hoodie/hadoop/pom.xml @@ -39,9 +39,6 @@ sparkworker sparkadhoc prestobase - trinobase - trinocoordinator - trinoworker diff --git a/docker/hoodie/hadoop/sparkadhoc/adhoc.sh b/docker/hoodie/hadoop/sparkadhoc/adhoc.sh index 86fbbf4e775a0..d2529e9ce165d 100644 --- a/docker/hoodie/hadoop/sparkadhoc/adhoc.sh +++ b/docker/hoodie/hadoop/sparkadhoc/adhoc.sh @@ -23,12 +23,10 @@ export SPARK_HOME=/opt/spark export PRESTO_CLI_CMD="/usr/local/bin/presto --server presto-coordinator-1:8090" -export TRINO_CLI_CMD="/usr/local/bin/trino --server trino-coordinator-1:8091" date echo "SPARK HOME is : $SPARK_HOME" echo "PRESTO CLI CMD is : $PRESTO_CLI_CMD" -echo "TRINO CLI CMD is : $TRINO_CLI_CMD" tail -f /dev/null diff --git a/docker/hoodie/hadoop/trinobase/Dockerfile b/docker/hoodie/hadoop/trinobase/Dockerfile deleted file mode 100644 index a6f7701a4ef4a..0000000000000 --- a/docker/hoodie/hadoop/trinobase/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG HIVE_VERSION=2.3.3 -ARG BASE_IMAGE_TAG=java11 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest as hadoop-base - -ENV TRINO_VERSION=368 -ENV TRINO_HOME=/usr/local/trino -ENV BASE_URL=https://repo1.maven.org/maven2 - -RUN apt-get update -RUN apt-get install -y \ - curl \ - tar \ - sudo \ - rsync \ - python \ - wget \ - python3-pip \ - python-dev \ - build-essential \ - uuid-runtime \ - less - -ENV JAVA_HOME=/usr/java/default -ENV PATH=$PATH:$JAVA_HOME/bin - -WORKDIR /usr/local/bin -RUN wget -q ${BASE_URL}/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar -RUN chmod +x trino-cli-${TRINO_VERSION}-executable.jar -RUN mv trino-cli-${TRINO_VERSION}-executable.jar trino-cli - -WORKDIR /usr/local -RUN wget -q ${BASE_URL}/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz -RUN tar xvzf trino-server-${TRINO_VERSION}.tar.gz -C /usr/local/ -RUN ln -s /usr/local/trino-server-${TRINO_VERSION} $TRINO_HOME - -ENV TRINO_BASE_WS=/var/hoodie/ws/docker/hoodie/hadoop/trinobase -RUN mkdir -p ${TRINO_BASE_WS}/target/ -ADD target/ ${TRINO_BASE_WS}/target/ -ENV HUDI_TRINO_BUNDLE=${TRINO_BASE_WS}/target/hudi-trino-bundle.jar -RUN cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/ - -ADD scripts ${TRINO_HOME}/scripts -RUN chmod +x ${TRINO_HOME}/scripts/trino.sh - -RUN mkdir -p $TRINO_HOME/data -VOLUME ["$TRINO_HOME/data"] diff --git a/docker/hoodie/hadoop/trinobase/pom.xml b/docker/hoodie/hadoop/trinobase/pom.xml deleted file mode 100644 index 595d272e54588..0000000000000 --- a/docker/hoodie/hadoop/trinobase/pom.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - - hudi-hadoop-docker - org.apache.hudi - 1.2.0 - - 4.0.0 - pom - hudi-hadoop-trinobase-docker - Trino Base Docker Image with Hudi - - - UTF-8 - true - ${project.parent.parent.basedir} - - - - - - org.apache.hudi - hudi-hadoop-base-java11-docker - ${project.version} - pom - import - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - package - - - - - - - run - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile.maven.version} - - - tag-latest - pre-integration-test - - build - tag - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version} - - true - latest - - - - tag-version - pre-integration-test - - build - tag - - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version} - - true - ${project.version} - - - - - - - diff --git a/docker/hoodie/hadoop/trinobase/scripts/trino.sh b/docker/hoodie/hadoop/trinobase/scripts/trino.sh deleted file mode 100644 index 4efaed0cd8d31..0000000000000 --- a/docker/hoodie/hadoop/trinobase/scripts/trino.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Copy the trino bundle at run time so that locally built bundle overrides the one that is present in the image -echo "Copying trino bundle to ${TRINO_HOME}/plugin/hive/" -cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/ - -/usr/local/trino/bin/launcher run diff --git a/docker/hoodie/hadoop/trinocoordinator/Dockerfile b/docker/hoodie/hadoop/trinocoordinator/Dockerfile deleted file mode 100644 index 67a31448d7a65..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG TRINO_VERSION=368 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-trinobase_${TRINO_VERSION}:latest as trino-base - -ADD etc /usr/local/trino/etc -EXPOSE 8091 - -WORKDIR /usr/local/trino -ENTRYPOINT [ "./scripts/trino.sh" ] diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties b/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties deleted file mode 100644 index ed7fce1b3e640..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -connector.name=hive -hive.metastore.uri=thrift://hivemetastore:9083 -hive.config.resources=/etc/hadoop/core-site.xml,/etc/hadoop/hdfs-site.xml -hive.hdfs.authentication.type=NONE diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/config.properties b/docker/hoodie/hadoop/trinocoordinator/etc/config.properties deleted file mode 100644 index 9876a0fe0f008..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/config.properties +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -coordinator=true -node-scheduler.include-coordinator=false -http-server.http.port=8091 -query.max-memory=50GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=2GB -discovery-server.enabled=true -discovery.uri=http://trino-coordinator-1:8091 diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config b/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config deleted file mode 100644 index fb17203ca211b..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# --server --Xmx16G --XX:+UseG1GC --XX:G1HeapRegionSize=32M --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:OnOutOfMemoryError=kill -9 %p --Djdk.attach.allowAttachSelf=true diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/log.properties b/docker/hoodie/hadoop/trinocoordinator/etc/log.properties deleted file mode 100644 index 23b063080b4fe..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/log.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -io.trinosql=INFO diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/node.properties b/docker/hoodie/hadoop/trinocoordinator/etc/node.properties deleted file mode 100644 index d97d547485998..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/node.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -node.environment=development -node.id=3044b958-f077-4fce-87ed-ca8308f800b6 -node.data-dir=/usr/local/trino/data diff --git a/docker/hoodie/hadoop/trinocoordinator/pom.xml b/docker/hoodie/hadoop/trinocoordinator/pom.xml deleted file mode 100644 index 90e250ad0f461..0000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - - - hudi-hadoop-docker - org.apache.hudi - 1.2.0 - - 4.0.0 - pom - hudi-hadoop-trinocoordinator-docker - Trino Coordinator Docker Image with Hudi - - - UTF-8 - true - ${project.parent.parent.basedir} - - - - - - org.apache.hudi - hudi-hadoop-trinobase-docker - ${project.version} - pom - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile.maven.version} - - - tag-latest - pre-integration-test - - build - tag - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinocoordinator_${docker.trino.version} - - true - latest - - - - tag-version - pre-integration-test - - build - tag - - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinocoordinator_${docker.trino.version} - - true - ${project.version} - - - - - - - diff --git a/docker/hoodie/hadoop/trinoworker/Dockerfile b/docker/hoodie/hadoop/trinoworker/Dockerfile deleted file mode 100644 index ae5b2766dc9d9..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG TRINO_VERSION=368 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-trinobase_${TRINO_VERSION}:latest as trino-base - -ADD etc /usr/local/trino/etc -EXPOSE 8092 - -WORKDIR /usr/local/trino -ENTRYPOINT [ "./scripts/trino.sh" ] diff --git a/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties b/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties deleted file mode 100644 index ed7fce1b3e640..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -connector.name=hive -hive.metastore.uri=thrift://hivemetastore:9083 -hive.config.resources=/etc/hadoop/core-site.xml,/etc/hadoop/hdfs-site.xml -hive.hdfs.authentication.type=NONE diff --git a/docker/hoodie/hadoop/trinoworker/etc/config.properties b/docker/hoodie/hadoop/trinoworker/etc/config.properties deleted file mode 100644 index 0e15d3d7c1e9c..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/config.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -coordinator=false -http-server.http.port=8091 -query.max-memory=50GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=2GB -discovery.uri=http://trino-coordinator-1:8091 diff --git a/docker/hoodie/hadoop/trinoworker/etc/jvm.config b/docker/hoodie/hadoop/trinoworker/etc/jvm.config deleted file mode 100644 index fb17203ca211b..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/jvm.config +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# --server --Xmx16G --XX:+UseG1GC --XX:G1HeapRegionSize=32M --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:OnOutOfMemoryError=kill -9 %p --Djdk.attach.allowAttachSelf=true diff --git a/docker/hoodie/hadoop/trinoworker/etc/log.properties b/docker/hoodie/hadoop/trinoworker/etc/log.properties deleted file mode 100644 index 23b063080b4fe..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/log.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -io.trinosql=INFO diff --git a/docker/hoodie/hadoop/trinoworker/etc/node.properties b/docker/hoodie/hadoop/trinoworker/etc/node.properties deleted file mode 100644 index 6cfebf995602e..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/node.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -node.environment=development -node.id=6606f0b3-6ae7-4152-a4b1-ddadb6345fe6 -node.data-dir=/var/trino/data diff --git a/docker/hoodie/hadoop/trinoworker/pom.xml b/docker/hoodie/hadoop/trinoworker/pom.xml deleted file mode 100644 index ab3066322407f..0000000000000 --- a/docker/hoodie/hadoop/trinoworker/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - - - hudi-hadoop-docker - org.apache.hudi - 1.2.0 - - 4.0.0 - pom - hudi-hadoop-trinoworker-docker - Trino Worker Docker Image with Hudi - - - UTF-8 - true - ${project.parent.parent.basedir} - - - - - - org.apache.hudi - hudi-hadoop-trinobase-docker - ${project.version} - pom - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile.maven.version} - - - tag-latest - pre-integration-test - - build - tag - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinoworker_${docker.trino.version} - - true - latest - - - - tag-version - pre-integration-test - - build - tag - - - - ${docker.build.skip} - false - - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinoworker_${docker.trino.version} - - true - ${project.version} - - - - - - - diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/PKG-INFO b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/PKG-INFO new file mode 100644 index 0000000000000..e55cb3aedd60e --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/PKG-INFO @@ -0,0 +1,163 @@ +Metadata-Version: 2.4 +Name: hudi-agent-gateway +Version: 0.1.0 +Summary: The Apache Hudi AI gateway: agent loop, MCP server, and chat UI over lakehouse tools +License-Expression: Apache-2.0 +Project-URL: Homepage, https://hudi.apache.org +Project-URL: Source, https://github.com/apache/hudi +Requires-Python: >=3.11 +Description-Content-Type: text/markdown +Requires-Dist: fastapi>=0.115 +Requires-Dist: uvicorn[standard]>=0.30 +Requires-Dist: langgraph<2,>=1.0 +Requires-Dist: langchain-core<2,>=1.0 +Requires-Dist: langchain-anthropic>=1.0 +Requires-Dist: langchain-openai>=1.0 +Requires-Dist: langchain-ollama>=1.0 +Requires-Dist: fastmcp<3,>=2.10 +Requires-Dist: trino>=0.330 +Requires-Dist: sqlglot>=25.0 +Requires-Dist: pydantic>=2.7 +Requires-Dist: pydantic-settings>=2.3 +Requires-Dist: sse-starlette>=2.1 +Requires-Dist: httpx>=0.27 +Provides-Extra: dev +Requires-Dist: pytest>=8.0; extra == "dev" +Requires-Dist: pytest-asyncio>=0.24; extra == "dev" +Requires-Dist: ruff>=0.6; extra == "dev" +Requires-Dist: mypy>=1.11; extra == "dev" + + +# hudi-agent-gateway + +**One deployable service that serves your Hudi lakehouse to AI.** A single +process hosts three surfaces over the same set of guarded lakehouse tools: + +| Surface | Where | What | +|---|---|---| +| Agent chat API | `POST /v1/chat` | prompt in → LangGraph agent loop (model ↔ tools) → grounded answer out; multi-turn sessions; optional SSE streaming | +| MCP server | `/mcp` (streamable HTTP) | external agents (Claude, anything MCP) call the lakehouse tools directly | +| Chat UI | `/ui/` | first-party ChatGPT-style web UI (zero third-party code) | + +The v1 tools query the lakehouse through Trino: `query_lakehouse` (guarded, +read-only SQL), `list_tables`, `describe_table`. Every model-written query +passes AST-level guardrails (single statement, SELECT-only, row cap injected +as a real `LIMIT`) and every invocation is logged as structured JSON — the +seed of the gateway's trace collection. + +## Quickstart (local process) + +```bash +cd hudi-agent-gateway +python3.12 -m venv .venv && .venv/bin/pip install -e ".[dev]" + +# point at a Trino with the Hudi connector (e.g. the local-dev stack, port-forwarded): +GATEWAY_TRINO_HOST=localhost GATEWAY_TRINO_PORT=18080 \ +GATEWAY_LLM_PROVIDER=anthropic GATEWAY_LLM_MODEL=claude-haiku-4-5-20251001 \ + .venv/bin/hudi-agent-gateway serve + +curl -X POST localhost:8000/v1/chat -H 'Content-Type: application/json' \ + -d '{"message": "How many trips per city?", "session_id": "s1"}' +open http://localhost:8000/ui/ +``` + +`GET /v1/models` lists the models the configured provider offers (live: +Anthropic and OpenAI model APIs, Ollama's local tags, vLLM's served models), +and `POST /v1/chat` accepts an optional `"model"` to pick one per request — +the chat UI exposes this as a model picker. Sessions survive model switches. + +For a fully local model, install [Ollama](https://ollama.com), pull a +tool-capable model, and use the default provider: + +```bash +ollama pull qwen3:8b +GATEWAY_LLM_PROVIDER=ollama GATEWAY_LLM_MODEL=qwen3:8b hudi-agent-gateway serve +``` + +Connect an MCP client: + +```bash +claude mcp add --transport http hudi-lakehouse http://localhost:8000/mcp/ +``` + +## Deploying on Kubernetes + +See `hudi-lakehouse/charts/hudi-agent-gateway` — the product Helm chart — +and `hudi-lakehouse/local-dev/` for a complete laptop environment +(MinIO + Hive Metastore + Trino + this gateway) where the gateway is +installed alongside Trino by default. + +## Configuration + +Environment variables (prefix `GATEWAY_` except the standard key names): + +| Variable | Default | Purpose | +|---|---|---| +| `GATEWAY_LLM_PROVIDER` | `ollama` | `anthropic` \| `openai` \| `ollama` \| `openai-compatible` | +| `GATEWAY_LLM_MODEL` | `qwen3:8b` | model name for the provider | +| `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` | — | required by the matching provider | +| `GATEWAY_OLLAMA_BASE_URL` | `http://localhost:11434` | Ollama endpoint | +| `GATEWAY_OPENAI_BASE_URL` | — | endpoint for `openai-compatible` (vLLM, Together, …) | +| `GATEWAY_LLM_TIMEOUT_SECONDS` | `120` | per-model-call timeout | +| `GATEWAY_TRINO_HOST` / `_PORT` | `hudi-trino.hudi-lakehouse.svc` / `8080` | Trino coordinator | +| `GATEWAY_TRINO_CATALOG` / `_SCHEMA` / `_USER` | `hudi` / `default` / `hudi-agent-gateway` | query defaults | +| `GATEWAY_SQL_ROW_CAP` | `200` | LIMIT enforced on every query | +| `GATEWAY_SQL_TIMEOUT_SECONDS` | `120` | per-query timeout | +| `GATEWAY_TOOL_RESULT_MAX_BYTES` | `50000` | tool results truncated beyond this (with notice) | +| `GATEWAY_AGENT_MAX_ITERATIONS` | `25` | agent loop recursion limit | +| `GATEWAY_SESSION_TTL_SECONDS` / `GATEWAY_MAX_SESSIONS` | `3600` / `1000` | session store bounds | +| `GATEWAY_MAX_MESSAGES_PER_SESSION` | `40` | context window per session (trimmed pre-model) | +| `GATEWAY_SYSTEM_PROMPT_EXTRA` | — | appended to the built-in system prompt | +| `GATEWAY_MCP_ENABLED` | `true` | serve /mcp | +| `GATEWAY_HOST` / `GATEWAY_PORT` / `GATEWAY_LOG_LEVEL` | `0.0.0.0` / `8000` / `INFO` | server basics | + +Startup never depends on the LLM or Trino being reachable: `/health` is +liveness, `/ready` reports per-dependency status (and gates the Kubernetes +readiness probe). + +## Development + +```bash +.venv/bin/pytest # offline suite (fake Trino + scripted model) +.venv/bin/ruff check src tests +.venv/bin/mypy src + +# live integration (against a port-forwarded local-dev stack): +GATEWAY_IT_TRINO_HOST=localhost GATEWAY_IT_TRINO_PORT=18080 .venv/bin/pytest tests/integration +``` + +Adding a tool: write a module under `src/hudi_agent_gateway/tools/` with a +`register(registry, ...)` function and call it from +`tools/__init__.py:build_registry`. One registration exposes it to the agent +loop, the MCP server, `GET /v1/tools`, and the invocation log. + +## Design notes & limits (v1) + +- **No authentication in v1**: `/v1/chat`, `/mcp` and the other endpoints are + open to anyone who can reach the pod (and `GATEWAY_HOST` defaults to + `0.0.0.0`). Deploy behind an authenticating proxy or on a trusted network. +- **Single replica**: sessions live in an in-memory LangGraph checkpointer. + The seam for horizontal scale is swapping in + `langgraph-checkpoint-postgres` inside `sessions.py`. +- **Read-only by construction**: non-SELECT statements are rejected at the + AST level (sqlglot, fail-closed); `EXPLAIN` is also blocked in v1. +- The chat UI is deliberately first-party and dependency-free (no CDN, no + npm) so the whole service works air-gapped and stays license-clean. The + one remote reference is the Hudi logo image, hotlinked from + hudi.apache.org; offline it degrades to alt text. diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/SOURCES.txt b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/SOURCES.txt new file mode 100644 index 0000000000000..df0fb57efdff4 --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/SOURCES.txt @@ -0,0 +1,43 @@ +README.md +pyproject.toml +src/hudi_agent_gateway/__init__.py +src/hudi_agent_gateway/__main__.py +src/hudi_agent_gateway/agent.py +src/hudi_agent_gateway/app.py +src/hudi_agent_gateway/cli.py +src/hudi_agent_gateway/config.py +src/hudi_agent_gateway/llm.py +src/hudi_agent_gateway/log.py +src/hudi_agent_gateway/mcp_server.py +src/hudi_agent_gateway/py.typed +src/hudi_agent_gateway/sessions.py +src/hudi_agent_gateway.egg-info/PKG-INFO +src/hudi_agent_gateway.egg-info/SOURCES.txt +src/hudi_agent_gateway.egg-info/dependency_links.txt +src/hudi_agent_gateway.egg-info/entry_points.txt +src/hudi_agent_gateway.egg-info/requires.txt +src/hudi_agent_gateway.egg-info/top_level.txt +src/hudi_agent_gateway/api/__init__.py +src/hudi_agent_gateway/api/chat.py +src/hudi_agent_gateway/api/meta.py +src/hudi_agent_gateway/api/models.py +src/hudi_agent_gateway/tools/__init__.py +src/hudi_agent_gateway/tools/guardrails.py +src/hudi_agent_gateway/tools/registry.py +src/hudi_agent_gateway/tools/trino_client.py +src/hudi_agent_gateway/tools/trino_tools.py +src/hudi_agent_gateway/ui/app.js +src/hudi_agent_gateway/ui/index.html +src/hudi_agent_gateway/ui/markdown.js +src/hudi_agent_gateway/ui/style.css +tests/test_agent_loop.py +tests/test_api_meta.py +tests/test_chat_sse.py +tests/test_config.py +tests/test_guardrails.py +tests/test_llm.py +tests/test_mcp.py +tests/test_registry.py +tests/test_sessions.py +tests/test_trino_tools.py +tests/test_ui.py \ No newline at end of file diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/dependency_links.txt b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/dependency_links.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/entry_points.txt b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/entry_points.txt new file mode 100644 index 0000000000000..84a249ac48f40 --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +hudi-agent-gateway = hudi_agent_gateway.cli:main diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/requires.txt b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/requires.txt new file mode 100644 index 0000000000000..8a2b84a1f23c6 --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/requires.txt @@ -0,0 +1,20 @@ +fastapi>=0.115 +uvicorn[standard]>=0.30 +langgraph<2,>=1.0 +langchain-core<2,>=1.0 +langchain-anthropic>=1.0 +langchain-openai>=1.0 +langchain-ollama>=1.0 +fastmcp<3,>=2.10 +trino>=0.330 +sqlglot>=25.0 +pydantic>=2.7 +pydantic-settings>=2.3 +sse-starlette>=2.1 +httpx>=0.27 + +[dev] +pytest>=8.0 +pytest-asyncio>=0.24 +ruff>=0.6 +mypy>=1.11 diff --git a/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/top_level.txt b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/top_level.txt new file mode 100644 index 0000000000000..a739e485153b5 --- /dev/null +++ b/hudi-agent-gateway/src/hudi_agent_gateway.egg-info/top_level.txt @@ -0,0 +1 @@ +hudi_agent_gateway diff --git a/hudi-trino-plugin/pom.xml b/hudi-trino-plugin/pom.xml deleted file mode 100644 index c67ab6a07f512..0000000000000 --- a/hudi-trino-plugin/pom.xml +++ /dev/null @@ -1,482 +0,0 @@ - - - 4.0.0 - - - io.trino - trino-root - 472 - - - - - trino-hudi - trino-plugin - Trino - Hudi connector - - - true - 1.0.2 - 1.15.2 - - - - - - com.esotericsoftware - kryo - 4.0.2 - - - - com.google.errorprone - error_prone_annotations - true - - - - com.google.guava - guava - - - - com.google.inject - guice - - - - io.airlift - bootstrap - - - - io.airlift - concurrent - - - - io.airlift - configuration - - - - io.airlift - json - - - - io.airlift - log - - - - io.airlift - units - - - - io.trino - trino-cache - - - - io.trino - trino-filesystem - - - - io.trino - trino-filesystem-manager - - - - io.trino - trino-hive - - - - io.trino - trino-memory-context - - - - io.trino - trino-metastore - - - - io.trino - trino-parquet - - - - io.trino - trino-plugin-toolkit - - - - jakarta.validation - jakarta.validation-api - - - - joda-time - joda-time - - - - org.apache.avro - avro - - - - org.apache.hudi - hudi-common - ${dep.hudi.version} - - - io.dropwizard.metrics - * - - - org.apache.hbase - * - - - org.apache.httpcomponents - * - - - org.apache.orc - * - - - - - - org.apache.hudi - hudi-hive-sync - ${dep.hudi.version} - - - org.apache.hudi - hudi-hadoop-common - - - - - - org.apache.hudi - hudi-io - ${dep.hudi.version} - - - com.google.protobuf - protobuf-java - - - - - - org.apache.hudi - hudi-sync-common - ${dep.hudi.version} - - - org.apache.hudi - hudi-hadoop-common - - - - - - org.apache.parquet - parquet-column - - - - org.weakref - jmxutils - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - - io.airlift - slice - provided - - - - io.opentelemetry - opentelemetry-api - provided - - - - io.opentelemetry - opentelemetry-api-incubator - provided - - - - io.opentelemetry - opentelemetry-context - provided - - - - io.trino - trino-spi - provided - - - - org.openjdk.jol - jol-core - provided - - - - com.github.ben-manes.caffeine - caffeine - runtime - - - - io.airlift - log-manager - runtime - - - - io.dropwizard.metrics - metrics-core - runtime - - - - io.opentelemetry - opentelemetry-sdk-trace - runtime - - - - io.trino - trino-hive-formats - runtime - - - - org.jetbrains - annotations - runtime - - - - io.airlift - junit-extensions - test - - - - io.airlift - testing - test - - - - io.trino - trino-client - test - - - - io.trino - trino-filesystem - test-jar - test - - - - io.trino - trino-hdfs - test - - - - io.trino - trino-hive - test-jar - test - - - - io.trino - trino-main - test - - - - io.trino - trino-main - test-jar - test - - - - io.trino - trino-parser - test - - - - io.trino - trino-spi - test-jar - test - - - - io.trino - trino-testing - test - - - - io.trino - trino-testing-containers - test - - - - io.trino - trino-testing-services - test - - - - io.trino - trino-tpch - test - - - - io.trino.hadoop - hadoop-apache - test - - - - io.trino.tpch - tpch - test - - - - org.apache.hudi - hudi-client-common - ${dep.hudi.version} - test - - - * - * - - - - - - org.apache.hudi - hudi-hadoop-common - ${dep.hudi.version} - test - - - * - * - - - - - - org.apache.hudi - hudi-java-client - ${dep.hudi.version} - test - - - org.apache.hudi - * - - - - - - org.apache.parquet - parquet-avro - ${trino.parquet.version} - test - - - - org.apache.parquet - parquet-hadoop - test - - - - org.assertj - assertj-core - test - - - - org.json - json - 20250107 - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.jupiter - junit-jupiter-params - test - - - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - - - log4j.properties - log4j-surefire.properties - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java deleted file mode 100644 index 36c5342920f4e..0000000000000 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hudi.reader; - -import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hudi.util.HudiAvroSerializer; -import io.trino.plugin.hudi.util.SynthesizedColumnHandler; -import io.trino.spi.Page; -import io.trino.spi.connector.ConnectorPageSource; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Record; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.config.RecordMergeMode; -import org.apache.hudi.common.engine.HoodieReaderContext; -import org.apache.hudi.common.model.HoodieAvroIndexedRecord; -import org.apache.hudi.common.model.HoodieAvroRecordMerger; -import org.apache.hudi.common.model.HoodieEmptyRecord; -import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieRecordMerger; -import org.apache.hudi.common.table.read.BufferedRecord; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.ClosableIterator; -import org.apache.hudi.storage.HoodieStorage; -import org.apache.hudi.storage.StoragePath; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.UnaryOperator; - -public class HudiTrinoReaderContext - extends HoodieReaderContext -{ - ConnectorPageSource pageSource; - private final HudiAvroSerializer avroSerializer; - Map colToPosMap; - List dataHandles; - List columnHandles; - - public HudiTrinoReaderContext( - ConnectorPageSource pageSource, - List dataHandles, - List columnHandles, - SynthesizedColumnHandler synthesizedColumnHandler) - { - this.pageSource = pageSource; - this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler); - this.dataHandles = dataHandles; - this.columnHandles = columnHandles; - this.colToPosMap = new HashMap<>(); - for (int i = 0; i < columnHandles.size(); i++) { - HiveColumnHandle handle = columnHandles.get(i); - colToPosMap.put(handle.getBaseColumnName(), i); - } - } - - @Override - public ClosableIterator getFileRecordIterator( - StoragePath storagePath, - long start, - long length, - Schema dataSchema, - Schema requiredSchema, - HoodieStorage storage) - { - return new ClosableIterator<>() - { - private Page currentPage; - private int currentPosition; - - @Override - public void close() - { - try { - pageSource.close(); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public boolean hasNext() - { - // If all records in the current page are consume, try to get next page - if (currentPage == null || currentPosition >= currentPage.getPositionCount()) { - if (pageSource.isFinished()) { - return false; - } - - // Get next page and reset currentPosition - currentPage = pageSource.getNextPage(); - currentPosition = 0; - - // If no more pages are available - return currentPage != null; - } - - return true; - } - - @Override - public IndexedRecord next() - { - if (!hasNext()) { - // TODO: This can probably be removed or ignored, added this as a sanity check - throw new RuntimeException("No more records in the iterator"); - } - - IndexedRecord record = avroSerializer.serialize(currentPage, currentPosition); - currentPosition++; - return record; - } - }; - } - - @Override - public IndexedRecord convertAvroRecord(IndexedRecord record) - { - return record; - } - - @Override - public GenericRecord convertToAvroRecord(IndexedRecord record, Schema schema) - { - GenericRecord ret = new GenericData.Record(schema); - for (Schema.Field field : schema.getFields()) { - ret.put(field.name(), record.get(field.pos())); - } - return ret; - } - - @Override - public Option getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) - { - return Option.of(HoodieAvroRecordMerger.INSTANCE); - } - - @Override - public Object getValue(IndexedRecord record, Schema schema, String fieldName) - { - if (colToPosMap.containsKey(fieldName)) { - return record.get(colToPosMap.get(fieldName)); - } - else { - // record doesn't have the queried field, return null - return null; - } - } - - @Override - public IndexedRecord seal(IndexedRecord record) - { - // TODO: this can rely on colToPos map directly instead of schema - Schema schema = record.getSchema(); - IndexedRecord newRecord = new Record(schema); - List fields = schema.getFields(); - for (Schema.Field field : fields) { - int pos = schema.getField(field.name()).pos(); - newRecord.put(pos, record.get(pos)); - } - return newRecord; - } - - @Override - public IndexedRecord toBinaryRow(Schema schema, IndexedRecord record) - { - return record; - } - - @Override - public ClosableIterator mergeBootstrapReaders( - ClosableIterator closableIterator, Schema schema, - ClosableIterator closableIterator1, Schema schema1) - { - return null; - } - - @Override - public UnaryOperator projectRecord( - Schema from, - Schema to, - Map renamedColumns) - { - List toFields = to.getFields(); - int[] projection = new int[toFields.size()]; - for (int i = 0; i < projection.length; i++) { - projection[i] = from.getField(toFields.get(i).name()).pos(); - } - - return fromRecord -> { - IndexedRecord toRecord = new Record(to); - for (int i = 0; i < projection.length; i++) { - toRecord.put(i, fromRecord.get(projection[i])); - } - return toRecord; - }; - } - - @Override - public HoodieRecord constructHoodieRecord( - BufferedRecord bufferedRecord) - { - if (bufferedRecord.isDelete()) { - return new HoodieEmptyRecord<>( - new HoodieKey(bufferedRecord.getRecordKey(), null), - HoodieRecord.HoodieRecordType.AVRO); - } - - return new HoodieAvroIndexedRecord(bufferedRecord.getRecord()); - } -} diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java deleted file mode 100644 index 7fa0f71399edc..0000000000000 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hudi.reader; - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.model.HoodieAvroIndexedRecord; -import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.model.HoodieOperation; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.MetadataValues; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.keygen.BaseKeyGenerator; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; - -public class HudiTrinoRecord - extends HoodieRecord -{ - public HudiTrinoRecord() - { - } - - @Override - public HoodieRecord newInstance() - { - return null; - } - - @Override - public HoodieRecord newInstance(HoodieKey hoodieKey, HoodieOperation hoodieOperation) - { - return null; - } - - @Override - public HoodieRecord newInstance(HoodieKey hoodieKey) - { - return null; - } - - @Override - public Comparable doGetOrderingValue(Schema schema, Properties properties) - { - return null; - } - - @Override - public HoodieRecordType getRecordType() - { - return null; - } - - @Override - public String getRecordKey(Schema schema, Option option) - { - return ""; - } - - @Override - public String getRecordKey(Schema schema, String s) - { - return ""; - } - - @Override - protected void writeRecordPayload(IndexedRecord page, Kryo kryo, Output output) - { - } - - @Override - protected IndexedRecord readRecordPayload(Kryo kryo, Input input) - { - return null; - } - - @Override - public Object[] getColumnValues(Schema schema, String[] strings, boolean b) - { - return new Object[0]; - } - - @Override - public HoodieRecord joinWith(HoodieRecord hoodieRecord, Schema schema) - { - return null; - } - - @Override - public HoodieRecord prependMetaFields(Schema schema, Schema schema1, - MetadataValues metadataValues, Properties properties) - { - return null; - } - - @Override - public HoodieRecord rewriteRecordWithNewSchema(Schema schema, Properties properties, - Schema schema1, Map map) - { - return null; - } - - @Override - public boolean isDelete(Schema schema, Properties properties) - throws IOException - { - return false; - } - - @Override - public boolean shouldIgnore(Schema schema, Properties properties) - throws IOException - { - return false; - } - - @Override - public HoodieRecord copy() - { - return null; - } - - @Override - public Option> getMetadata() - { - return null; - } - - @Override - public HoodieRecord wrapIntoHoodieRecordPayloadWithParams(Schema schema, Properties properties, - Option> option, Boolean aBoolean, Option option1, - Boolean aBoolean1, Option option2) - throws IOException - { - return null; - } - - @Override - public HoodieRecord wrapIntoHoodieRecordPayloadWithKeyGen(Schema schema, Properties properties, - Option option) - { - return null; - } - - @Override - public HoodieRecord truncateRecordKey(Schema schema, Properties properties, String s) - throws IOException - { - return null; - } - - @Override - public Option toIndexedRecord(Schema schema, Properties properties) - throws IOException - { - return null; - } - - @Override - public ByteArrayOutputStream getAvroBytes(Schema schema, Properties properties) - throws IOException - { - return null; - } -} diff --git a/hudi-trino-plugin/.mvn/modernizer/violations-production-code-only.xml b/hudi-trino/.mvn/modernizer/violations-production-code-only.xml similarity index 100% rename from hudi-trino-plugin/.mvn/modernizer/violations-production-code-only.xml rename to hudi-trino/.mvn/modernizer/violations-production-code-only.xml diff --git a/hudi-trino-plugin/.mvn/modernizer/violations.xml b/hudi-trino/.mvn/modernizer/violations.xml similarity index 100% rename from hudi-trino-plugin/.mvn/modernizer/violations.xml rename to hudi-trino/.mvn/modernizer/violations.xml diff --git a/hudi-trino/README.md b/hudi-trino/README.md new file mode 100644 index 0000000000000..c163121284dce --- /dev/null +++ b/hudi-trino/README.md @@ -0,0 +1,57 @@ + + +# hudi-trino + +Hudi connector for Trino (RFC-105). Published as `org.apache.hudi:hudi-trino` -- a regular non-shaded JAR. The Trino-side `trino-hudi` plugin module depends on this artifact and Trino's URLClassLoader isolates the plugin's transitive deps from the rest of the server, so no shading is required. + +## Build + +Excluded from default builds. Activate the `hudi-trino` Maven profile: + +``` +# tests need Trino test-jars not on Maven Central (see Running tests); skip them in the default build +mvn -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true +``` + +Requires JDK 25 (enforced via `maven-enforcer-plugin`). + +## Running tests + +Tests depend on Trino test-jars (`trino-spi`, `trino-filesystem`, `trino-hive`, `trino-main` at the `tests` classifier). Trino does not publish three of those to Maven Central, so the test deps live behind the `hudi-trino-tests` profile, off by default. + +To run the tests: + +1. Build the matching Trino version locally so its `*-tests.jar` artifacts land in your `~/.m2` (see `trino.version` in the root pom for the version to build). +2. Activate both profiles: + +``` +mvn -Phudi-trino,hudi-trino-tests -pl hudi-trino test +``` + +CI follows the same two steps: `.github/workflows/hudi_trino_ci.yml` installs the test-jars from a source checkout of the pinned Trino tag, then runs with both profiles enabled. + +## IDE setup + +Only this module needs JDK 25. Leave the rest of Hudi on its native JDK (11 or 17) so you are not toggling the project default. + +1. Activate the `hudi-trino` Maven profile so the IDE picks up the module. Tick `hudi-trino-tests` too if you want the test classpath to resolve. + - IntelliJ: Maven tool window, Profiles, tick both `hudi-trino` and `hudi-trino-tests`. +2. Override the SDK for the `hudi-trino` module only, to Temurin 25 with Language level 25. + - IntelliJ: `File > Project Structure > Modules > hudi-trino > Dependencies > Module SDK`. + +The enforcer rule only runs during `mvn`, not during the IDE's incremental compile. diff --git a/hudi-trino/pom.xml b/hudi-trino/pom.xml new file mode 100644 index 0000000000000..58f33855bbc03 --- /dev/null +++ b/hudi-trino/pom.xml @@ -0,0 +1,718 @@ + + + + 4.0.0 + + + org.apache.hudi + hudi + 1.3.0-SNAPSHOT + ../pom.xml + + + hudi-trino + jar + Hudi connector for Trino (RFC-105) + + + + 1.15.2 + + 25 + ${project.parent.basedir} + true + + -Xmx4g -Xms128m -XX:-OmitStackTraceInFastThrow --add-modules jdk.incubator.vector + + + + + + + io.trino + trino-root + ${trino.version} + pom + import + + + + com.fasterxml.jackson.core + jackson-core + 2.21.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.21.3 + + + + com.fasterxml.jackson.core + jackson-annotations + 2.21 + + + + org.eclipse.jetty + jetty-server + 12.1.9 + + + + joda-time + joda-time + 2.14.2 + + + + org.glassfish.jersey.core + jersey-server + 4.0.2 + + + org.glassfish.jersey.core + jersey-client + 4.0.2 + + + org.glassfish.jersey.media + jersey-media-jaxb + 4.0.2 + + + + org.junit.platform + junit-platform-commons + ${junit.platform.version} + + + org.junit.platform + junit-platform-engine + ${junit.platform.version} + + + org.junit.platform + junit-platform-launcher + ${junit.platform.version} + + + + + + + com.esotericsoftware + kryo + 4.0.2 + compile + + + + com.google.errorprone + error_prone_annotations + true + + + + com.google.guava + guava + + + + + com.google.inject + guice + classes + + + + io.airlift + bootstrap + + + + io.airlift + concurrent + + + + io.airlift + configuration + + + + io.airlift + json + + + + io.airlift + log + + + + io.airlift + units + + + + io.trino + trino-cache + + + + io.trino + trino-filesystem + + + + io.trino + trino-filesystem-manager + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + + io.trino + trino-hive + + + + io.trino + trino-memory-context + + + + io.trino + trino-metastore + + + + io.trino + trino-parquet + + + + io.trino + trino-plugin-toolkit + + + + jakarta.validation + jakarta.validation-api + + + + joda-time + joda-time + + + + org.apache.avro + avro + + + + org.apache.hudi + hudi-common + ${project.version} + + + io.dropwizard.metrics + * + + + org.apache.hbase + * + + + org.apache.httpcomponents + * + + + org.apache.orc + * + + + + org.eclipse.jetty + * + + + + org.rocksdb + * + + + org.apache.arrow + * + + + org.lance + * + + + + + + org.apache.hudi + hudi-hive-sync + ${project.version} + + + org.apache.hudi + hudi-hadoop-common + + + + + + org.apache.hudi + hudi-io + ${project.version} + shaded + + + com.google.protobuf + protobuf-java + + + org.rocksdb + * + + + org.apache.arrow + * + + + org.lance + * + + + + + + org.apache.hudi + hudi-sync-common + ${project.version} + + + org.apache.hudi + hudi-hadoop-common + + + + + + org.apache.parquet + parquet-column + + + + org.weakref + jmxutils + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + opentelemetry-api-incubator + provided + + + + io.opentelemetry + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + com.github.ben-manes.caffeine + caffeine + runtime + + + + io.airlift + log-manager + runtime + + + + io.dropwizard.metrics + metrics-core + runtime + + + + io.opentelemetry + opentelemetry-sdk-trace + runtime + + + + io.trino + trino-hive-formats + runtime + + + + org.jetbrains + annotations + runtime + + + + io.airlift + configuration-testing + test + + + + io.airlift + junit-extensions + test + + + + io.airlift + testing + test + + + + io.trino + trino-client + test + + + + io.trino + trino-hdfs + test + + + + io.trino + trino-main + test + + + + io.trino + trino-parser + test + + + + io.trino + trino-testing + test + + + + io.trino + trino-testing-containers + test + + + + io.trino + trino-testing-services + test + + + + io.trino + trino-tpch + test + + + + io.trino.hadoop + hadoop-apache + test + + + + io.trino.tpch + tpch + test + + + + org.apache.parquet + parquet-avro + ${trino.parquet.version} + test + + + + org.apache.parquet + parquet-hadoop + test + + + + org.assertj + assertj-core + test + + + + org.json + json + 20250107 + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.platform + junit-platform-launcher + test + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + + + + log4j.properties + log4j-surefire.properties + google/protobuf/.* + + + + org.apache.parquet.conf.ParquetConfiguration + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${hudi.trino.java.version} + ${hudi.trino.java.version} + ${hudi.trino.java.version} + + none + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-jdk25 + + enforce + + + + + [${hudi.trino.java.version},) + hudi-trino requires JDK ${hudi.trino.java.version} or newer. + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + com.esotericsoftware:kryo + + + + + + + + + + hudi-trino-tests + + + io.trino + trino-filesystem + ${trino.version} + test-jar + test + + + io.trino + trino-hive + ${trino.version} + test-jar + test + + + io.trino + trino-main + ${trino.version} + test-jar + test + + + io.trino + trino-spi + ${trino.version} + test-jar + test + + + org.apache.hudi + hudi-client-common + ${project.version} + test + + + * + * + + + + org.apache.hudi + hudi-timeline-service + + + + + org.apache.hudi + hudi-hadoop-common + ${project.version} + test + + + * + * + + + + + org.apache.hudi + hudi-java-client + ${project.version} + test + + + org.apache.hudi + * + + + + + + + diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java similarity index 91% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java index 1180638a7cc47..5564e2cdffc0e 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java @@ -19,6 +19,7 @@ import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.SourcePage; import java.io.IOException; import java.util.HashMap; @@ -84,9 +85,9 @@ public boolean isFinished() } @Override - public Page getNextPage() + public SourcePage getNextSourcePage() { - Page physicalSourcePage = dataPageSource.getNextPage(); + SourcePage physicalSourcePage = dataPageSource.getNextSourcePage(); if (physicalSourcePage == null) { return null; } @@ -97,6 +98,11 @@ public Page getNextPage() return physicalSourcePage; } + if (allOutputColumns.isEmpty()) { + // Forward the zero-block page so positionCount survives -- new Page(new Block[0]) would infer positionCount=0. + return physicalSourcePage; + } + Block[] outputBlocks = new Block[allOutputColumns.size()]; for (int i = 0; i < allOutputColumns.size(); i++) { HiveColumnHandle outputColumn = allOutputColumns.get(i); @@ -108,7 +114,7 @@ public Page getNextPage() outputBlocks[i] = synthesizedColumnHandler.createRleSynthesizedBlock(outputColumn, positionCount); } } - return new Page(outputBlocks); + return SourcePage.create(new Page(outputBlocks)); } @Override diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java similarity index 95% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java index 2355744b6976b..8ecb76eeda9d8 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java @@ -38,6 +38,7 @@ public class HudiConfig { private List columnsToHide = ImmutableList.of(); + private List recordMergerImpls = ImmutableList.of(); private boolean tableStatisticsEnabled = true; private int tableStatisticsExecutorParallelism = 4; private boolean metadataEnabled = true; @@ -55,7 +56,7 @@ public class HudiConfig private boolean queryPartitionFilterRequired; private boolean ignoreAbsentPartitions; private Duration dynamicFilteringWaitTimeout = new Duration(1, SECONDS); - private boolean resolveColumnNameCasingEnabled = true; + private boolean resolveColumnNameCasingEnabled; // Internal configuration for debugging and testing private boolean isRecordLevelIndexEnabled = true; @@ -84,6 +85,21 @@ public HudiConfig setColumnsToHide(List columnsToHide) return this; } + public List getRecordMergerImpls() + { + return recordMergerImpls; + } + + @Config("hudi.record-merger-impls") + @ConfigDescription("Comma-separated list of fully qualified HoodieRecordMerger implementation class names used to " + + "resolve a custom record merger for Merge-On-Read tables whose record merge mode is CUSTOM. " + + "The merger must produce Avro records (HoodieRecordType.AVRO). By default, no custom mergers are registered.") + public HudiConfig setRecordMergerImpls(List recordMergerImpls) + { + this.recordMergerImpls = ImmutableList.copyOf(recordMergerImpls); + return this; + } + @Config("hudi.table-statistics-enabled") @ConfigDescription("Enable table statistics for query planning.") public HudiConfig setTableStatisticsEnabled(boolean tableStatisticsEnabled) diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnector.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConnector.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnector.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConnector.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java similarity index 79% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java index 0db65192baf8f..cb5a565691c48 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java @@ -22,18 +22,14 @@ import io.airlift.bootstrap.LifeCycleManager; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.manager.FileSystemModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.base.session.SessionPropertiesProvider; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.HiveMetastoreModule; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -41,7 +37,6 @@ import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -80,18 +75,11 @@ public static Connector createConnector( new MBeanModule(), new JsonModule(), new HudiModule(), - new HiveMetastoreModule(Optional.empty()), + new HiveMetastoreModule(Optional.empty(), false), new HudiFileSystemModule(catalogName, context), new MBeanServerModule(), - module.orElse(EMPTY_MODULE), - binder -> { - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion())); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context), + module.orElse(EMPTY_MODULE)); Injector injector = app .doNotInitializeLogging() @@ -123,21 +111,19 @@ private static class HudiFileSystemModule extends AbstractConfigurationAwareModule { private final String catalogName; - private final NodeManager nodeManager; - private final OpenTelemetry openTelemetry; + private final ConnectorContext context; public HudiFileSystemModule(String catalogName, ConnectorContext context) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); - this.nodeManager = context.getNodeManager(); - this.openTelemetry = context.getOpenTelemetry(); + this.context = requireNonNull(context, "context is null"); } @Override protected void setup(Binder binder) { boolean metadataCacheEnabled = buildConfigObject(HudiConfig.class).isMetadataCacheEnabled(); - install(new FileSystemModule(catalogName, nodeManager, openTelemetry, metadataCacheEnabled)); + install(new FileSystemModule(catalogName, context, metadataCacheEnabled)); } } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiMetadata.java similarity index 94% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiMetadata.java index 917c94ea43d9f..65b385f70d98d 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiMetadata.java @@ -38,6 +38,7 @@ import io.trino.spi.connector.ConnectorTableVersion; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.ConstraintApplicationResult; +import io.trino.spi.connector.LimitApplicationResult; import io.trino.spi.connector.RelationColumnsMetadata; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SchemaTablePrefix; @@ -47,8 +48,8 @@ import io.trino.spi.statistics.Estimate; import io.trino.spi.statistics.TableStatistics; import io.trino.spi.type.TypeManager; -import org.apache.avro.Schema; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2; @@ -56,7 +57,7 @@ import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; import org.apache.hudi.metadata.MetadataPartitionType; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.Collection; import java.util.Collections; @@ -64,6 +65,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -91,6 +93,8 @@ import static io.trino.plugin.hudi.HudiTableProperties.PARTITIONED_BY_PROPERTY; import static io.trino.plugin.hudi.HudiUtil.buildTableMetaClient; import static io.trino.plugin.hudi.HudiUtil.getLatestTableSchema; +import static io.trino.plugin.hudi.HudiSessionProperties.getRecordMergerImpls; +import static io.trino.plugin.hudi.HudiUtil.getMergeRequiredColumnHandles; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.StandardErrorCode.QUERY_REJECTED; import static io.trino.spi.StandardErrorCode.UNSUPPORTED_TABLE_TYPE; @@ -162,7 +166,7 @@ public HudiTableHandle getTableHandle(ConnectorSession session, SchemaTableName String inputFormat = table.getStorage().getStorageFormat().getInputFormat(); HoodieTableType hoodieTableType = HudiTableTypeUtils.fromInputFormat(inputFormat); Lazy lazyMetaClient = Lazy.lazily(() -> buildTableMetaClient(fileSystem, tableName.toString(), basePath)); - Optional> hudiTableSchema = isResolveColumnNameCasingEnabled(session) ? + Optional> hudiTableSchema = isResolveColumnNameCasingEnabled(session) ? Optional.of(Lazy.lazily(() -> getLatestTableSchema(lazyMetaClient.get(), tableName.getTableName()))) : Optional.empty(); return new HudiTableHandle( @@ -173,9 +177,11 @@ public HudiTableHandle getTableHandle(ConnectorSession session, SchemaTableName table.getStorage().getLocation(), hoodieTableType, getPartitionKeyColumnHandles(table, typeManager), + Lazy.lazily(() -> getMergeRequiredColumnHandles(table, typeManager, lazyMetaClient, getRecordMergerImpls(session), NANOSECONDS)), ImmutableSet.of(), TupleDomain.all(), TupleDomain.all(), + OptionalLong.empty(), hudiTableSchema); } @@ -259,6 +265,22 @@ public Optional> applyFilter(C false)); } + @Override + public Optional> applyLimit(ConnectorSession session, ConnectorTableHandle handle, long limit) + { + HudiTableHandle table = (HudiTableHandle) handle; + + if (table.getLimit().isPresent() && table.getLimit().getAsLong() <= limit) { + return Optional.empty(); + } + + // limitGuaranteed=false: the connector can't bound row count across splits without + // coordinator-side coordination. Trino keeps the Limit operator above the TableScan. + // The stored limit is consumed by HudiSplitSource for split-listing short-circuit when + // row-count estimates from the Hudi metadata table become available (TODO). + return Optional.of(new LimitApplicationResult<>(table.withLimit(limit), false, false)); + } + @Override public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { @@ -276,7 +298,7 @@ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTable } @Override - public Optional getInfo(ConnectorTableHandle tableHandle) + public Optional getInfo(ConnectorSession session, ConnectorTableHandle tableHandle) { HudiTableHandle table = (HudiTableHandle) tableHandle; return Optional.of(new HudiTableInfo(table.getSchemaTableName(), table.getTableType().name(), table.getBasePath())); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiModule.java similarity index 95% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiModule.java index bd3c1923ebadf..cd17429b90d36 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiModule.java @@ -26,7 +26,6 @@ import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.HiveNodePartitioningProvider; import io.trino.plugin.hive.HiveTransactionHandle; -import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; import io.trino.plugin.hudi.cache.HudiCacheKeyProvider; @@ -44,7 +43,7 @@ import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.trino.plugin.base.ClosingBinder.closingBinder; +import static io.airlift.bootstrap.ClosingBinder.closingBinder; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -59,7 +58,6 @@ public void configure(Binder binder) configBinder(binder).bindConfig(HudiConfig.class); - binder.bind(boolean.class).annotatedWith(TranslateHiveViews.class).toInstance(false); binder.bind(boolean.class).annotatedWith(HideDeltaLakeTables.class).toInstance(false); newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(HudiSessionProperties.class).in(Scopes.SINGLETON); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java similarity index 60% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java index e81887414425b..1d5eb13be1c87 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java @@ -19,10 +19,13 @@ import io.trino.plugin.hudi.util.SynthesizedColumnHandler; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.SourcePage; import io.trino.spi.metrics.Metrics; import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.util.collection.ClosableIterator; import java.io.IOException; import java.util.List; @@ -30,6 +33,8 @@ import java.util.concurrent.CompletableFuture; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Throwables.getCausalChain; +import static com.google.common.base.Throwables.throwIfUnchecked; public class HudiPageSource implements ConnectorPageSource @@ -41,6 +46,7 @@ public class HudiPageSource PageBuilder pageBuilder; HudiAvroSerializer avroSerializer; List columnHandles; + ClosableIterator recordIterator; public HudiPageSource( ConnectorPageSource pageSource, @@ -51,11 +57,41 @@ public HudiPageSource( { this.pageSource = pageSource; this.fileGroupReader = fileGroupReader; - this.initFileGroupReader(); this.readerContext = readerContext; this.columnHandles = columnHandles; this.pageBuilder = new PageBuilder(columnHandles.stream().map(HiveColumnHandle::getType).toList()); this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler); + try { + this.recordIterator = fileGroupReader.getClosableIterator(); + } + catch (Throwable e) { + // Hudi's log scanning wraps failures in a generic HoodieException ("Exception when + // reading log file"), which buries connector errors; if the cause chain holds a + // TrinoException, throw that one instead so its error code and actionable message + // surface as the query failure. + Throwable toThrow = getCausalChain(e).stream() + .filter(TrinoException.class::isInstance) + .findFirst() + .orElse(e); + // getClosableIterator() can fail with checked (IOException) or unchecked + // (HoodieIOException, NPE/IAE from schema/file validation) exceptions; clean up + // in all cases so we don't leak the reader/page-source handles. + try { + fileGroupReader.close(); + } + catch (Exception closeException) { + toThrow.addSuppressed(closeException); + } + try { + pageSource.close(); + } + catch (Exception closeException) { + toThrow.addSuppressed(closeException); + } + // Preserve the original exception type (RuntimeException/Error) instead of masking it. + throwIfUnchecked(toThrow); + throw new RuntimeException("Failed to initialize file group reader!", toThrow); + } } @Override @@ -79,30 +115,20 @@ public long getReadTimeNanos() @Override public boolean isFinished() { - try { - return !fileGroupReader.hasNext(); - } - catch (IOException e) { - throw new RuntimeException(e); - } + return !recordIterator.hasNext(); } @Override - public Page getNextPage() + public SourcePage getNextSourcePage() { checkState(pageBuilder.isEmpty(), "PageBuilder is not empty at the beginning of a new page"); - try { - while (fileGroupReader.hasNext()) { - avroSerializer.buildRecordInPage(pageBuilder, fileGroupReader.next()); - } - } - catch (IOException e) { - throw new RuntimeException(e); + while (recordIterator.hasNext()) { + avroSerializer.buildRecordInPage(pageBuilder, recordIterator.next()); } Page newPage = pageBuilder.build(); pageBuilder.reset(); - return newPage; + return SourcePage.create(newPage); } @Override @@ -115,8 +141,10 @@ public long getMemoryUsage() public void close() throws IOException { - fileGroupReader.close(); - pageSource.close(); + // recordIterator is the outermost wrapper; closing it cascades down through the file + // group reader to the underlying Trino pageSource, releasing each resource exactly once. + // Closing fileGroupReader/pageSource here too would double-close the same handles. + recordIterator.close(); } @Override @@ -130,14 +158,4 @@ public Metrics getMetrics() { return pageSource.getMetrics(); } - - protected void initFileGroupReader() - { - try { - this.fileGroupReader.initRecordIterators(); - } - catch (IOException e) { - throw new RuntimeException("Failed to initialize file group reader!", e); - } - } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java similarity index 78% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index 9b8411fdf9079..b8d0cec0de8ff 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -34,12 +34,9 @@ import io.trino.parquet.reader.RowGroupInfo; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hive.ReaderColumns; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hudi.file.HudiBaseFile; import io.trino.plugin.hudi.reader.HudiTrinoReaderContext; -import io.trino.plugin.hudi.storage.HudiTrinoStorage; -import io.trino.plugin.hudi.storage.TrinoStorageConfiguration; import io.trino.plugin.hudi.util.SynthesizedColumnHandler; import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; @@ -54,11 +51,12 @@ import io.trino.spi.predicate.TupleDomain; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.read.HoodieFileGroupReader; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.storage.StoragePath; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.io.MessageColumnIO; @@ -68,7 +66,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -82,8 +79,6 @@ import static io.trino.parquet.ParquetTypeUtils.getDescriptors; import static io.trino.parquet.predicate.PredicateUtils.buildPredicate; import static io.trino.parquet.predicate.PredicateUtils.getFilteredRowGroups; -import static io.trino.plugin.hive.HiveColumnHandle.partitionColumnHandle; -import static io.trino.plugin.hive.HivePageSourceProvider.projectBaseColumns; import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.ParquetReaderProvider; import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createDataSource; import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createParquetPageSource; @@ -95,6 +90,7 @@ import static io.trino.plugin.hudi.HudiSessionProperties.getParquetMaxReadBlockRowCount; import static io.trino.plugin.hudi.HudiSessionProperties.getParquetMaxReadBlockSize; import static io.trino.plugin.hudi.HudiSessionProperties.getParquetSmallFileThreshold; +import static io.trino.plugin.hudi.HudiSessionProperties.getRecordMergerImpls; import static io.trino.plugin.hudi.HudiSessionProperties.isParquetIgnoreStatistics; import static io.trino.plugin.hudi.HudiSessionProperties.isParquetUseColumnIndex; import static io.trino.plugin.hudi.HudiSessionProperties.isParquetVectorizedDecodingEnabled; @@ -104,10 +100,10 @@ import static io.trino.plugin.hudi.HudiUtil.constructSchema; import static io.trino.plugin.hudi.HudiUtil.convertToFileSlice; import static io.trino.plugin.hudi.HudiUtil.getLatestTableSchema; -import static io.trino.plugin.hudi.HudiUtil.prependHudiMetaColumns; +import static io.trino.plugin.hudi.HudiUtil.prependHudiMetaAndMergeRequiredColumns; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toUnmodifiableList; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; public class HudiPageSourceProvider implements ConnectorPageSourceProvider @@ -175,7 +171,7 @@ public ConnectorPageSource createPageSource( // Enable predicate pushdown for splits containing only base files boolean isBaseFileOnly = hudiSplit.getLogFiles().isEmpty(); // Convert columns to HiveColumnHandles - List hiveColumnHandles = getHiveColumns(columns, isBaseFileOnly); + List hiveColumnHandles = getHiveColumns(columns); // Get non-synthesized columns (columns that are available in data file) List dataColumnHandles = hiveColumnHandles.stream() @@ -184,31 +180,35 @@ public ConnectorPageSource createPageSource( // The `columns` list could be empty when count(*) is issued, // prepending hoodie meta columns for Hudi split with log files // to allow a non-empty dataPageSource to be returned - List hudiMetaAndDataColumnHandles = prependHudiMetaColumns(dataColumnHandles); + List hudiMetaAndDataColumnHandles = prependHudiMetaAndMergeRequiredColumns(hudiTableHandle, dataColumnHandles); TrinoFileSystem fileSystem = fileSystemFactory.create(session); + ParquetReaderOptions sessionOptions = ParquetReaderOptions.builder(options) + .withIgnoreStatistics(isParquetIgnoreStatistics(session)) + .withMaxReadBlockSize(getParquetMaxReadBlockSize(session)) + .withMaxReadBlockRowCount(getParquetMaxReadBlockRowCount(session)) + .withSmallFileThreshold(getParquetSmallFileThreshold(session)) + .withUseColumnIndex(isParquetUseColumnIndex(session)) + .withBloomFilter(useParquetBloomFilter(session)) + .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)) + .build(); ConnectorPageSource dataPageSource = createPageSource( session, isBaseFileOnly ? dataColumnHandles : hudiMetaAndDataColumnHandles, hudiSplit, fileSystem.newInputFile(Location.of(hudiBaseFileOpt.get().getPath()), hudiBaseFileOpt.get().getFileSize()), + hudiBaseFileOpt.get().getPath(), + start, + length, + OptionalLong.of(hudiBaseFileOpt.get().getFileSize()), dataSourceStats, - options - .withIgnoreStatistics(isParquetIgnoreStatistics(session)) - .withMaxReadBlockSize(getParquetMaxReadBlockSize(session)) - .withMaxReadBlockRowCount(getParquetMaxReadBlockRowCount(session)) - .withSmallFileThreshold(getParquetSmallFileThreshold(session)) - .withUseColumnIndex(isParquetUseColumnIndex(session)) - .withBloomFilter(useParquetBloomFilter(session)) - .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)), + sessionOptions, timeZone, dynamicFilter, isBaseFileOnly); SynthesizedColumnHandler synthesizedColumnHandler = SynthesizedColumnHandler.create(hudiSplit); // Avoid avro serialization if split/filegroup only contains base files if (isBaseFileOnly) { - ValidationUtils.checkArgument(!hiveColumnHandles.isEmpty(), - "Column handles should always be present for providing Hudi data page source on a base file"); return new HudiBaseFileOnlyPageSource( dataPageSource, hiveColumnHandles, @@ -219,34 +219,52 @@ public ConnectorPageSource createPageSource( // TODO: Move this into HudiTableHandle HoodieTableMetaClient metaClient = buildTableMetaClient( fileSystemFactory.create(session), hudiTableHandle.getSchemaTableName().toString(), hudiTableHandle.getBasePath()); - + // Build native (RFC-103) delta-log parquet page sources on demand, projected on the file-group + // reader's requiredSchema with predicate pushdown OFF so every log record is read and merged. + HudiTrinoReaderContext.LogFileParquetPageSourceFactory logPageSourceFactory = + (logPath, logStart, logLength, projection) -> createPageSource( + session, + projection, + hudiSplit, + fileSystem.newInputFile(Location.of(logPath)), + logPath, + logStart, + logLength, + OptionalLong.empty(), + dataSourceStats, + sessionOptions, + timeZone, + DynamicFilter.EMPTY, + false); HudiTrinoReaderContext readerContext = new HudiTrinoReaderContext( + metaClient.getStorageConf(), + metaClient.getTableConfig(), dataPageSource, dataColumnHandles, hudiMetaAndDataColumnHandles, - synthesizedColumnHandler); - Schema dataSchema = + synthesizedColumnHandler, + logPageSourceFactory); + HoodieSchema dataSchema = Optional.ofNullable(hudiTableHandle.getTableSchema()) .orElseGet(() -> getLatestTableSchema(metaClient, hudiTableHandle.getTableName())); - // Construct an Avro schema for log file reader - Schema requestedSchema = constructSchema(dataSchema, hudiMetaAndDataColumnHandles.stream().map(HiveColumnHandle::getName).toList()); + Schema requestedSchema = constructSchema(dataSchema.toAvroSchema(), hudiMetaAndDataColumnHandles.stream().map(HiveColumnHandle::getName).toList()); + FileSlice fileSlice = convertToFileSlice(hudiSplit, hudiTableHandle.getBasePath()); HoodieFileGroupReader fileGroupReader = - new HoodieFileGroupReader<>( - readerContext, - new HudiTrinoStorage(fileSystemFactory.create(session), new TrinoStorageConfiguration()), - hudiTableHandle.getBasePath(), - hudiTableHandle.getLatestCommitTime(), - convertToFileSlice(hudiSplit, hudiTableHandle.getBasePath()), - dataSchema, - requestedSchema, - Option.empty(), - metaClient, - metaClient.getTableConfig().getProps(), - start, - length, - false); - + HoodieFileGroupReader.builder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(metaClient) + .withBaseFileOption(fileSlice.getBaseFile()) + .withLogFiles(fileSlice.getLogFiles()) + .withPartitionPath(fileSlice.getPartitionPath()) + .withDataSchema(dataSchema) + .withRequestedSchema(HoodieSchema.fromAvroSchema(requestedSchema)) + .withLatestCommitTime(hudiTableHandle.getLatestCommitTime()) + .withProps(buildReaderProperties(session, metaClient)) + .withShouldUseRecordPosition(false) + .withStart(start) + .withLength(length) + .build(); return new HudiPageSource( dataPageSource, fileGroupReader, @@ -255,11 +273,32 @@ public ConnectorPageSource createPageSource( synthesizedColumnHandler); } + /** + * Builds the properties passed to the {@link HoodieFileGroupReader}, starting from the persisted table config + * and layering in any custom record merger implementation classes configured on the connector or session. + * The merger impl classes are a read/write config and are not persisted in {@code hoodie.properties}, so they + * must be supplied here for {@link HudiTrinoReaderContext#getRecordMerger} to resolve a CUSTOM record merger. + */ + private static TypedProperties buildReaderProperties(ConnectorSession session, HoodieTableMetaClient metaClient) + { + TypedProperties props = new TypedProperties(); + TypedProperties.putAll(props, metaClient.getTableConfig().getProps()); + List recordMergerImpls = getRecordMergerImpls(session); + if (!recordMergerImpls.isEmpty()) { + props.setProperty(RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY, String.join(",", recordMergerImpls)); + } + return props; + } + static ConnectorPageSource createPageSource( ConnectorSession session, List columns, HudiSplit hudiSplit, TrinoInputFile inputFile, + String path, + long start, + long length, + OptionalLong estimatedFileSize, FileFormatDataSourceStats dataSourceStats, ParquetReaderOptions options, DateTimeZone timeZone, @@ -268,13 +307,9 @@ static ConnectorPageSource createPageSource( { ParquetDataSource dataSource = null; boolean useColumnNames = shouldUseParquetColumnNames(session); - HudiBaseFile baseFile = hudiSplit.getBaseFile().get(); - String path = baseFile.getPath(); - long start = baseFile.getStart(); - long length = baseFile.getLength(); try { AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); - dataSource = createDataSource(inputFile, OptionalLong.of(baseFile.getFileSize()), options, memoryContext, dataSourceStats); + dataSource = createDataSource(inputFile, estimatedFileSize, options, memoryContext, dataSourceStats); ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); @@ -311,17 +346,12 @@ static ConnectorPageSource createPageSource( DOMAIN_COMPACTION_THRESHOLD, options); - Optional readerProjections = projectBaseColumns(columns); - List baseColumns = readerProjections.map(projection -> - projection.get().stream() - .map(HiveColumnHandle.class::cast) - .collect(toUnmodifiableList())) - .orElse(columns); ParquetDataSourceId dataSourceId = dataSource.getId(); ParquetDataSource finalDataSource = dataSource; - ParquetReaderProvider parquetReaderProvider = fields -> new ParquetReader( + ParquetReaderProvider parquetReaderProvider = (fields, appendRowNumberColumn) -> new ParquetReader( Optional.ofNullable(fileMetaData.getCreatedBy()), fields, + appendRowNumberColumn, rowGroups, finalDataSource, timeZone, @@ -329,8 +359,9 @@ static ConnectorPageSource createPageSource( options, exception -> handleException(dataSourceId, exception), Optional.of(parquetPredicate), - Optional.empty()); - return createParquetPageSource(baseColumns, fileSchema, messageColumn, useColumnNames, parquetReaderProvider); + Optional.empty(), + parquetMetadata.getDecryptionContext()); + return createParquetPageSource(columns, fileSchema, messageColumn, useColumnNames, parquetReaderProvider); } catch (IOException | RuntimeException e) { try { @@ -427,18 +458,10 @@ private static TupleDomain getCombinedPredicate(HudiSplit hudi return combinedPredicate; } - private static List getHiveColumns(List columns, - boolean isBaseFileOnly) + private static List getHiveColumns(List columns) { - if (!isBaseFileOnly || !columns.isEmpty()) { - return columns.stream() - .map(HiveColumnHandle.class::cast) - .toList(); - } - - // The `columns` list containing the requested columns to read could be empty - // when count(*) is in the statement; to make sure the page source works properly, - // the synthesized partition column is added in this case. - return Collections.singletonList(partitionColumnHandle()); + return columns.stream() + .map(HiveColumnHandle.class::cast) + .toList(); } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPlugin.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPlugin.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPlugin.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPlugin.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPredicates.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPredicates.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPredicates.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPredicates.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java similarity index 95% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java index e3645e0582c2f..32ac46a95977f 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java @@ -44,6 +44,7 @@ public class HudiSessionProperties implements SessionPropertiesProvider { private static final String COLUMNS_TO_HIDE = "columns_to_hide"; + static final String RECORD_MERGER_IMPLS = "record_merger_impls"; static final String TABLE_STATISTICS_ENABLED = "table_statistics_enabled"; static final String METADATA_TABLE_ENABLED = "metadata_enabled"; private static final String USE_PARQUET_COLUMN_NAMES = "use_parquet_column_names"; @@ -93,6 +94,18 @@ public HudiSessionProperties(HudiConfig hudiConfig, ParquetReaderConfig parquetR .map(name -> ((String) name).toLowerCase(ENGLISH)) .collect(toImmutableList()), value -> value), + new PropertyMetadata<>( + RECORD_MERGER_IMPLS, + "Fully qualified HoodieRecordMerger implementation class names used to resolve a custom record " + + "merger for Merge-On-Read tables whose record merge mode is CUSTOM", + new ArrayType(VARCHAR), + List.class, + hudiConfig.getRecordMergerImpls(), + false, + value -> ((Collection) value).stream() + .map(String.class::cast) + .collect(toImmutableList()), + value -> value), booleanProperty( TABLE_STATISTICS_ENABLED, "Expose table statistics", @@ -265,6 +278,12 @@ public static List getColumnsToHide(ConnectorSession session) return (List) session.getProperty(COLUMNS_TO_HIDE, List.class); } + @SuppressWarnings("unchecked") + public static List getRecordMergerImpls(ConnectorSession session) + { + return (List) session.getProperty(RECORD_MERGER_IMPLS, List.class); + } + public static boolean isTableStatisticsEnabled(ConnectorSession session) { return session.getProperty(TABLE_STATISTICS_ENABLED, Boolean.class); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplit.java similarity index 99% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplit.java index e2de10ca98a4d..32f1d32aa3f64 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplit.java @@ -80,7 +80,6 @@ public HudiSplit( this.cachingHostAddresses = requireNonNull(cachingHostAddresses, "cachingHostAddresses is null"); } - @Override public Map getSplitInfo() { return ImmutableMap.builder() diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java similarity index 93% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java index 79f03ccd07b27..7084435b8733d 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java @@ -17,7 +17,6 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import io.airlift.log.Logger; -import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.metastore.HiveMetastore; import io.trino.metastore.Partition; import io.trino.metastore.StorageFormat; @@ -35,7 +34,7 @@ import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.security.ConnectorIdentity; import org.apache.hudi.common.util.HoodieTimer; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Map; @@ -61,19 +60,16 @@ public class HudiSplitManager private final BiFunction metastoreProvider; private final ExecutorService executor; private final ScheduledExecutorService splitLoaderExecutorService; - private final CachingHostAddressProvider cachingHostAddressProvider; @Inject public HudiSplitManager( BiFunction metastoreProvider, @ForHudiSplitManager ExecutorService executor, - @ForHudiSplitSource ScheduledExecutorService splitLoaderExecutorService, - CachingHostAddressProvider cachingHostAddressProvider) + @ForHudiSplitSource ScheduledExecutorService splitLoaderExecutorService) { this.metastoreProvider = requireNonNull(metastoreProvider, "metastoreProvider is null"); this.executor = requireNonNull(executor, "executor is null"); this.splitLoaderExecutorService = requireNonNull(splitLoaderExecutorService, "splitLoaderExecutorService is null"); - this.cachingHostAddressProvider = requireNonNull(cachingHostAddressProvider, "cachingHostAddressProvider is null"); } @Override @@ -103,8 +99,7 @@ public ConnectorSplitSource getSplits( getMaxOutstandingSplits(session), lazyAllPartitions, dynamicFilter, - getDynamicFilteringWaitTimeout(session), - cachingHostAddressProvider); + getDynamicFilteringWaitTimeout(session)); return new ClassLoaderSafeConnectorSplitSource(splitSource, HudiSplitManager.class.getClassLoader()); } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java similarity index 94% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java index 278ca2e463c7e..ab6bae93cfc81 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java @@ -19,7 +19,6 @@ import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; -import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.metastore.Partition; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HivePartitionKey; @@ -45,7 +44,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.metadata.HoodieTableMetadata; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.metadata.NativeTableMetadataFactory; +import org.apache.hudi.common.util.Lazy; import java.util.HashMap; import java.util.List; @@ -92,8 +92,7 @@ public HudiSplitSource( int maxOutstandingSplits, Lazy> lazyPartitions, DynamicFilter dynamicFilter, - Duration dynamicFilteringWaitTimeoutMillis, - CachingHostAddressProvider cachingHostAddressProvider) + Duration dynamicFilteringWaitTimeoutMillis) { boolean enableMetadataTable = isHudiMetadataTableEnabled(session); Lazy lazyTableMetadata = Lazy.lazily(() -> { @@ -104,9 +103,11 @@ public HudiSplitSource( HoodieTableMetaClient metaClient = tableHandle.getMetaClient(); HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorage().getConf()); - HoodieTableMetadata tableMetadata = HoodieTableMetadata.create( - engineContext, - tableHandle.getMetaClient().getStorage(), metadataConfig, metaClient.getBasePath().toString(), true); + // Defer to the native factory, which creates a HoodieBackedTableMetadata when the + // metadata table is enabled and initialized, and falls back to FileSystemBackedTableMetadata + // otherwise. + HoodieTableMetadata tableMetadata = NativeTableMetadataFactory.getInstance().create( + engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString(), true); log.info("Loaded table metadata for table: %s in %s ms", tableHandle.getSchemaTableName(), timer.endTimer()); return tableMetadata; }); @@ -128,7 +129,6 @@ public HudiSplitSource( lazyPartitions, enableMetadataTable, lazyTableMetadata, - cachingHostAddressProvider, throwable -> { trinoException.compareAndSet(null, new TrinoException(HUDI_CANNOT_OPEN_SPLIT, "Failed to generate splits for " + tableHandle.getSchemaTableName(), throwable)); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java similarity index 77% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java index 75ae962286a41..0e9ce55f65a25 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java @@ -24,15 +24,16 @@ import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.TupleDomain; -import org.apache.avro.Schema; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.StringUtils; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.function.Supplier; @@ -49,11 +50,13 @@ public class HudiTableHandle private final String basePath; private final HoodieTableType tableType; private final List partitionColumns; + private final Lazy> lazyMergeRequiredColumns; // Used only for validation when config property hudi.query-partition-filter-required is enabled private final Set constraintColumns; private final TupleDomain partitionPredicates; private final TupleDomain regularPredicates; - private final Optional> hudiTableSchema; + private final OptionalLong limit; + private final Optional> hudiTableSchema; // Coordinator-only private final transient Optional

    table; private final transient Optional> lazyMetaClient; @@ -66,13 +69,15 @@ public HudiTableHandle( @JsonProperty("basePath") String basePath, @JsonProperty("tableType") HoodieTableType tableType, @JsonProperty("partitionColumns") List partitionColumns, + @JsonProperty("mergeRequiredColumns") List mergeRequiredColumns, @JsonProperty("partitionPredicates") TupleDomain partitionPredicates, @JsonProperty("regularPredicates") TupleDomain regularPredicates, + @JsonProperty("limit") OptionalLong limit, @JsonProperty("tableSchemaStr") String tableSchemaStr, @JsonProperty("latestCommitTime") String latestCommitTime) { - this(Optional.empty(), Optional.empty(), schemaName, tableName, basePath, tableType, partitionColumns, ImmutableSet.of(), - partitionPredicates, regularPredicates, buildTableSchema(tableSchemaStr), () -> latestCommitTime); + this(Optional.empty(), Optional.empty(), schemaName, tableName, basePath, tableType, partitionColumns, Lazy.eagerly(mergeRequiredColumns), ImmutableSet.of(), + partitionPredicates, regularPredicates, limit, buildTableSchema(tableSchemaStr), () -> latestCommitTime); } public HudiTableHandle( @@ -83,10 +88,12 @@ public HudiTableHandle( String basePath, HoodieTableType tableType, List partitionColumns, + Lazy> lazyMergeRequiredColumns, Set constraintColumns, TupleDomain partitionPredicates, TupleDomain regularPredicates, - Optional> hudiTableSchema) + OptionalLong limit, + Optional> hudiTableSchema) { this( Optional.of(table), @@ -96,9 +103,11 @@ public HudiTableHandle( basePath, tableType, partitionColumns, + lazyMergeRequiredColumns, constraintColumns, partitionPredicates, regularPredicates, + limit, hudiTableSchema, () -> lazyMetaClient .get() @@ -120,10 +129,12 @@ public HudiTableHandle( String basePath, HoodieTableType tableType, List partitionColumns, + Lazy> lazyMergeRequiredColumns, Set constraintColumns, TupleDomain partitionPredicates, TupleDomain regularPredicates, - Optional> hudiTableSchema, + OptionalLong limit, + Optional> hudiTableSchema, Supplier latestCommitTimeSupplier) { this.table = requireNonNull(table, "table is null"); @@ -133,27 +144,29 @@ public HudiTableHandle( this.basePath = requireNonNull(basePath, "basePath is null"); this.tableType = requireNonNull(tableType, "tableType is null"); this.partitionColumns = requireNonNull(partitionColumns, "partitionColumns is null"); + this.lazyMergeRequiredColumns = requireNonNull(lazyMergeRequiredColumns, "lazyMergeRequiredColumns is null"); this.constraintColumns = requireNonNull(constraintColumns, "constraintColumns is null"); this.partitionPredicates = requireNonNull(partitionPredicates, "partitionPredicates is null"); this.regularPredicates = requireNonNull(regularPredicates, "regularPredicates is null"); + this.limit = requireNonNull(limit, "limit is null"); this.hudiTableSchema = requireNonNull(hudiTableSchema, "hudiTableSchema is null"); this.lazyLatestCommitTime = Lazy.lazily(latestCommitTimeSupplier); } /** - * Builds a lazily-parsed Avro schema from the given schema string. + * Builds a lazily-parsed schema from the given Avro schema JSON string. *

    * Returns {@code Optional.empty()} if the input string is null/empty * or if parsing the schema fails. */ - private static Optional> buildTableSchema(String tableSchemaStr) + private static Optional> buildTableSchema(String tableSchemaStr) { if (StringUtils.isNullOrEmpty(tableSchemaStr)) { return Optional.empty(); } try { - Lazy lazySchema = Lazy.lazily(() -> new Schema.Parser().parse(tableSchemaStr)); + Lazy lazySchema = Lazy.lazily(() -> HoodieSchema.parse(tableSchemaStr)); return Optional.of(lazySchema); } catch (Exception e) { @@ -225,12 +238,12 @@ public String getTableSchemaStr() { return hudiTableSchema .map(Lazy::get) - .map(Schema::toString) + .map(HoodieSchema::toString) .orElse(""); } @JsonIgnore - public Schema getTableSchema() + public HoodieSchema getTableSchema() { return hudiTableSchema.map(Lazy::get).orElse(null); } @@ -248,6 +261,23 @@ public TupleDomain getRegularPredicates() return regularPredicates; } + @JsonProperty + public OptionalLong getLimit() + { + return limit; + } + + /** + * Columns that must be read for the file group reader to merge correctly: the ordering columns, plus any + * mandatory fields declared by a configured custom record merger. See + * {@link HudiUtil#getMergeRequiredColumnHandles}. + */ + @JsonProperty + public List getMergeRequiredColumns() + { + return lazyMergeRequiredColumns.get(); + } + public SchemaTableName getSchemaTableName() { return schemaTableName(schemaName, tableName); @@ -266,9 +296,30 @@ HudiTableHandle applyPredicates( basePath, tableType, partitionColumns, + lazyMergeRequiredColumns, constraintColumns, partitionPredicates.intersect(partitionTupleDomain), regularPredicates.intersect(regularTupleDomain), + limit, + hudiTableSchema, + this::getLatestCommitTime); + } + + HudiTableHandle withLimit(long newLimit) + { + return new HudiTableHandle( + table, + lazyMetaClient, + schemaName, + tableName, + basePath, + tableType, + partitionColumns, + lazyMergeRequiredColumns, + constraintColumns, + partitionPredicates, + regularPredicates, + OptionalLong.of(newLimit), hudiTableSchema, this::getLatestCommitTime); } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableName.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableName.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableName.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableName.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java similarity index 65% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java index 5389e80f57187..27484ecc2d1c0 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java @@ -22,10 +22,13 @@ import io.trino.filesystem.FileIterator; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; +import io.trino.metastore.Column; import io.trino.metastore.HivePartition; import io.trino.metastore.HiveType; +import io.trino.metastore.Table; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HivePartitionKey; +import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.avro.AvroHiveFileUtils; import io.trino.plugin.hudi.storage.HudiTrinoStorage; import io.trino.plugin.hudi.storage.TrinoStorageConfiguration; @@ -35,27 +38,42 @@ import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.NullableValue; import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.TypeManager; import io.trino.spi.type.VarcharType; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.data.HoodiePairData; +import org.apache.hudi.common.engine.EngineType; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.CollectionUtils; +import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.TableNotFoundException; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.common.util.Lazy; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,10 +82,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static io.airlift.slice.SizeOf.estimatedSizeOf; +import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; +import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static io.trino.plugin.hive.util.HiveTypeUtil.getType; +import static io.trino.plugin.hive.util.HiveTypeUtil.typeSupported; import static io.trino.plugin.hive.util.HiveUtil.checkCondition; import static io.trino.plugin.hive.util.HiveUtil.parsePartitionValue; import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMNS; @@ -78,10 +99,17 @@ import static io.trino.plugin.hudi.HudiErrorCode.HUDI_SCHEMA_ERROR; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_UNSUPPORTED_FILE_FORMAT; import static java.lang.Math.toIntExact; -import static org.apache.hudi.common.model.HoodieRecord.HOODIE_META_COLUMNS; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; +import static org.apache.hudi.common.model.HoodieRecord.PARTITION_PATH_METADATA_FIELD; +import static org.apache.hudi.common.model.HoodieRecord.RECORD_KEY_METADATA_FIELD; public final class HudiUtil { + // Minimal meta-column subset the file-group reader/merger requires, distinct from the upstream + // 5-field HoodieRecord.HOODIE_META_COLUMNS (commit time, seqno, key, partition path, file name). + public static final List HUDI_REQUIRED_META_COLUMNS = + CollectionUtils.createImmutableList(RECORD_KEY_METADATA_FIELD, PARTITION_PATH_METADATA_FIELD); + private static final Logger log = Logger.get(HudiUtil.class); private static final Cache> SCHEMA_FIELD_CACHE = EvictableCacheBuilder.newBuilder() @@ -291,7 +319,6 @@ private static Map buildFieldLookup(Schema schema) *

  • First, attempts an exact match on the column name.
    • *
  • If not found, falls back to a case-insensitive match using a cached lookup table
    • * - *

    * * @param columnName Column name to search for. * @param schema Avro {@link Schema} in which to search. @@ -321,37 +348,33 @@ public static Schema.Field getFieldFromSchema(String columnName, Schema schema) "Failed to get column " + columnName + " from table schema"); } - public static List prependHudiMetaColumns(List dataColumns) + public static List prependHudiMetaAndMergeRequiredColumns(HudiTableHandle tableHandle, List dataColumns) { - //For efficient lookup - Set dataColumnNames = dataColumns.stream() + Set existingColumns = dataColumns.stream() .map(HiveColumnHandle::getName) - .collect(Collectors.toSet()); - - // If all Hudi meta columns are already present, return the original list - if (dataColumnNames.containsAll(HOODIE_META_COLUMNS)) { - return dataColumns; - } - - // Identify only the meta columns that are missing from dataColumns to avoid duplicates - List missingMetaColumns = HOODIE_META_COLUMNS.stream() - .filter(metaColumn -> !dataColumnNames.contains(metaColumn)) - .toList(); + .collect(Collectors.toCollection(HashSet::new)); List columns = new ArrayList<>(); - // Create and prepend the new HiveColumnHandles for the missing meta columns - columns.addAll(IntStream.range(0, missingMetaColumns.size()) - .boxed() - .map(i -> new HiveColumnHandle( - missingMetaColumns.get(i), + // Add missing Hudi meta columns first + for (int i = 0; i < HUDI_REQUIRED_META_COLUMNS.size(); i++) { + String metaColumn = HUDI_REQUIRED_META_COLUMNS.get(i); + if (existingColumns.add(metaColumn)) { // add() returns false if already present + columns.add(new HiveColumnHandle( + metaColumn, i, HiveType.HIVE_STRING, VarcharType.VARCHAR, Optional.empty(), HiveColumnHandle.ColumnType.REGULAR, - Optional.empty())) - .toList()); + Optional.empty())); + } + } + + // Add missing merge-required columns next (ordering columns, plus a custom merger's mandatory fields) + tableHandle.getMergeRequiredColumns().stream() + .filter(col -> existingColumns.add(col.getName())) + .forEach(columns::add); // Add all the original data columns after the new meta columns columns.addAll(dataColumns); @@ -384,11 +407,11 @@ public static HoodieTableFileSystemView getFileSystemView( tableMetadata, metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); } - public static Schema getLatestTableSchema(HoodieTableMetaClient metaClient, String tableName) + public static HoodieSchema getLatestTableSchema(HoodieTableMetaClient metaClient, String tableName) { try { HoodieTimer timer = HoodieTimer.start(); - Schema schema = new TableSchemaResolver(metaClient).getTableAvroSchema(); + HoodieSchema schema = new TableSchemaResolver(metaClient).getTableSchema(); log.info("Fetched table schema for table %s in %s ms", tableName, timer.endTimer()); return schema; } @@ -397,4 +420,104 @@ public static Schema getLatestTableSchema(HoodieTableMetaClient metaClient, Stri throw new TrinoException(HUDI_FILESYSTEM_ERROR, e); } } + + /** + * Returns the column handles that must be present in the read schema for the file group reader to merge + * correctly: the ordering columns, plus the mandatory merge columns declared by a configured custom record + * merger (via {@link HoodieRecordMerger#getMandatoryFieldsForMerging}). + *

    + * For COMMIT_TIME / EVENT_TIME tables this is exactly the ordering columns (so behavior is unchanged). For a + * CUSTOM merge mode with a registered merger, it additionally includes any data columns the merger reads at + * merge time (e.g. an arbitrary decision column) so that those columns are read from the base file even when + * the query does not project them -- without this the merger would see null for an un-projected column. + */ + public static List getMergeRequiredColumnHandles( + Table table, + TypeManager typeManager, + Lazy lazyMetaClient, + List recordMergerImpls, + HiveTimestampPrecision timestampPrecision) + { + HoodieTableMetaClient metaClient = lazyMetaClient.get(); + HoodieTableConfig tableConfig = metaClient.getTableConfig(); + RecordMergeMode recordMergeMode = tableConfig.getRecordMergeMode(); + + LinkedHashSet requiredColumnNames = new LinkedHashSet<>(); + if (recordMergeMode != null && recordMergeMode != RecordMergeMode.COMMIT_TIME_ORDERING) { + requiredColumnNames.addAll(tableConfig.getOrderingFields()); + } + + // For a CUSTOM merge mode, ask the configured merger which fields it needs at merge time and include them + // so they are read even when not projected. Only the merger's declared columns are added (not all columns), + // so non-custom tables and mergers that only use the key/ordering fields incur no extra reads. + if (recordMergeMode == RecordMergeMode.CUSTOM && recordMergerImpls != null && !recordMergerImpls.isEmpty()) { + Option merger = HoodieRecordUtils.createValidRecordMerger( + EngineType.JAVA, String.join(",", recordMergerImpls), tableConfig.getRecordMergeStrategyId()); + if (merger.isPresent()) { + TypedProperties props = new TypedProperties(); + props.putAll(tableConfig.getProps()); + props.setProperty(RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY, String.join(",", recordMergerImpls)); + HoodieSchema tableSchema; + try { + tableSchema = new TableSchemaResolver(metaClient).getTableSchema(); + } + catch (Exception e) { + throw new TrinoException(HUDI_SCHEMA_ERROR, "Failed to resolve table schema for merge column resolution", e); + } + String[] mandatoryFields = merger.get().getMandatoryFieldsForMerging(tableSchema, tableConfig, props); + if (mandatoryFields != null) { + Collections.addAll(requiredColumnNames, mandatoryFields); + } + } + } + + if (requiredColumnNames.isEmpty()) { + return Collections.emptyList(); + } + return buildColumnHandles(table, typeManager, requiredColumnNames, timestampPrecision); + } + + /** + * Builds {@link HiveColumnHandle}s, preserving physical (data-column) index, for the data columns whose names + * appear in {@code columnNames}. Names that are not data columns (e.g. Hudi meta fields) or whose types are not + * supported by the storage format are skipped. + */ + private static List buildColumnHandles(Table table, TypeManager typeManager, Set columnNames, HiveTimestampPrecision timestampPrecision) + { + ImmutableList.Builder columns = ImmutableList.builder(); + int hiveColumnIndex = 0; + for (Column field : table.getDataColumns()) { + if (columnNames.contains(field.getName())) { + HiveType hiveType = field.getType(); + // ignore unsupported types rather than failing + if (typeSupported(hiveType.getTypeInfo(), table.getStorage().getStorageFormat())) { + columns.add(createBaseColumn(field.getName(), hiveColumnIndex, hiveType, getType(hiveType, typeManager, timestampPrecision), REGULAR, field.getComment())); + } + } + hiveColumnIndex++; + } + return columns.build(); + } + + /** + * Converts the given {@link HoodiePairData} into a {@link Map}. + *

    + * Special handling is applied for null keys: + *

      + *
    • If a key is null, it is stored in the map as a {@code null} entry.
      • + *
    • If multiple entries share the same key (including null), the latest value overwrites the previous one.
      • + *
    + * + * @param pairData the HoodiePairData containing key-value pairs + * @param the type of keys maintained by the resulting map + * @param the type of mapped values + * @return a {@link Map} containing all key-value pairs from the input data + */ + public static Map collectAsMap(HoodiePairData pairData) + { + // HashMap allows null keys, so put directly; on duplicate keys the later entry wins. + Map result = new HashMap<>(); + pairData.collectAsList().forEach(pair -> result.put(pair.getKey(), pair.getValue())); + return result; + } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TableType.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/TableType.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TableType.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/TableType.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TimelineTable.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/TimelineTable.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TimelineTable.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/TimelineTable.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/cache/HudiCacheKeyProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/cache/HudiCacheKeyProvider.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/cache/HudiCacheKeyProvider.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/cache/HudiCacheKeyProvider.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiFile.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiFile.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiFile.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiFile.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java similarity index 55% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java index 7e8bb967ffc72..cf666d5e9883f 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java @@ -13,15 +13,17 @@ */ package io.trino.plugin.hudi.io; -import org.apache.avro.Schema; import org.apache.hudi.common.config.HoodieConfig; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.util.Option; -import org.apache.hudi.io.storage.HoodieAvroBootstrapFileReader; -import org.apache.hudi.io.storage.HoodieFileReader; -import org.apache.hudi.io.storage.HoodieFileReaderFactory; -import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader; +import org.apache.hudi.core.io.storage.HFileReaderFactory; +import org.apache.hudi.core.io.storage.HoodieAvroBootstrapFileReader; +import org.apache.hudi.core.io.storage.HoodieFileReader; +import org.apache.hudi.core.io.storage.HoodieFileReaderFactory; +import org.apache.hudi.core.io.storage.HoodieNativeAvroHFileReader; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import java.io.IOException; @@ -42,10 +44,14 @@ protected HoodieFileReader newParquetFileReader(StoragePath path) @Override protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig, StoragePath path, - Option schemaOption) + Option schemaOption) throws IOException { - return new HoodieNativeAvroHFileReader(storage, path, schemaOption); + HFileReaderFactory readerFactory = HFileReaderFactory.builder() + .withStorage(storage).withProps(hoodieConfig.getProps()) + .withPath(path).build(); + return HoodieNativeAvroHFileReader.builder() + .readerFactory(readerFactory).path(path).schema(schemaOption).build(); } @Override @@ -53,10 +59,25 @@ protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig, StoragePath path, HoodieStorage storage, byte[] content, - Option schemaOption) + Option schemaOption) throws IOException { - return new HoodieNativeAvroHFileReader(this.storage, content, schemaOption); + HFileReaderFactory readerFactory = HFileReaderFactory.builder() + .withStorage(storage).withProps(hoodieConfig.getProps()) + .withContent(content).build(); + return HoodieNativeAvroHFileReader.builder() + .readerFactory(readerFactory).path(path).schema(schemaOption).build(); + } + + @Override + protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig, StoragePathInfo pathInfo, Option schemaOption) + throws IOException + { + HFileReaderFactory readerFactory = HFileReaderFactory.builder() + .withStorage(storage).withProps(hoodieConfig.getProps()) + .withPath(pathInfo.getPath()).build(); + return HoodieNativeAvroHFileReader.builder() + .readerFactory(readerFactory).path(pathInfo.getPath()).schema(schemaOption).build(); } @Override diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java similarity index 80% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java index 8044e92b58ec7..ea9c837d47c31 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java @@ -17,9 +17,9 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.FileFormatUtils; -import org.apache.hudi.io.storage.HoodieFileReaderFactory; -import org.apache.hudi.io.storage.HoodieFileWriterFactory; -import org.apache.hudi.io.storage.HoodieIOFactory; +import org.apache.hudi.core.io.storage.HoodieFileReaderFactory; +import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; +import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; @@ -46,7 +46,11 @@ public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType re @Override public FileFormatUtils getFileFormatUtils(HoodieFileFormat fileFormat) { - throw new UnsupportedOperationException("FileFormatUtils not supported in HudiTrinoIOFactory"); + if (fileFormat == HoodieFileFormat.PARQUET) { + return new HudiTrinoParquetFileFormatUtils(); + } + throw new UnsupportedOperationException( + "Native " + fileFormat + " log files are not supported by the Hudi Trino connector"); } @Override diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoParquetFileFormatUtils.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoParquetFileFormatUtils.java new file mode 100644 index 0000000000000..c0a38189119c9 --- /dev/null +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoParquetFileFormatUtils.java @@ -0,0 +1,210 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.io; + +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoInputFile; +import io.trino.parquet.ParquetDataSource; +import io.trino.parquet.reader.MetadataReader; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; +import io.trino.plugin.hive.parquet.ParquetReaderConfig; +import io.trino.plugin.hudi.storage.HudiTrinoStorage; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.util.FileFormatUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.keygen.BaseKeyGenerator; +import org.apache.hudi.metadata.HoodieIndexVersion; +import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Properties; +import java.util.Set; + +import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createDataSource; + +/** + * {@link FileFormatUtils} for the Hudi Trino connector, backed by Trino's own Parquet reader. + *

    + * The connector reads Parquet data through Trino's engine-native reader rather than Hudi's Hadoop-based + * reader, so {@code hudi-hadoop-common} (and its {@code ParquetUtils}) is deliberately excluded from the + * runtime. The only Hudi read path that still needs a {@link FileFormatUtils} is reading the log-block + * header out of an RFC-103 native (Parquet) delta-log file footer, which goes through + * {@link #readFooter(HoodieStorage, boolean, StoragePath, String...)}. Every other method is unused on the + * read path and throws. + */ +public class HudiTrinoParquetFileFormatUtils + extends FileFormatUtils +{ + private static final String UNSUPPORTED_MESSAGE = + "HudiTrinoParquetFileFormatUtils only supports reading Parquet footer metadata"; + + @Override + public Map readFooter(HoodieStorage storage, boolean required, StoragePath filePath, String... footerNames) + { + Map footerVals = new HashMap<>(); + TrinoFileSystem fileSystem = (TrinoFileSystem) storage.getFileSystem(); + try { + long fileSize = storage.getPathInfo(filePath).getLength(); + TrinoInputFile inputFile = fileSystem.newInputFile(HudiTrinoStorage.convertToLocation(filePath), fileSize); + try (ParquetDataSource dataSource = createDataSource( + inputFile, + OptionalLong.of(fileSize), + new ParquetReaderConfig().toParquetReaderOptions(), + newSimpleAggregatedMemoryContext(), + new FileFormatDataSourceStats())) { + Map metadata = MetadataReader.readFooter(dataSource, Optional.empty()) + .getFileMetaData() + .getKeyValueMetaData(); + for (String footerName : footerNames) { + if (metadata.containsKey(footerName)) { + footerVals.put(footerName, metadata.get(footerName)); + } + else if (required) { + throw new HoodieException("Could not find footer key " + footerName + " in Parquet file " + filePath); + } + } + } + } + catch (IOException e) { + throw new HoodieIOException("Failed to read Parquet footer from " + filePath, e); + } + return footerVals; + } + + @Override + public HoodieFileFormat getFormat() + { + return HoodieFileFormat.PARQUET; + } + + @Override + public List readAvroRecords(HoodieStorage storage, StoragePath filePath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public List readAvroRecords(HoodieStorage storage, StoragePath filePath, HoodieSchema schema) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public long getRowCount(HoodieStorage storage, StoragePath filePath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public Set> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set filter) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public ClosableIterator> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public ClosableIterator getHoodieKeyIterator(HoodieStorage storage, + StoragePath filePath, + Option keyGeneratorOpt, + Option partitionPath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public ClosableIterator getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public ClosableIterator> fetchRecordKeysWithPositions(HoodieStorage storage, + StoragePath filePath, + Option keyGeneratorOpt, + Option partitionPath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public HoodieSchema readSchema(HoodieStorage storage, StoragePath filePath) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + @SuppressWarnings("rawtype") + public List> readColumnStatsFromMetadata(HoodieStorage storage, + StoragePath filePath, + List columnList, + HoodieIndexVersion indexVersion) + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public void writeMetaFile(HoodieStorage storage, StoragePath filePath, Properties props) + throws IOException + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public ByteArrayOutputStream serializeRecordsToLogBlock(HoodieStorage storage, + List records, + HoodieSchema writerSchema, + HoodieSchema readerSchema, + String keyFieldName, + Map paramsMap) + throws IOException + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } + + @Override + public Pair serializeRecordsToLogBlock(HoodieStorage storage, + Iterator records, + HoodieRecord.HoodieRecordType recordType, + HoodieSchema writerSchema, + HoodieSchema readerSchema, + String keyFieldName, + Map paramsMap) + throws IOException + { + throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE); + } +} diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java similarity index 98% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java index f75ec7a55f90b..b67b0012905bc 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java @@ -26,7 +26,7 @@ * Example InlineFS URL: * 

      * inlinefs://tests_7af7f087-c807-4f5e-a759-65fd9c21063b/hudi_multi_fg_pt_v8_mor/.hoodie/metadata/column_stats/
- * .col-stats-0001-0_20250429145946675.log.1_1-120-382/local/?start_offset=8036&length=6959
+ * .col-stats-0001-0_20250429145946675.log.1_1-120-382/local/?start_offset=8036&length=6959
  *
    *

    * Key behaviors: diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java similarity index 99% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java index f177282bf439e..b9a794df871f1 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java @@ -26,7 +26,7 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.metadata.HoodieTableMetadata; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Optional; diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java similarity index 98% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java index 044908f9c49f4..28086ce2fd9ed 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java @@ -18,7 +18,7 @@ import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Map; diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java similarity index 97% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java index 0d1c9eaa6cd06..356e91685204d 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java @@ -30,16 +30,17 @@ import io.trino.spi.type.VarcharType; import org.apache.avro.generic.GenericRecord; import org.apache.hudi.avro.model.HoodieMetadataColumnStats; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.model.BaseFile; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.util.HoodieTimer; -import org.apache.hudi.common.util.hash.ColumnIndexID; +import org.apache.hudi.metadata.ColumnStatsIndexPrefixRawKey; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.ArrayList; import java.util.List; @@ -92,9 +93,9 @@ public HudiColumnStatsIndexSupport(Logger log, ConnectorSession session, SchemaT } else { // Get filter columns - List encodedTargetColumnNames = regularColumns + List rawKeys = regularColumns .stream() - .map(col -> new ColumnIndexID(col).asBase64EncodedString()).collect(Collectors.toList()); + .map(ColumnStatsIndexPrefixRawKey::new).toList(); Map columnTypes = regularColumnPredicates.getDomains().get().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getType())); @@ -107,7 +108,8 @@ public HudiColumnStatsIndexSupport(Logger log, ConnectorSession session, SchemaT } Map> domainsWithStats = - lazyTableMetadata.get().getRecordsByKeyPrefixes(encodedTargetColumnNames, + lazyTableMetadata.get().getRecordsByKeyPrefixes( + HoodieListData.lazy(rawKeys), HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS, true) .collectAsList() .stream() diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java similarity index 96% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java index e79eb342812b8..28b8de82c5a52 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java @@ -17,7 +17,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.TupleDomain; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; /** * Noop index support to ensure that MDT enabled split generation is entered. diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java similarity index 94% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java index 3491d10cccfe4..5b6dc5d233215 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java @@ -21,13 +21,14 @@ import io.trino.spi.predicate.TupleDomain; import io.trino.spi.type.Type; import org.apache.hudi.avro.model.HoodieMetadataColumnStats; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.HoodieTimer; -import org.apache.hudi.common.util.hash.ColumnIndexID; +import org.apache.hudi.metadata.ColumnStatsIndexPrefixRawKey; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.ArrayList; import java.util.List; @@ -66,15 +67,15 @@ public Optional> prunePartitions( List regularColumns = new ArrayList<>(filteredRegularPredicates.getDomains().get().keySet()); // Get columns to filter on - List encodedTargetColumnNames = regularColumns.stream() - .map(col -> new ColumnIndexID(col).asBase64EncodedString()).toList(); + List columnStatsIndexPrefixRawKeys = regularColumns.stream() + .map(ColumnStatsIndexPrefixRawKey::new).toList(); Map columnTypes = regularColumnPredicates.getDomains().get().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getType())); // Map of domains with partition stats keyed by partition name and column name Map> domainsWithStats = lazyMetadataTable.get().getRecordsByKeyPrefixes( - encodedTargetColumnNames, + HoodieListData.eager(columnStatsIndexPrefixRawKeys), HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS, true) .collectAsList() .stream() diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java similarity index 98% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java index 78f503f4b21ea..276d3a56f8819 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java @@ -22,6 +22,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.TupleDomain; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -29,7 +30,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.ArrayList; import java.util.Arrays; @@ -47,6 +48,7 @@ import static io.trino.plugin.hudi.HudiErrorCode.HUDI_BAD_DATA; import static io.trino.plugin.hudi.HudiSessionProperties.getRecordIndexWaitTimeout; +import static io.trino.plugin.hudi.HudiUtil.collectAsMap; import static java.util.concurrent.TimeUnit.MILLISECONDS; public class HudiRecordLevelIndexSupport @@ -94,7 +96,7 @@ public HudiRecordLevelIndexSupport(ConnectorSession session, SchemaTableName sch // Perform index lookup in metadataTable // TODO: document here what this map is keyed by - Map recordIndex = lazyTableMetadata.get().readRecordIndex(recordKeys); + Map recordIndex = collectAsMap(lazyTableMetadata.get().readRecordIndexLocationsWithKeys(HoodieListData.eager(recordKeys))); if (recordIndex.isEmpty()) { log.debug("Record level index lookup took %s ms but returned no locations for the given keys %s for table %s", timer.endTimer(), recordKeys, schemaTableName); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java similarity index 97% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java index 23121902f72c4..7d67d6ab3582d 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java @@ -19,14 +19,16 @@ import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.TupleDomain; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.HoodieDataUtils; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Map; @@ -85,7 +87,7 @@ public HudiSecondaryIndexSupport(ConnectorSession session, SchemaTableName schem // Perform index lookup in metadataTable // TODO: document here what this map is keyed by - Map recordKeyLocationsMap = lazyTableMetadata.get().readSecondaryIndex(secondaryKeys, indexName); + Map recordKeyLocationsMap = HoodieDataUtils.dedupeAndCollectAsMap(lazyTableMetadata.get().readSecondaryIndexLocationsWithKeys(HoodieListData.eager(secondaryKeys), indexName)); if (recordKeyLocationsMap.isEmpty()) { log.debug("Took %s ms, but secondary index lookup returned no locations for the given keys for table %s", timer.endTimer(), schemaTableName); // Return all original fileSlices diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java similarity index 98% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java index 5d98177896c44..ffc07a7db337e 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java @@ -21,7 +21,7 @@ import io.trino.spi.predicate.TupleDomain; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.metadata.HoodieTableMetadata; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.List; import java.util.Optional; @@ -131,7 +131,7 @@ private static TupleDomain transformTupleDomain(ConnectorSession session if (isResolveColumnNameCasingEnabled(session)) { // if column case reconciliation is enabled, transform the tuple domain keys to match the column names from the Hudi table. return tupleDomain.transformKeys(hiveColumnHandle -> - getFieldFromSchema(hiveColumnHandle.getName(), hudiTableHandle.getTableSchema()).name()); + getFieldFromSchema(hiveColumnHandle.getName(), hudiTableHandle.getTableSchema().toAvroSchema()).name()); } return tupleDomain.transformKeys(HiveColumnHandle::getName); } diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java new file mode 100644 index 0000000000000..60bb33eb5405a --- /dev/null +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java @@ -0,0 +1,288 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.reader; + +import io.trino.metastore.HiveType; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hudi.util.HudiAvroSerializer; +import io.trino.plugin.hudi.util.SynthesizedColumnHandler; +import io.trino.spi.Page; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.SourcePage; +import io.trino.spi.type.VarcharType; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.avro.AvroRecordContext; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.engine.HoodieReaderContext; +import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.HoodieAvroRecordMerger; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.model.OverwriteWithLatestMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.HoodieRecordUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; +import org.apache.hudi.storage.inline.InLineFSUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static java.lang.String.format; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; + +public class HudiTrinoReaderContext + extends HoodieReaderContext +{ + ConnectorPageSource pageSource; + private final HudiAvroSerializer avroSerializer; + private final SynthesizedColumnHandler synthesizedColumnHandler; + private final LogFileParquetPageSourceFactory logPageSourceFactory; + Map colToPosMap; + Map colNameToHandle; + List dataHandles; + List columnHandles; + + /** + * Factory for building a Trino parquet page source over a native (RFC-103) delta-log file on demand. + * Kept as an injected interface so this reader context stays free of Trino-parquet/session types. + */ + @FunctionalInterface + public interface LogFileParquetPageSourceFactory + { + ConnectorPageSource create(String path, long start, long length, List projection); + } + + public HudiTrinoReaderContext( + StorageConfiguration storageConfiguration, + HoodieTableConfig tableConfig, + ConnectorPageSource pageSource, + List dataHandles, + List columnHandles, + SynthesizedColumnHandler synthesizedColumnHandler, + LogFileParquetPageSourceFactory logPageSourceFactory) + { + super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), new AvroRecordContext(tableConfig, tableConfig.getPayloadClass())); + this.pageSource = pageSource; + this.synthesizedColumnHandler = synthesizedColumnHandler; + this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler); + this.dataHandles = dataHandles; + this.columnHandles = columnHandles; + this.logPageSourceFactory = logPageSourceFactory; + this.colToPosMap = new HashMap<>(); + this.colNameToHandle = new HashMap<>(); + for (int i = 0; i < columnHandles.size(); i++) { + HiveColumnHandle handle = columnHandles.get(i); + colToPosMap.put(handle.getBaseColumnName(), i); + colNameToHandle.put(handle.getBaseColumnName().toLowerCase(Locale.ROOT), handle); + } + } + + @Override + public ClosableIterator getFileRecordIterator( + StoragePath storagePath, + long start, + long length, + HoodieSchema dataSchema, + HoodieSchema requiredSchema, + HoodieStorage storage) + { + return getFileRecordIterator(storagePath, start, length, requiredSchema); + } + + @Override + public ClosableIterator getFileRecordIterator( + StoragePathInfo storagePathInfo, + long start, + long length, + HoodieSchema dataSchema, + HoodieSchema requiredSchema, + HoodieStorage storage) + { + return getFileRecordIterator(storagePathInfo.getPath(), start, length, requiredSchema); + } + + /** + * Reads the given file and projects {@code requiredSchema}. For a native (RFC-103) delta-log parquet file a + * fresh page source is built on demand with predicate pushdown disabled so every log record is read and + * merged; for the base file the pre-built base page source is reused. Classic Avro log blocks never reach + * here (they deserialize inline). + */ + private ClosableIterator getFileRecordIterator( + StoragePath path, + long start, + long length, + HoodieSchema requiredSchema) + { + if (FSUtils.isLogFile(path)) { + // Inline parquet log blocks (inlinefs:// scheme) need a separate inline-aware reader; out of scope here. + if (InLineFSUtils.SCHEME.equals(path.toUri().getScheme())) { + throw new UnsupportedOperationException("Inline log blocks are not supported by the Hudi Trino connector: " + path); + } + List logProjection = buildRequiredColumnHandles(requiredSchema); + ConnectorPageSource logSource = logPageSourceFactory.create(path.toString(), start, length, logProjection); + HudiAvroSerializer logSerializer = new HudiAvroSerializer(logProjection, synthesizedColumnHandler); + return createRecordIterator(logSource, logSerializer); + } + return createRecordIterator(pageSource, avroSerializer); + } + + /** + * Resolves the {@link HiveColumnHandle} for each field of {@code requiredSchema} against the reader's column + * handles (keyed by lowercased base column name) so the on-demand log page source reads exactly the columns + * the file-group reader needs to merge. The file-group reader can add meta fields to {@code requiredSchema} + * that the connector projection does not carry -- the projection holds only the query columns plus + * {@code HUDI_REQUIRED_META_COLUMNS} (record key + partition path), whereas the merge may also need e.g. + * {@code _hoodie_commit_time}; for such a Hudi meta column a handle is synthesized (see the inline note) + * rather than failing the read. + */ + private List buildRequiredColumnHandles(HoodieSchema requiredSchema) + { + List handles = new ArrayList<>(); + for (HoodieSchemaField field : requiredSchema.getFields()) { + String name = field.name(); + HiveColumnHandle handle = colNameToHandle.get(name.toLowerCase(Locale.ROOT)); + if (handle == null) { + if (!HoodieRecord.HOODIE_META_COLUMNS.contains(name)) { + // A data column outside the projection cannot be typed at this layer. The file-group reader + // only asks for one when the table's custom merger is not projection compatible (it then + // reads the FULL table schema), which this connector does not support. + throw new TrinoException(NOT_SUPPORTED, format( + "Column '%s' is required for merging but is not in the connector's read projection. " + + "This usually means the table's custom record merger is not projection compatible; " + + "the Hudi Trino connector requires custom mergers to override isProjectionCompatible() " + + "to true and declare getMandatoryFieldsForMerging().", name)); + } + // Synthesize a handle for a Hudi meta column absent from the connector projection. This is safe: + // 1. Every Hudi meta column is a UTF8 string on disk, so HIVE_STRING/VARCHAR is the correct + // physical type -- the same handle HudiUtil.prependHudiMetaAndMergeRequiredColumns builds for the + // HUDI_REQUIRED_META_COLUMNS. + // 2. hiveColumnIndex (0) is a throwaway placeholder: HudiPageSourceProvider.createPageSource + // resolves parquet columns by NAME (directly when useColumnNames=true, otherwise + // remapColumnIndicesToPhysical rebuilds every index from the file schema by name), so this + // ordinal is never read. + // TODO(apache/hudi#19249): remove this synthesis. Build colNameToHandle from the full data schema + // (all meta + data columns, typed once from the table schema) so every requiredSchema field + // resolves by lookup, dropping both the hand-built handle and the HOODIE_META_COLUMNS guard + // above -- which would also lift the projection-compatible-merger restriction. + handle = new HiveColumnHandle(name, 0, HiveType.HIVE_STRING, VarcharType.VARCHAR, + Optional.empty(), HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); + } + handles.add(handle); + } + return handles; + } + + private ClosableIterator createRecordIterator(ConnectorPageSource source, HudiAvroSerializer serializer) + { + return new ClosableIterator<>() + { + private Page currentPage; + private int currentPosition; + + @Override + public void close() + { + try { + source.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean hasNext() + { + // If all records in the current page are consume, try to get next page + if (currentPage == null || currentPosition >= currentPage.getPositionCount()) { + if (source.isFinished()) { + return false; + } + + // Get next page and reset currentPosition. Unwrap the SourcePage to the + // underlying Page so the serializer's Block accessors keep working. + SourcePage nextSourcePage = source.getNextSourcePage(); + currentPage = nextSourcePage == null ? null : nextSourcePage.getPage(); + currentPosition = 0; + + // If no more pages are available + return currentPage != null; + } + + return true; + } + + @Override + public IndexedRecord next() + { + if (!hasNext()) { + // TODO: This can probably be removed or ignored, added this as a sanity check + throw new RuntimeException("No more records in the iterator"); + } + + IndexedRecord record = serializer.serialize(currentPage, currentPosition); + currentPosition++; + return record; + } + }; + } + + @Override + protected Option getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) + { + // Dispatch on the table's merge mode, mirroring HoodieAvroReaderContext. The Trino reader + // operates on IndexedRecord, so the Avro mergers apply directly. Using the read-time merger + // (combineAndGetUpdateValue) rather than a fixed preCombine merger keeps COMMIT_TIME_ORDERING + // and custom-payload tables correct on MoR reads. + // TODO(apache/hudi#18898): add MoR read tests for delete markers and custom payloads to + // exercise the EVENT_TIME_ORDERING (combineAndGetUpdateValue) and CUSTOM branches below. + switch (mergeMode) { + case EVENT_TIME_ORDERING: + return Option.of(new HoodieAvroRecordMerger()); + case COMMIT_TIME_ORDERING: + return Option.of(new OverwriteWithLatestMerger()); + case CUSTOM: + default: + Option recordMerger = HoodieRecordUtils.createValidRecordMerger(EngineType.JAVA, mergeImplClasses, mergeStrategyId); + if (recordMerger.isEmpty()) { + throw new IllegalArgumentException("No valid merger implementation set for `" + RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY + "`"); + } + return recordMerger; + } + } + + @Override + public ClosableIterator mergeBootstrapReaders(ClosableIterator skeletonFileIterator, HoodieSchema skeletonRequiredSchema, ClosableIterator dataFileIterator, HoodieSchema dataRequiredSchema, List> requiredPartitionFieldAndValues) + { + // Bootstrap merge is not exercised by the Trino connector; reads of bootstrap tables go + // through the regular page-source path. Throwing surfaces accidental use loudly. + throw new UnsupportedOperationException("HudiTrinoReaderContext does not support bootstrap merge"); + } +} diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java similarity index 98% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java index 999e36f1ea0d8..b48e790f5d012 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java @@ -19,7 +19,6 @@ import io.airlift.concurrent.BoundedExecutor; import io.airlift.log.Logger; import io.trino.filesystem.Location; -import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.metastore.Column; import io.trino.metastore.Partition; import io.trino.metastore.StorageFormat; @@ -44,7 +43,7 @@ import org.apache.hudi.hive.SinglePartPartitionValueExtractor; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.sync.common.model.PartitionValueExtractor; -import org.apache.hudi.util.Lazy; +import org.apache.hudi.common.util.Lazy; import java.util.ArrayList; import java.util.Collections; @@ -101,14 +100,13 @@ public HudiBackgroundSplitLoader( Lazy> lazyPartitionMap, boolean enableMetadataTable, Lazy lazyTableMetadata, - CachingHostAddressProvider cachingHostAddressProvider, Consumer errorListener) { this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); this.hudiDirectoryLister = requireNonNull(hudiDirectoryLister, "hudiDirectoryLister is null"); this.asyncQueue = requireNonNull(asyncQueue, "asyncQueue is null"); this.splitGeneratorNumThreads = getSplitGeneratorParallelism(session); - this.hudiSplitFactory = new HudiSplitFactory(tableHandle, hudiSplitWeightProvider, getTargetSplitSize(session), cachingHostAddressProvider); + this.hudiSplitFactory = new HudiSplitFactory(tableHandle, hudiSplitWeightProvider, getTargetSplitSize(session)); this.lazyPartitionMap = requireNonNull(lazyPartitionMap, "partitions is null"); this.enableMetadataTable = enableMetadataTable; this.executor = requireNonNull(executor, "executor is null"); diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java similarity index 85% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java index 09dbf5bd5020c..6bab996e199bb 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; -import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.plugin.hive.HivePartitionKey; import io.trino.plugin.hudi.HudiSplit; import io.trino.plugin.hudi.HudiTableHandle; @@ -43,23 +42,20 @@ public class HudiSplitFactory private final HudiTableHandle hudiTableHandle; private final HudiSplitWeightProvider hudiSplitWeightProvider; private final DataSize targetSplitSize; - private final CachingHostAddressProvider cachingHostAddressProvider; public HudiSplitFactory( HudiTableHandle hudiTableHandle, HudiSplitWeightProvider hudiSplitWeightProvider, - DataSize targetSplitSize, - CachingHostAddressProvider cachingHostAddressProvider) + DataSize targetSplitSize) { this.hudiTableHandle = requireNonNull(hudiTableHandle, "hudiTableHandle is null"); this.hudiSplitWeightProvider = requireNonNull(hudiSplitWeightProvider, "hudiSplitWeightProvider is null"); this.targetSplitSize = requireNonNull(targetSplitSize, "targetSplitSize is null"); - this.cachingHostAddressProvider = requireNonNull(cachingHostAddressProvider, "cachingHostAddressProvider is null"); } public List createSplits(List partitionKeys, FileSlice fileSlice, String commitTime) { - return createHudiSplits(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, targetSplitSize, cachingHostAddressProvider); + return createHudiSplits(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, targetSplitSize); } /** @@ -75,8 +71,7 @@ public static List createHudiSplits( FileSlice fileSlice, String commitTime, HudiSplitWeightProvider hudiSplitWeightProvider, - DataSize targetSplitSize, - CachingHostAddressProvider cachingHostAddressProvider) + DataSize targetSplitSize) { if (fileSlice.isEmpty()) { throw new TrinoException(HUDI_FILESYSTEM_ERROR, format("Not a valid file slice: %s", fileSlice)); @@ -85,9 +80,9 @@ public static List createHudiSplits( if (isCopyOnWriteOrReadOptimized(hudiTableHandle, fileSlice)) { // Handle MERGE_ON_READ tables to be read in read_optimized mode // IMPORTANT: These tables will have a COPY_ON_WRITE table type due to how `HudiTableTypeUtils#fromInputFormat` - return createSplitsForBaseFile(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, targetSplitSize, cachingHostAddressProvider); + return createSplitsForBaseFile(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, targetSplitSize); } - return createSplitForMergeOnRead(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, cachingHostAddressProvider); + return createSplitForMergeOnRead(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider); } /** @@ -108,15 +103,15 @@ private static List createSplitsForBaseFile( FileSlice fileSlice, String commitTime, HudiSplitWeightProvider hudiSplitWeightProvider, - DataSize targetSplitSize, - CachingHostAddressProvider cachingHostAddressProvider) + DataSize targetSplitSize) { checkArgument(fileSlice.getBaseFile().isPresent(), "Hudi base file must exist if there are no log files in the file slice"); HoodieBaseFile baseFile = fileSlice.getBaseFile().get(); long fileSize = baseFile.getFileSize(); - List addresses = cachingHostAddressProvider.getHosts(baseFile.getPath(), ImmutableList.of()); + // Object-storage reads have no host affinity. + List addresses = ImmutableList.of(); // If the file is empty, create a single split to represent it if (fileSize == 0) { @@ -168,14 +163,11 @@ private static List createSplitForMergeOnRead( List partitionKeys, FileSlice fileSlice, String commitTime, - HudiSplitWeightProvider hudiSplitWeightProvider, - CachingHostAddressProvider cachingHostAddressProvider) + HudiSplitWeightProvider hudiSplitWeightProvider) { // NOTE: Some file slices may not have base files Option baseFileOption = fileSlice.getBaseFile(); - List addresses = baseFileOption - .map(baseFile -> cachingHostAddressProvider.getHosts(baseFile.getPath(), ImmutableList.of())) - .orElse(ImmutableList.of()); + List addresses = ImmutableList.of(); HudiSplit split = new HudiSplit( baseFileOption.map(HudiBaseFile::of).orElse(null), diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java similarity index 50% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java index e87122de07953..1546473ada485 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java @@ -16,21 +16,14 @@ import org.apache.hudi.avro.model.HoodieMetadataColumnStats; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.HoodieEngineContext; -import org.apache.hudi.common.model.HoodieColumnRangeMetadata; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.common.util.hash.ColumnIndexID; -import org.apache.hudi.common.util.hash.FileIndexID; -import org.apache.hudi.common.util.hash.PartitionIndexID; import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.metadata.HoodieBackedTableMetadata; -import org.apache.hudi.metadata.HoodieMetadataMetrics; -import org.apache.hudi.metadata.HoodieMetadataPayload; -import org.apache.hudi.metadata.HoodieTableMetadataUtil; -import org.apache.hudi.metadata.MetadataPartitionType; +import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata; +import org.apache.hudi.metadata.stats.ValueMetadata; import org.apache.hudi.storage.HoodieStorage; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -55,44 +48,13 @@ public class TableMetadataReader * @return a map from column name to their corresponding {@link HoodieColumnRangeMetadata} * @throws HoodieMetadataException if an error occurs while fetching the column statistics */ - Map getColumnStats(List> partitionNameFileNameList, List columnNames) + Map getColumnRanges(List> partitionNameFileNameList, List columnNames) throws HoodieMetadataException { - return computeFileToColumnStatsMap(computeColumnStatsLookupKeys(partitionNameFileNameList, columnNames)); - } - - /** - * @param partitionNameFileNameList a list of partition and file name pairs for which column stats need to be retrieved - * @param columnNames list of column names for which stats are needed - * @return a list of column stats keys to look up in the metadata table col_stats partition. - */ - private List computeColumnStatsLookupKeys( - final List> partitionNameFileNameList, - final List columnNames) - { - return columnNames.stream() - .flatMap(columnName -> partitionNameFileNameList.stream() - .map(partitionNameFileNamePair -> HoodieMetadataPayload.getColumnStatsIndexKey( - new PartitionIndexID(HoodieTableMetadataUtil.getColumnStatsIndexPartitionIdentifier(partitionNameFileNamePair.getLeft())), - new FileIndexID(partitionNameFileNamePair.getRight()), - new ColumnIndexID(columnName)))) - .toList(); - } - - /** - * @param columnStatsLookupKeys a map from column stats key to partition and file name pair - * @return a map from column name to merged HoodieMetadataColumnStats - */ - private Map computeFileToColumnStatsMap(List columnStatsLookupKeys) - { - HoodieTimer timer = HoodieTimer.start(); - Map> hoodieRecords = - getRecordsByKeys(columnStatsLookupKeys, MetadataPartitionType.COLUMN_STATS.getPartitionPath()); - metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer())); - return hoodieRecords.values().stream() - .collect(Collectors.groupingBy( - r -> r.getData().getColumnStatMetadata().get().getColumnName(), - Collectors.mapping(r -> r.getData().getColumnStatMetadata().get(), Collectors.toList()))) + Map, List> columnStatsMap = getColumnStats(partitionNameFileNameList, columnNames); + return columnStatsMap.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.groupingBy(HoodieMetadataColumnStats::getColumnName, Collectors.toList())) .entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, @@ -108,7 +70,7 @@ private Map computeFileToColumnStatsMap(List< totalUncompressedSize += stats.getTotalUncompressedSize(); } return HoodieColumnRangeMetadata.create( - "", e.getKey(), null, null, nullCount, valueCount, totalSize, totalUncompressedSize); + "", e.getKey(), null, null, nullCount, valueCount, totalSize, totalUncompressedSize, ValueMetadata.NULL_METADATA); })); } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java similarity index 97% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java index 55d424db40caa..8499e647a02a9 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java @@ -24,11 +24,11 @@ import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.engine.HoodieLocalEngineContext; -import org.apache.hudi.common.model.HoodieColumnRangeMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata; import java.util.List; import java.util.Map; @@ -113,6 +113,6 @@ private static Map getColumnStats( .stream().flatMap(entry -> entry.getValue() .map(baseFile -> Pair.of(entry.getKey(), baseFile.getFileName()))) .toList(); - return tableMetadata.getColumnStats(filePaths, columnNames); + return tableMetadata.getColumnRanges(filePaths, columnNames); } } diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java similarity index 99% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index 2fc020020252e..24c653511fe0d 100644 --- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -135,7 +135,7 @@ public IndexedRecord serialize(Page sourcePage, int position) public Object getValue(Page sourcePage, int channel, int position) { - return columnTypes.get(channel).getObjectValue(null, sourcePage.getBlock(channel), position); + return columnTypes.get(channel).getObjectValue(sourcePage.getBlock(channel), position); } public void buildRecordInPage(PageBuilder pageBuilder, IndexedRecord record) diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/TupleDomainUtils.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/TupleDomainUtils.java similarity index 100% rename from hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/TupleDomainUtils.java rename to hudi-trino/src/main/java/io/trino/plugin/hudi/util/TupleDomainUtils.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorSmokeTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorSmokeTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorSmokeTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorSmokeTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java similarity index 97% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java index 0114eb8400a2a..e11efbf3a569a 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java @@ -35,9 +35,9 @@ import java.util.Optional; import static io.trino.testing.TestingSession.testSessionBuilder; -import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; +import static io.trino.testing.containers.Minio.MINIO_ROOT_USER; import static io.trino.testing.containers.Minio.MINIO_REGION; -import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static io.trino.testing.containers.Minio.MINIO_ROOT_PASSWORD; import static java.util.Objects.requireNonNull; public final class HudiQueryRunner @@ -60,8 +60,8 @@ public static Builder builder(Hive3MinioDataLake hiveMinioDataLake) { return new Builder("s3://" + hiveMinioDataLake.getBucketName() + "/") .addConnectorProperty("fs.native-s3.enabled", "true") - .addConnectorProperty("s3.aws-access-key", MINIO_ACCESS_KEY) - .addConnectorProperty("s3.aws-secret-key", MINIO_SECRET_KEY) + .addConnectorProperty("s3.aws-access-key", MINIO_ROOT_USER) + .addConnectorProperty("s3.aws-secret-key", MINIO_ROOT_PASSWORD) .addConnectorProperty("s3.region", MINIO_REGION) .addConnectorProperty("s3.endpoint", hiveMinioDataLake.getMinio().getMinioAddress()) .addConnectorProperty("s3.path-style-access", "true"); diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/HudiUtilTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/HudiUtilTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/HudiUtilTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/HudiUtilTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/SessionBuilder.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/SessionBuilder.java similarity index 86% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/SessionBuilder.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/SessionBuilder.java index aa26d2d35f8e9..8ddc1ab4d4691 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/SessionBuilder.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/SessionBuilder.java @@ -15,6 +15,9 @@ import io.trino.Session; +import java.util.Arrays; +import java.util.stream.Collectors; + import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; import static io.trino.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static io.trino.plugin.hudi.HudiSessionProperties.COLUMN_STATS_INDEX_ENABLED; @@ -24,7 +27,9 @@ import static io.trino.plugin.hudi.HudiSessionProperties.PARTITION_STATS_INDEX_ENABLED; import static io.trino.plugin.hudi.HudiSessionProperties.QUERY_PARTITION_FILTER_REQUIRED; import static io.trino.plugin.hudi.HudiSessionProperties.RECORD_INDEX_WAIT_TIMEOUT; +import static io.trino.plugin.hudi.HudiSessionProperties.RECORD_MERGER_IMPLS; import static io.trino.plugin.hudi.HudiSessionProperties.RECORD_LEVEL_INDEX_ENABLED; +import static io.trino.plugin.hudi.HudiSessionProperties.RESOLVE_COLUMN_NAME_CASING_ENABLED; import static io.trino.plugin.hudi.HudiSessionProperties.SECONDARY_INDEX_ENABLED; import static io.trino.plugin.hudi.HudiSessionProperties.SECONDARY_INDEX_WAIT_TIMEOUT; import static io.trino.plugin.hudi.HudiSessionProperties.TABLE_STATISTICS_ENABLED; @@ -140,4 +145,20 @@ public SessionBuilder withRecordIndexTimeout(String durationProp) { return setCatalogProperty(RECORD_INDEX_WAIT_TIMEOUT, durationProp); } + + /** + * Sets {@code record_merger_impls}. The property is an array type, so the value is rendered as a JSON list. + */ + public SessionBuilder withRecordMergerImpls(String... mergerClassNames) + { + String jsonList = Arrays.stream(mergerClassNames) + .map(name -> '"' + name + '"') + .collect(Collectors.joining(",", "[", "]")); + return setCatalogProperty(RECORD_MERGER_IMPLS, jsonList); + } + + public SessionBuilder withResolveColumnNameCasingEnabled(boolean enabled) + { + return setCatalogProperty(RESOLVE_COLUMN_NAME_CASING_ENABLED, String.valueOf(enabled)); + } } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java similarity index 91% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java index 911c5d4c3e7d4..22f569dcf50de 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java @@ -38,11 +38,9 @@ import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.getCacheOperationSpans; import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_FG_PT_V8_MOR; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.INDEX_DEFINITION; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.LOG; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE_PROPERTIES; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TABLE_PROPERTIES; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TIMELINE; import static io.trino.testing.MultisetAssertions.assertMultisetsEqual; import static io.trino.testing.assertions.Assert.assertEventually; import static java.util.stream.Collectors.toCollection; @@ -90,8 +88,6 @@ public void testSelectWithFilter() assertFileSystemAccesses( query, ImmutableMultiset.builder() - .addCopies(new FileOperation("InputFile.length", TIMELINE), 2) - .addCopies(new FileOperation("InputFile.length", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -100,8 +96,6 @@ public void testSelectWithFilter() assertFileSystemAccesses( query, ImmutableMultiset.builder() - .addCopies(new FileOperation("InputFile.length", TIMELINE), 2) - .addCopies(new FileOperation("InputFile.length", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -118,8 +112,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() - .addCopies(new FileOperation("InputFile.length", TIMELINE), 4) - .addCopies(new FileOperation("InputFile.length", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) @@ -127,8 +119,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() - .addCopies(new FileOperation("InputFile.length", TIMELINE), 4) - .addCopies(new FileOperation("InputFile.length", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCachingSmokeTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCachingSmokeTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCachingSmokeTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCachingSmokeTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java similarity index 94% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java index 545693f1ea076..d5353b896a1a5 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java @@ -33,6 +33,7 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(HudiConfig.class) .setColumnsToHide(ImmutableList.of()) + .setRecordMergerImpls(ImmutableList.of()) .setTableStatisticsEnabled(true) .setMetadataEnabled(true) .setUseParquetColumnNames(true) @@ -59,7 +60,7 @@ public void testDefaults() .setSecondaryIndexWaitTimeout(Duration.valueOf("2s")) .setMetadataPartitionListingEnabled(true) .setMetadataCacheEnabled(true) - .setResolveColumnNameCasingEnabled(true)); + .setResolveColumnNameCasingEnabled(false)); } @Test @@ -67,6 +68,7 @@ public void testExplicitPropertyMappings() { Map properties = ImmutableMap.builder() .put("hudi.columns-to-hide", "_hoodie_record_key") + .put("hudi.record-merger-impls", "com.example.MergerOne,com.example.MergerTwo") .put("hudi.table-statistics-enabled", "false") .put("hudi.metadata-enabled", "false") .put("hudi.parquet.use-column-names", "false") @@ -93,11 +95,12 @@ public void testExplicitPropertyMappings() .put("hudi.index.secondary-index.wait-timeout", "1s") .put("hudi.metadata.cache.enabled", "false") .put("hudi.metadata.partition-listing.enabled", "false") - .put("hudi.table.resolve-column-name-casing.enabled", "false") + .put("hudi.table.resolve-column-name-casing.enabled", "true") .buildOrThrow(); HudiConfig expected = new HudiConfig() .setColumnsToHide(ImmutableList.of("_hoodie_record_key")) + .setRecordMergerImpls(ImmutableList.of("com.example.MergerOne", "com.example.MergerTwo")) .setTableStatisticsEnabled(false) .setMetadataEnabled(false) .setUseParquetColumnNames(false) @@ -124,7 +127,7 @@ public void testExplicitPropertyMappings() .setSecondaryIndexWaitTimeout(Duration.valueOf("1s")) .setMetadataPartitionListingEnabled(false) .setMetadataCacheEnabled(false) - .setResolveColumnNameCasingEnabled(false); + .setResolveColumnNameCasingEnabled(true); assertFullMapping(properties, expected); } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorFactory.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorFactory.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorFactory.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorFactory.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java similarity index 89% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java index 84a4cbd604402..acb85a2fe25fb 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java @@ -49,6 +49,11 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_DELETE, SUPPORTS_DEREFERENCE_PUSHDOWN, SUPPORTS_INSERT, + // HudiMetadata.applyLimit returns limitGuaranteed=false (multi-split connector + // cannot bound total rows across workers). BaseConnectorTest.testLimitPushdown + // requires Output->TableScan with no Limit node, which needs guaranteed=true. + // Stays off, matching Iceberg / Delta Lake / Hive. + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN, SUPPORTS_RENAME_TABLE, diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java new file mode 100644 index 0000000000000..285d61eec70a7 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.plugin.hudi.testing.CustomMergerHudiTablesInitializer; +import io.trino.plugin.hudi.testing.KeyBasedTestRecordMerger; +import io.trino.plugin.hudi.testing.NonProjectionCompatibleTestRecordMerger; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Verifies that the Hudi Trino connector resolves and applies a user-supplied custom record merger + * (configured via {@code hudi.record-merger-impls}) for Merge-On-Read tables whose record merge mode is + * {@code CUSTOM}. + *

    + * The test table is generated by {@link CustomMergerHudiTablesInitializer}: two record keys are inserted and + * then upserted so each file group has a base file plus a log file. {@link KeyBasedTestRecordMerger} keeps the + * newer record for keys ending in an odd digit and the older record otherwise, which makes the merged result + * distinguishable from both the read-optimized (base-only) view and the built-in newest-wins behavior: + *

      + *
    • {@code k1} (ends in odd digit): keeps the update -> value 99, name k1_updated
      • + *
    • {@code k2} (ends in even digit): keeps the base -> value 100, name k2_base
      • + *
    + * So on the real-time table {@code sum(value)} is 199 (99 + 100), whereas the base-only view yields 110 + * (10 + 100) and the built-in newest-wins would yield 104 (99 + 5). + */ +public class TestHudiCustomMerger + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return HudiQueryRunner.builder() + .setDataLoader(new CustomMergerHudiTablesInitializer()) + .addConnectorProperties(ImmutableMap.of( + "hudi.record-merger-impls", KeyBasedTestRecordMerger.class.getName())) + .build(); + } + + @Test + public void testReadOptimizedTableReturnsBaseFileValues() + { + // The read-optimized table reads base files only, so it reflects the initial insert. + assertQuery( + "SELECT key, name, value FROM " + CustomMergerHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + assertThat(computeScalar("SELECT sum(value) FROM " + CustomMergerHudiTablesInitializer.TABLE_NAME)) + .isEqualTo(110L); + } + + @Test + public void testRealtimeTableAppliesCustomMerger() + { + // The real-time table merges base + log files. With the key-based custom merger: + // - k1 (odd) keeps the update (value 99, name k1_updated) -> proves merging happened (base was 10) + // - k2 (even) keeps the base (value 100, name k2_base) -> proves it is the custom rule, not newest-wins + assertQuery( + "SELECT key, name, value FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(99 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + } + + @Test + public void testRealtimeTableSumIsDistinctFromBaseAndNewestWins() + { + // 199 uniquely identifies the custom merger: base-only would be 110, built-in newest-wins would be 104. + assertThat(computeScalar("SELECT sum(value) FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME)) + .isEqualTo(199L); + } + + @Test + public void testNonProjectionCompatibleMergerIsRejected() + { + // A merger that is not projection compatible makes the file-group reader ask for the FULL table schema. + // The connector can only resolve columns in the read projection (plus Hudi meta columns), so a data + // column outside the projection must fail loudly rather than silently merge against a null value. + Session session = SessionBuilder.from(getSession()) + .withRecordMergerImpls(NonProjectionCompatibleTestRecordMerger.class.getName()) + .build(); + // The guard throws inside the file-group reader and Hudi wraps it in a generic HoodieException + // ("Exception when reading log file"), but HudiPageSource rethrows the TrinoException from the + // cause chain, so the actionable text must be the top-level query failure message. + assertThatThrownBy(() -> computeScalar(session, "SELECT sum(value) FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME)) + .hasMessageContaining("is required for merging but is not in the connector's read projection") + .hasMessageContaining("requires custom mergers to override isProjectionCompatible()"); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMergerEndToEnd.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMergerEndToEnd.java new file mode 100644 index 0000000000000..fbd4b186556ec --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMergerEndToEnd.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.hudi.testing.IncrementalCustomMergerHudiTablesInitializer; +import io.trino.plugin.hudi.testing.MaxRankRecordMerger; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.MaterializedRow; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static io.trino.plugin.hudi.testing.IncrementalCustomMergerHudiTablesInitializer.RT_TABLE_NAME; +import static io.trino.plugin.hudi.testing.IncrementalCustomMergerHudiTablesInitializer.TABLE_NAME; +import static io.trino.plugin.hudi.testing.IncrementalCustomMergerHudiTablesInitializer.TOTAL_COMMITS; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * End-to-end test of the custom record merger ({@link MaxRankRecordMerger}, configured via + * {@code hudi.record-merger-impls}) on a 30-column Merge-On-Read table with {@link + * IncrementalCustomMergerHudiTablesInitializer#NUM_RECORDS} records. + *

    + * The table is written one commit at a time (a bulk insert followed by {@link + * IncrementalCustomMergerHudiTablesInitializer#TOTAL_COMMITS} - 1 upserts). After every commit the real-time + * table is read back through Trino and every column of every record is compared against the closed-form expected + * merge result. The expectation is computed independently of the connector by folding the same max-rank policy + * over the committed values, so a regression in merger resolution, merge ordering, or column handling fails the + * assertion. + */ +public class TestHudiCustomMergerEndToEnd + extends AbstractTestQueryFramework +{ + private IncrementalCustomMergerHudiTablesInitializer writer; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + writer = new IncrementalCustomMergerHudiTablesInitializer(); + return HudiQueryRunner.builder() + .setDataLoader(writer) + .addConnectorProperties(ImmutableMap.of( + "hudi.record-merger-impls", MaxRankRecordMerger.class.getName())) + .build(); + } + + @AfterAll + public void tearDown() + throws Exception + { + if (writer != null) { + writer.close(); + writer = null; + } + } + + @Test + public void testCustomMergerValidatedAfterEveryCommit() + { + // The first commit was written and synced by initializeTables; validate it, then drive the remaining commits. + validateRows(RT_TABLE_NAME, writer.expectedRows()); + for (int commit = 2; commit <= TOTAL_COMMITS; commit++) { + writer.writeAndSyncNextCommit(); + validateRows(RT_TABLE_NAME, writer.expectedRows()); + } + // Sanity check that the data actually distinguishes the custom merge from built-in newest-wins: + // for many keys the winning record is not the most recently committed one. + assertThat(writer.divergentKeyCount()).isGreaterThan(0); + + // Regression check for projection pushdown: a query that does NOT project the merge column + // (merge_rank) must still merge correctly, because MaxRankRecordMerger declares merge_rank as a + // mandatory merge field, so the reader includes it in the read schema even though it is not selected. + // Without that, the merger would read a pruned (null) merge_rank and fail / mis-merge. + int s0Index = writer.dataColumnNames().indexOf("s0"); + List projectedRows = computeActual( + "SELECT key, s0 FROM " + RT_TABLE_NAME + " ORDER BY key").getMaterializedRows(); + Map expected = writer.expectedRows(); + assertThat(projectedRows).hasSize(expected.size()); + for (MaterializedRow row : projectedRows) { + Object[] expectedRow = expected.get((String) row.getField(0)); + assertThat(expectedRow).as("unexpected key %s", row.getField(0)).isNotNull(); + assertThat(row.getField(1)).as("key %s, column s0", row.getField(0)).isEqualTo(expectedRow[s0Index]); + } + } + + @Test + public void testReadOptimizedReflectsBaseFilesOnly() + { + // The read-optimized table reads base files only, so it always reflects the first commit regardless of + // how many delta commits have since been written. + validateRows(TABLE_NAME, writer.baseRows()); + } + + private void validateRows(String table, Map expected) + { + List columns = writer.dataColumnNames(); + List rows = computeActual( + "SELECT " + String.join(", ", columns) + " FROM " + table + " ORDER BY key") + .getMaterializedRows(); + + assertThat(rows).hasSize(expected.size()); + for (MaterializedRow row : rows) { + String key = (String) row.getField(0); + Object[] expectedRow = expected.get(key); + assertThat(expectedRow).as("unexpected key %s in table %s", key, table).isNotNull(); + for (int i = 0; i < expectedRow.length; i++) { + assertThat(row.getField(i)) + .as("table %s, key %s, column %s", table, key, columns.get(i)) + .isEqualTo(expectedRow[i]); + } + } + } +} diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java similarity index 89% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java index 8e1b2fe819119..b5ada57414115 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java @@ -35,11 +35,9 @@ import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_FG_PT_V8_MOR; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.DATA; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.INDEX_DEFINITION; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.LOG; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE_PROPERTIES; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TABLE_PROPERTIES; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TIMELINE; import static io.trino.testing.MultisetAssertions.assertMultisetsEqual; import static java.util.stream.Collectors.toCollection; @@ -79,8 +77,6 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2) - .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2) - .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -90,8 +86,6 @@ public void testSelectWithFilter() query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2) - .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2) - .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) .add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) @@ -109,8 +103,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6) - .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4) - .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) @@ -119,8 +111,6 @@ public void testJoin() assertFileSystemAccesses(query, ImmutableMultiset.builder() .addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6) - .addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4) - .addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2) .addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java similarity index 88% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java index 55071241fff7a..e835e9249b0a5 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java @@ -35,11 +35,9 @@ import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_FG_PT_V8_MOR; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.DATA; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.INDEX_DEFINITION; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.LOG; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE_PROPERTIES; import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TABLE_PROPERTIES; -import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TIMELINE; import static io.trino.testing.MultisetAssertions.assertMultisetsEqual; import static java.util.stream.Collectors.toCollection; @@ -80,8 +78,6 @@ public void testSelectWithFilter() ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1) .add(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) .build()); @@ -91,8 +87,6 @@ public void testSelectWithFilter() ImmutableMultiset.builder() .addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1) .add(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES)) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2) .build()); @@ -112,8 +106,6 @@ public void testJoin() .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 2) .build()); assertFileSystemAccesses(query, @@ -122,8 +114,6 @@ public void testJoin() .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2) .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4) - .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 2) .build()); } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java similarity index 91% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java index 8bb46f8fb3f5a..9a36e7d385240 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSource.java @@ -16,7 +16,7 @@ import io.trino.spi.connector.ConnectorPageSource; import org.junit.jupiter.api.Test; -import static io.trino.spi.testing.InterfaceTestUtils.assertAllMethodsOverridden; +import static io.trino.testing.InterfaceTestUtils.assertAllMethodsOverridden; public class TestHudiPageSource { diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java similarity index 68% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java index c711a324b06d9..ed17a203089ed 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import static io.trino.plugin.hudi.HudiSessionProperties.getColumnsToHide; +import static io.trino.plugin.hudi.HudiSessionProperties.getRecordMergerImpls; import static org.assertj.core.api.Assertions.assertThat; public class TestHudiSessionProperties @@ -36,4 +37,17 @@ public void testSessionPropertyColumnsToHide() assertThat(getColumnsToHide(session)) .containsExactlyInAnyOrderElementsOf(ImmutableList.of("col1", "col2")); } + + @Test + public void testSessionPropertyRecordMergerImpls() + { + HudiConfig config = new HudiConfig() + .setRecordMergerImpls(ImmutableList.of("com.example.MergerOne", "com.example.MergerTwo")); + HudiSessionProperties sessionProperties = new HudiSessionProperties(config, new ParquetReaderConfig()); + ConnectorSession session = TestingConnectorSession.builder() + .setPropertyMetadata(sessionProperties.getSessionProperties()) + .build(); + assertThat(getRecordMergerImpls(session)) + .containsExactly("com.example.MergerOne", "com.example.MergerTwo"); + } } diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java similarity index 94% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java index 32994032265fe..c97f836087e2e 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSharedMetastore.java @@ -17,7 +17,7 @@ import com.google.common.collect.ImmutableMap; import io.trino.Session; import io.trino.filesystem.Location; -import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.plugin.hive.HivePlugin; import io.trino.plugin.hudi.testing.TpchHudiTablesInitializer; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; @@ -65,8 +65,15 @@ protected QueryRunner createQueryRunner() "hive.metastore.catalog.dir", dataDirectory.toString(), "fs.hadoop.enabled", "true")); - queryRunner.installPlugin(new TestingHivePlugin(dataDirectory)); - queryRunner.createCatalog("hive", "hive"); + queryRunner.installPlugin(new HivePlugin()); + queryRunner.createCatalog( + "hive", + "hive", + ImmutableMap.of( + // Intentionally sharing the file metastore directory with Hudi + "hive.metastore", "file", + "hive.metastore.catalog.dir", dataDirectory.toString(), + "fs.hadoop.enabled", "true")); queryRunner.execute("CREATE SCHEMA hive.default"); diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java similarity index 99% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java index 0e19f2a051771..6f8971f3bbe6c 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -727,6 +728,7 @@ public void testFileSkippingWithColumnNameUsingUppercaseLetters(ResourceHudiTabl .withRecordLevelIndexEnabled(false) .withSecondaryIndexEnabled(false) .withPartitionStatsIndexEnabled(false) + .withResolveColumnNameCasingEnabled(true) .build(); MaterializedResult prunedRes = getQueryRunner().execute(session, "SELECT * FROM " + table + " WHERE name='Alice'"); MaterializedResult totalRes = getQueryRunner().execute(session, "SELECT * FROM " + table); @@ -751,9 +753,10 @@ public void testRLIWithColumnNameUsingUppercaseLetters() .withRecordIndexTimeout("10s") .withSecondaryIndexEnabled(false) .withPartitionStatsIndexEnabled(false) + .withResolveColumnNameCasingEnabled(true) .build(); - MaterializedResult totalRes = getQueryRunner().execute(session, "SELECT * FROM " + HUDI_COW_TABLE_WITH_FIELD_NAMES_IN_CAPS); MaterializedResult prunedRes = getQueryRunner().execute(session, "SELECT * FROM " + HUDI_COW_TABLE_WITH_FIELD_NAMES_IN_CAPS + " WHERE id='1'"); + MaterializedResult totalRes = getQueryRunner().execute(session, "SELECT * FROM " + HUDI_COW_TABLE_WITH_FIELD_NAMES_IN_CAPS); int totalSplits = totalRes.getStatementStats().get().getTotalSplits(); int totalRows = totalRes.getRowCount(); int prunedSplits = prunedRes.getStatementStats().get().getTotalSplits(); @@ -775,6 +778,7 @@ public void testMultiKeyRLIWithColumnNameUsingUppercaseLetters() .withRecordIndexTimeout("10s") .withSecondaryIndexEnabled(false) .withPartitionStatsIndexEnabled(false) + .withResolveColumnNameCasingEnabled(true) .build(); MaterializedResult totalRes = getQueryRunner().execute(session, "SELECT * FROM " + HUDI_COW_TABLE_WITH_MULTI_KEYS_AND_FIELD_NAMES_IN_CAPS); MaterializedResult prunedRes = getQueryRunner().execute(session, "SELECT * FROM " + HUDI_COW_TABLE_WITH_MULTI_KEYS_AND_FIELD_NAMES_IN_CAPS + " WHERE id='1' and age=30"); @@ -1301,8 +1305,12 @@ private void testTimestampMicros(HiveTimestampPrecision timestampPrecision, Loca List.of(createBaseColumn("created", 0, HIVE_TIMESTAMP, columnType, REGULAR, Optional.empty())), hudiSplit, new LocalInputFile(parquetFile), + parquetFile.getPath(), + 0L, + parquetFile.length(), + OptionalLong.of(parquetFile.length()), new FileFormatDataSourceStats(), - new ParquetReaderOptions(), + ParquetReaderOptions.builder().build(), DateTimeZone.UTC, DynamicFilter.EMPTY, true)) { MaterializedResult result = materializeSourceDataStream(session, pageSource, List.of(columnType)).toTestTypes(); assertThat(result.getMaterializedRows()) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSystemTables.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSystemTables.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSystemTables.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSystemTables.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestingHudiConnectorFactory.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestingHudiConnectorFactory.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestingHudiConnectorFactory.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestingHudiConnectorFactory.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestingHudiPlugin.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestingHudiPlugin.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestingHudiPlugin.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/TestingHudiPlugin.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/io/TestInlineSeekableDataInputStream.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/io/TestInlineSeekableDataInputStream.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/io/TestInlineSeekableDataInputStream.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/io/TestInlineSeekableDataInputStream.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java similarity index 98% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java index 24c1b16dff5f7..6e9db6e1785b2 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/partition/TestHudiPartitionInfoLoader.java @@ -16,7 +16,6 @@ import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; import io.trino.filesystem.Location; -import io.trino.filesystem.cache.DefaultCachingHostAddressProvider; import io.trino.metastore.Partition; import io.trino.metastore.StorageFormat; import io.trino.plugin.hive.HiveColumnHandle; @@ -43,6 +42,7 @@ import java.util.Deque; import java.util.Iterator; import java.util.List; +import java.util.OptionalLong; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executors; @@ -161,12 +161,14 @@ private static HudiSplitFactory createSplitFactory() TABLE_PATH, HoodieTableType.COPY_ON_WRITE, ImmutableList.of(), + ImmutableList.of(), TupleDomain.all(), TupleDomain.all(), + OptionalLong.empty(), "", "101"); HudiSplitWeightProvider weightProvider = new SizeBasedSplitWeightProvider(0.05, DataSize.of(128, MEGABYTE)); - return new HudiSplitFactory(tableHandle, weightProvider, DataSize.of(128, MEGABYTE), new DefaultCachingHostAddressProvider()); + return new HudiSplitFactory(tableHandle, weightProvider, DataSize.of(128, MEGABYTE)); } private static HiveHudiPartitionInfo createTestPartition(String partitionPath) diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupportTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupportTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupportTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupportTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/query/index/TestingColumnHandle.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/query/index/TestingColumnHandle.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/query/index/TestingColumnHandle.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/query/index/TestingColumnHandle.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java similarity index 98% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java index 9c4f4a6176fcf..d659044d24569 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; -import io.trino.filesystem.cache.DefaultCachingHostAddressProvider; import io.trino.plugin.hive.HivePartitionKey; import io.trino.plugin.hudi.HudiSplit; import io.trino.plugin.hudi.HudiTableHandle; @@ -32,6 +31,7 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.OptionalLong; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static org.assertj.core.api.Assertions.assertThat; @@ -152,8 +152,7 @@ private static void testSplitCreation( FileSlice fileSlice = createFileSlice(baseFileSize, logFileSize); List splits = HudiSplitFactory.createHudiSplits( - tableHandle, PARTITION_KEYS, fileSlice, COMMIT_TIME, weightProvider, targetSplitSize, - new DefaultCachingHostAddressProvider()); + tableHandle, PARTITION_KEYS, fileSlice, COMMIT_TIME, weightProvider, targetSplitSize); assertThat(splits).hasSize(expectedSplitInfo.size()); @@ -181,8 +180,10 @@ private static HudiTableHandle createTableHandle() "/test/path", HoodieTableType.MERGE_ON_READ, ImmutableList.of(), + ImmutableList.of(), TupleDomain.all(), TupleDomain.all(), + OptionalLong.empty(), "", "101"); } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java new file mode 100644 index 0000000000000..1c5387419c5b3 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java @@ -0,0 +1,239 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Column; +import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.StorageFormat; +import io.trino.metastore.Table; +import io.trino.plugin.hudi.HudiConnector; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.QueryRunner; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.common.HoodieJavaEngineContext; +import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; +import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static java.nio.file.Files.createTempDirectory; + +/** + * Creates a non-partitioned Merge-On-Read table at test runtime that is configured to use a custom record + * merger ({@link KeyBasedTestRecordMerger}) via {@link RecordMergeMode#CUSTOM}. The table is written with one + * {@code insert} (producing base files) followed by one {@code upsert} of the same keys (producing log files), + * with inline compaction disabled so the log files survive and must be merged at read time. + *

    + * Two tables are registered in the metastore: a read-optimized table (base files only) and a real-time table + * (suffix {@code _rt}) that merges base + log files through the file group reader. + *

    + * Data is laid out so the key-based merge result is distinguishable from both the base-only view and the + * built-in newest-wins behavior (see {@code TestHudiCustomMerger}). + */ +public class CustomMergerHudiTablesInitializer + implements HudiTablesInitializer +{ + public static final String TABLE_NAME = "custom_merger_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + private static final String RECORD_KEY_FIELD = "key"; + private static final String ORDERING_FIELD = "ts"; + private static final String PARTITION_PATH = ""; + + private static final List DATA_COLUMNS = ImmutableList.of( + new Column("_hoodie_commit_time", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_commit_seqno", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_record_key", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_partition_path", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_file_name", HIVE_STRING, Optional.empty(), Map.of()), + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + + @Override + public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) + throws Exception + { + TrinoFileSystem fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(TrinoFileSystemFactory.class) + .create(ConnectorIdentity.ofUser("test")); + HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(HiveMetastoreFactory.class) + .createMetastore(Optional.empty()); + + Location tableLocation = externalLocation.appendPath(TABLE_NAME); + + java.nio.file.Path tempDir = createTempDirectory("custom-merger-mor"); + try { + java.nio.file.Path tempTableDir = tempDir.resolve(TABLE_NAME); + writeTable(new Path(tempTableDir.toUri())); + ResourceHudiTablesInitializer.copyDir(tempTableDir, fileSystem, tableLocation); + + metastore.createTable(createTableDefinition(schemaName, TABLE_NAME, tableLocation, false), PrincipalPrivileges.NO_PRIVILEGES); + metastore.createTable(createTableDefinition(schemaName, RT_TABLE_NAME, tableLocation, true), PrincipalPrivileges.NO_PRIVILEGES); + } + finally { + deleteRecursively(tempDir, ALLOW_INSECURE); + } + } + + private static void writeTable(Path tablePath) + { + Schema schema = createAvroSchema(); + try (HoodieJavaWriteClient writeClient = createWriteClient(schema, tablePath)) { + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = writeClient.startCommit(); + List firstStatuses = writeClient.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 1L), + record(schema, "k2", "k2_base", 100L, 1L)), firstCommit); + writeClient.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1 update has a larger value (99 > 10) -> keep-max keeps the update. + // k2 update has a smaller value (5 < 100) -> keep-max keeps the original base record. + String secondCommit = writeClient.startCommit(); + List secondStatuses = writeClient.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 2L), + record(schema, "k2", "k2_updated", 5L, 2L)), secondCommit); + writeClient.commit(secondCommit, secondStatuses); + } + } + + private static HoodieJavaWriteClient createWriteClient(Schema schema, Path tablePath) + { + Configuration conf = new Configuration(); + try { + HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.MERGE_ON_READ) + .setTableName(TABLE_NAME) + .setTimelineLayoutVersion(1) + .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordKeyFields(RECORD_KEY_FIELD) + .setOrderingFields(ORDERING_FIELD) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(KeyBasedTestRecordMerger.MERGE_STRATEGY_ID) + .initTable(new HadoopStorageConfiguration(conf), tablePath.toString()); + } + catch (IOException e) { + throw new RuntimeException("Could not init table " + TABLE_NAME, e); + } + + HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder() + .withPath(tablePath.toString()) + .withSchema(schema.toString()) + .withParallelism(2, 2) + .withDeleteParallelism(2) + .forTable(TABLE_NAME) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + // Ordering field is carried by the table config (setOrderingFields); avoid the deprecated + // withPreCombineField builder method which fails the -Werror compile gate. + .withRecordMergeMode(RecordMergeMode.CUSTOM) + .withRecordMergeStrategyId(KeyBasedTestRecordMerger.MERGE_STRATEGY_ID) + .withRecordMergeImplClasses(KeyBasedTestRecordMerger.class.getName()) + // Keep log files around so the custom merger runs at read time. + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .withMaxNumDeltaCommitsBeforeCompaction(100) + .build()) + .withEmbeddedTimelineServerEnabled(false) + .withMarkersType(MarkerType.DIRECT.name()) + // MDT writes require hbase deps not present in the Trino runtime; disable as other initializers do. + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), cfg); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(ORDERING_FIELD, ts); + HoodieKey hoodieKey = new HoodieKey(key, PARTITION_PATH); + return new HoodieAvroRecord<>(hoodieKey, new HoodieAvroPayload(Option.of(record)), null); + } + + private static Schema createAvroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + private static Table createTableDefinition(String schemaName, String tableName, Location location, boolean isRtTable) + { + StorageFormat storageFormat = StorageFormat.create( + PARQUET_HIVE_SERDE_CLASS, + isRtTable ? HUDI_PARQUET_REALTIME_INPUT_FORMAT : HUDI_PARQUET_INPUT_FORMAT, + MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); + + return Table.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setTableType(EXTERNAL_TABLE.name()) + .setOwner(Optional.of("public")) + .setDataColumns(DATA_COLUMNS) + .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) + .withStorage(storageBuilder -> storageBuilder + .setStorageFormat(storageFormat) + .setLocation(location.toString())) + .build(); + } +} diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTableUnzipper.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTableUnzipper.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTableUnzipper.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTableUnzipper.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTablesInitializer.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTablesInitializer.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTablesInitializer.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTestUtils.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTestUtils.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/HudiTestUtils.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/HudiTestUtils.java diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java new file mode 100644 index 0000000000000..e4f3f0e758a02 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java @@ -0,0 +1,481 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Column; +import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveType; +import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.StorageFormat; +import io.trino.metastore.Table; +import io.trino.plugin.hudi.HudiConnector; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.QueryRunner; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.common.HoodieJavaEngineContext; +import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; +import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; +import static io.trino.metastore.HiveType.HIVE_BOOLEAN; +import static io.trino.metastore.HiveType.HIVE_DOUBLE; +import static io.trino.metastore.HiveType.HIVE_INT; +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static java.lang.String.format; +import static java.nio.file.Files.createTempDirectory; + +/** + * Creates a non-partitioned Merge-On-Read table configured with the {@link MaxRankRecordMerger} custom merger + * ({@link RecordMergeMode#CUSTOM}) and drives a controlled sequence of commits so an end-to-end test can read the + * table through Trino after every commit and validate the merged result. + *

    + * The schema has 30 data columns (record key, the {@code merge_rank} decision column, an ordering field, and 27 + * additional columns spanning string/long/int/double/boolean types). {@link #NUM_RECORDS} record keys are + * bulk-inserted in the first commit; each subsequent commit upserts roughly two thirds of the keys with freshly + * derived values. Inline compaction is disabled so every delta commit survives as a log file and must be merged at + * read time. + *

    + * Every record column is a pure deterministic function of {@code (recordIndex, commitNumber)} (see + * {@link #valueFor}). Because the merge keeps the record with the maximum {@code merge_rank} (ties to the newer + * commit), the winning commit for each key is known in closed form, so the full expected row for the real-time + * table is reproduced exactly by {@link #expectedRows()} without reading anything back. + *

    + * Two metastore tables are registered: a read-optimized table (base files only) and a real-time table (suffix + * {@code _rt}) that merges base + log files through the file group reader. The Hudi write client writes to a local + * staging directory; after each commit the staging directory is mirrored into the Trino filesystem location the + * connector reads from (see {@link #syncToTrino()}). + */ +public class IncrementalCustomMergerHudiTablesInitializer + implements HudiTablesInitializer +{ + public static final String TABLE_NAME = "custom_merger_e2e"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public static final int TOTAL_COMMITS = 20; + public static final int NUM_RECORDS = 10_000; + + private static final String RECORD_KEY_FIELD = "key"; + private static final String RANK_FIELD = MaxRankRecordMerger.RANK_COLUMN; + private static final String ORDERING_FIELD = "ts"; + private static final String PARTITION_PATH = ""; + + private enum Kind + { + STRING, LONG, INT, DOUBLE, BOOLEAN + } + + private record ColumnSpec(String name, Kind kind) {} + + /** The 30 data columns, in the order they appear in the schema and in query projections. */ + private static final List COLUMN_SPECS = buildColumnSpecs(); + + private static final List DATA_COLUMNS = buildDataColumns(); + private static final Schema AVRO_SCHEMA = buildAvroSchema(); + + // Mutable state driven across commits. + private TrinoFileSystem fileSystem; + private Location tableLocation; + private java.nio.file.Path stagingDir; + private Path stagingTablePath; + private HoodieJavaWriteClient writeClient; + + /** Winning commit per record index under the max-rank merge policy (-1 = not yet inserted). */ + private final int[] winningCommit = new int[NUM_RECORDS]; + /** Most recent commit that touched each record index (used to measure divergence from newest-wins). */ + private final int[] latestCommit = new int[NUM_RECORDS]; + private int currentCommit; + + @Override + public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) + throws Exception + { + fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(TrinoFileSystemFactory.class) + .create(ConnectorIdentity.ofUser("test")); + HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(HiveMetastoreFactory.class) + .createMetastore(Optional.empty()); + + tableLocation = externalLocation.appendPath(TABLE_NAME); + stagingDir = createTempDirectory("custom-merger-e2e"); + stagingTablePath = new Path(stagingDir.resolve(TABLE_NAME).toUri()); + + java.util.Arrays.fill(winningCommit, -1); + java.util.Arrays.fill(latestCommit, -1); + + initTable(); + writeClient = createWriteClient(); + + // First commit: bulk insert all keys (produces the base parquet files the read-optimized table reads). + currentCommit = 1; + String firstCommit = writeClient.startCommit(); + List statuses = writeClient.bulkInsert(buildRecords(currentCommit), firstCommit); + writeClient.commit(firstCommit, statuses); + recordCommit(currentCommit); + syncToTrino(); + + metastore.createTable(createTableDefinition(schemaName, TABLE_NAME, tableLocation, false), PrincipalPrivileges.NO_PRIVILEGES); + metastore.createTable(createTableDefinition(schemaName, RT_TABLE_NAME, tableLocation, true), PrincipalPrivileges.NO_PRIVILEGES); + } + + /** + * Writes the next delta commit (upsert of ~2/3 of the keys) and mirrors it into the Trino filesystem so the + * connector observes the new commit on the next query. Must be called after {@link #initializeTables}. + */ + public void writeAndSyncNextCommit() + { + currentCommit++; + String commitTime = writeClient.startCommit(); + List statuses = writeClient.upsert(buildRecords(currentCommit), commitTime); + writeClient.commit(commitTime, statuses); + recordCommit(currentCommit); + syncToTrino(); + } + + /** Ordered data column names (record key first), matching {@link #expectedRows()} value positions. */ + public List dataColumnNames() + { + return COLUMN_SPECS.stream().map(ColumnSpec::name).collect(toImmutableList()); + } + + /** Expected real-time (merged) rows after the commits written so far: key -> values in column order. */ + public Map expectedRows() + { + Map rows = new LinkedHashMap<>(); + for (int ki = 0; ki < NUM_RECORDS; ki++) { + if (winningCommit[ki] < 0) { + continue; + } + Object[] row = rowFor(ki, winningCommit[ki]); + rows.put((String) row[0], row); + } + return rows; + } + + /** Expected read-optimized (base-file-only) rows: always the first-commit values for every key. */ + public Map baseRows() + { + Map rows = new LinkedHashMap<>(); + for (int ki = 0; ki < NUM_RECORDS; ki++) { + Object[] row = rowFor(ki, 1); + rows.put((String) row[0], row); + } + return rows; + } + + /** Number of keys whose merged winner is not their most recently committed version (custom != newest-wins). */ + public int divergentKeyCount() + { + int count = 0; + for (int ki = 0; ki < NUM_RECORDS; ki++) { + if (winningCommit[ki] >= 0 && winningCommit[ki] != latestCommit[ki]) { + count++; + } + } + return count; + } + + public void close() + throws IOException + { + if (writeClient != null) { + writeClient.close(); + writeClient = null; + } + if (stagingDir != null) { + deleteRecursively(stagingDir, ALLOW_INSECURE); + stagingDir = null; + } + } + + private void recordCommit(int commit) + { + for (int ki = 0; ki < NUM_RECORDS; ki++) { + if (!isUpdated(ki, commit)) { + continue; + } + latestCommit[ki] = commit; + int prev = winningCommit[ki]; + // Mirror MaxRankRecordMerger: the incoming (newer) record wins on a tie or a strictly larger rank. + if (prev < 0 || mergeRank(ki, commit) >= mergeRank(ki, prev)) { + winningCommit[ki] = commit; + } + } + } + + private List> buildRecords(int commit) + { + List> records = new ArrayList<>(); + for (int ki = 0; ki < NUM_RECORDS; ki++) { + if (isUpdated(ki, commit)) { + records.add(record(ki, commit)); + } + } + return records; + } + + private static boolean isUpdated(int recordIndex, int commit) + { + // The first commit inserts every key; later commits upsert ~2/3 of the keys, rotating which third is skipped. + return commit == 1 || Math.floorMod(recordIndex + commit, 3) != 0; + } + + private static HoodieRecord record(int recordIndex, int commit) + { + GenericRecord record = new GenericData.Record(AVRO_SCHEMA); + for (int i = 0; i < COLUMN_SPECS.size(); i++) { + record.put(COLUMN_SPECS.get(i).name(), valueFor(i, recordIndex, commit)); + } + HoodieKey hoodieKey = new HoodieKey(key(recordIndex), PARTITION_PATH); + return new HoodieAvroRecord<>(hoodieKey, new HoodieAvroPayload(Option.of(record)), null); + } + + private static Object[] rowFor(int recordIndex, int commit) + { + Object[] row = new Object[COLUMN_SPECS.size()]; + for (int i = 0; i < row.length; i++) { + row[i] = valueFor(i, recordIndex, commit); + } + return row; + } + + /** + * The single source of truth for every column value, used both when writing records and when reproducing the + * expected rows. Returns boxed types matching how Trino surfaces the column (String/Long/Integer/Double/Boolean). + */ + private static Object valueFor(int columnIndex, int recordIndex, int commit) + { + ColumnSpec spec = COLUMN_SPECS.get(columnIndex); + if (spec.name().equals(RECORD_KEY_FIELD)) { + return key(recordIndex); + } + if (spec.name().equals(RANK_FIELD)) { + return mergeRank(recordIndex, commit); + } + if (spec.name().equals(ORDERING_FIELD)) { + return (long) commit; + } + return switch (spec.kind()) { + case STRING -> "v_" + columnIndex + "_" + recordIndex + "_" + commit; + case LONG -> recordIndex * 1_000_003L + (long) commit * columnIndex; + case INT -> (int) Math.floorMod(recordIndex * 31L + (long) commit * columnIndex, 1_000_000L); + case DOUBLE -> recordIndex + commit * 0.5 + columnIndex * 0.125; + case BOOLEAN -> (recordIndex + commit + columnIndex) % 2 == 0; + }; + } + + private static String key(int recordIndex) + { + return format("key%05d", recordIndex); + } + + /** Deterministic, non-monotonic rank in [0, 100000) so the winning commit is rarely the latest one. */ + private static long mergeRank(int recordIndex, int commit) + { + return Math.floorMod(recordIndex * 2_654_435_761L + commit * 40_503L, 100_000L); + } + + private void syncToTrino() + { + try { + if (fileSystem.directoryExists(tableLocation).orElse(false)) { + fileSystem.deleteDirectory(tableLocation); + } + ResourceHudiTablesInitializer.copyDir(stagingDir.resolve(TABLE_NAME), fileSystem, tableLocation); + } + catch (IOException e) { + throw new RuntimeException("Failed to sync staged Hudi table to Trino filesystem", e); + } + } + + private void initTable() + { + Configuration conf = new Configuration(); + try { + HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.MERGE_ON_READ) + .setTableName(TABLE_NAME) + .setTimelineLayoutVersion(1) + .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordKeyFields(RECORD_KEY_FIELD) + .setOrderingFields(ORDERING_FIELD) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) + .initTable(new HadoopStorageConfiguration(conf), stagingTablePath.toString()); + } + catch (IOException e) { + throw new RuntimeException("Could not init table " + TABLE_NAME, e); + } + } + + private HoodieJavaWriteClient createWriteClient() + { + Configuration conf = new Configuration(); + HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder() + .withPath(stagingTablePath.toString()) + .withSchema(AVRO_SCHEMA.toString()) + .withParallelism(2, 2) + .withDeleteParallelism(2) + .forTable(TABLE_NAME) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + .withRecordMergeMode(RecordMergeMode.CUSTOM) + .withRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) + .withRecordMergeImplClasses(MaxRankRecordMerger.class.getName()) + // Keep log files around so the custom merger runs at read time for every delta commit. + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .withMaxNumDeltaCommitsBeforeCompaction(TOTAL_COMMITS + 100) + .build()) + .withEmbeddedTimelineServerEnabled(false) + .withMarkersType(MarkerType.DIRECT.name()) + // MDT writes require hbase deps not present in the Trino runtime; disable as other initializers do. + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .build(); + return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), cfg); + } + + private static Table createTableDefinition(String schemaName, String tableName, Location location, boolean isRtTable) + { + StorageFormat storageFormat = StorageFormat.create( + PARQUET_HIVE_SERDE_CLASS, + isRtTable ? HUDI_PARQUET_REALTIME_INPUT_FORMAT : HUDI_PARQUET_INPUT_FORMAT, + MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); + + return Table.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setTableType(EXTERNAL_TABLE.name()) + .setOwner(Optional.of("public")) + .setDataColumns(DATA_COLUMNS) + .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) + .withStorage(storageBuilder -> storageBuilder + .setStorageFormat(storageFormat) + .setLocation(location.toString())) + .build(); + } + + private static List buildColumnSpecs() + { + ImmutableList.Builder specs = ImmutableList.builder(); + specs.add(new ColumnSpec(RECORD_KEY_FIELD, Kind.STRING)); + specs.add(new ColumnSpec(RANK_FIELD, Kind.LONG)); + specs.add(new ColumnSpec(ORDERING_FIELD, Kind.LONG)); + for (int i = 0; i < 7; i++) { + specs.add(new ColumnSpec("s" + i, Kind.STRING)); + } + for (int i = 0; i < 7; i++) { + specs.add(new ColumnSpec("l" + i, Kind.LONG)); + } + for (int i = 0; i < 6; i++) { + specs.add(new ColumnSpec("i" + i, Kind.INT)); + } + for (int i = 0; i < 5; i++) { + specs.add(new ColumnSpec("d" + i, Kind.DOUBLE)); + } + for (int i = 0; i < 2; i++) { + specs.add(new ColumnSpec("b" + i, Kind.BOOLEAN)); + } + List built = specs.build(); + if (built.size() != 30) { + throw new IllegalStateException("Expected 30 data columns but built " + built.size()); + } + return built; + } + + private static List buildDataColumns() + { + ImmutableList.Builder columns = ImmutableList.builder(); + // Hudi metadata columns prepended to every table. + for (String meta : List.of("_hoodie_commit_time", "_hoodie_commit_seqno", "_hoodie_record_key", "_hoodie_partition_path", "_hoodie_file_name")) { + columns.add(new Column(meta, HIVE_STRING, Optional.empty(), Map.of())); + } + for (ColumnSpec spec : COLUMN_SPECS) { + columns.add(new Column(spec.name(), hiveType(spec.kind()), Optional.empty(), Map.of())); + } + return columns.build(); + } + + private static Schema buildAvroSchema() + { + List fields = new ArrayList<>(); + for (ColumnSpec spec : COLUMN_SPECS) { + fields.add(new Schema.Field(spec.name(), Schema.create(avroType(spec.kind())))); + } + return Schema.createRecord(TABLE_NAME, null, null, false, fields); + } + + private static HiveType hiveType(Kind kind) + { + return switch (kind) { + case STRING -> HIVE_STRING; + case LONG -> HIVE_LONG; + case INT -> HIVE_INT; + case DOUBLE -> HIVE_DOUBLE; + case BOOLEAN -> HIVE_BOOLEAN; + }; + } + + private static Schema.Type avroType(Kind kind) + { + return switch (kind) { + case STRING -> Schema.Type.STRING; + case LONG -> Schema.Type.LONG; + case INT -> Schema.Type.INT; + case DOUBLE -> Schema.Type.DOUBLE; + case BOOLEAN -> Schema.Type.BOOLEAN; + }; + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/KeyBasedTestRecordMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/KeyBasedTestRecordMerger.java new file mode 100644 index 0000000000000..c970e4086628c --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/KeyBasedTestRecordMerger.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.table.read.BufferedRecord; + +import java.io.IOException; + +/** + * Test-only custom {@link HoodieRecordMerger} used to verify that the Hudi Trino connector resolves and + * applies a user-supplied record merger for Merge-On-Read tables whose record merge mode is {@code CUSTOM}. + *

    + * The merge policy is deliberately distinct from the built-in mergers and depends only on the record key + * (which is projection-independent): for keys ending in an odd digit it keeps the newer record, otherwise it + * keeps the older one. This is neither the built-in newest-wins (event-time) behavior nor the base-only view, + * so its effect is observable in query results regardless of which columns a query projects. + *

    + * Operates on Avro records ({@link HoodieRecord.HoodieRecordType#AVRO}), which is the record type the Trino + * reader context uses ({@code EngineType.JAVA}). + */ +public class KeyBasedTestRecordMerger + implements HoodieRecordMerger +{ + /** + * Unique strategy id identifying this custom merger. The test table's + * {@code hoodie.record.merge.strategy.id} is set to this value so the reader resolves this implementation. + */ + public static final String MERGE_STRATEGY_ID = "f9b5c1a2-0d3e-4c7a-8b6f-2a1e4d9c0b7a"; + + @Override + public BufferedRecord merge(BufferedRecord older, BufferedRecord newer, RecordContext recordContext, TypedProperties props) + throws IOException + { + // Deletes are passed through so tombstones still win; this test does not exercise deletes. + if (older == null || older.isDelete() || newer.isDelete()) { + return newer; + } + return keepNewer(newer.getRecordKey()) ? newer : older; + } + + private static boolean keepNewer(String recordKey) + { + char last = recordKey.charAt(recordKey.length() - 1); + return Character.isDigit(last) && ((last - '0') % 2 == 1); + } + + /** + * Merging depends only on the record key, so it works on projected records. Without this override the + * file-group reader reads ALL table columns for merging, which the Trino connector does not support -- + * it can only resolve columns in the projection plus the merger's declared mandatory fields. + */ + @Override + public boolean isProjectionCompatible() + { + return true; + } + + @Override + public HoodieRecord.HoodieRecordType getRecordType() + { + return HoodieRecord.HoodieRecordType.AVRO; + } + + @Override + public String getMergingStrategy() + { + return MERGE_STRATEGY_ID; + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java new file mode 100644 index 0000000000000..eff623d0a6e02 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.read.BufferedRecord; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedHashSet; + +/** + * Test-only custom {@link HoodieRecordMerger} that keeps, for each record key, the record with the largest value + * in the {@code merge_rank} column. Ties are broken in favor of the newer (later-committed) record. + *

    + * This policy is deliberately distinct from every built-in merge mode: it depends on an arbitrary data column + * rather than commit time (newest-wins) or the ordering/precombine field (event-time ordering). Because the + * winning commit for a key is frequently not the most recent one, the merged result is observably + * different from both the read-optimized (base-only) view and the built-in newest-wins behavior, which is what + * {@code TestHudiCustomMergerEndToEnd} validates after every commit. + *

    + * Operates on Avro records ({@link HoodieRecord.HoodieRecordType#AVRO}), the record type the Trino reader + * context uses ({@code EngineType.JAVA}). + */ +public class MaxRankRecordMerger + implements HoodieRecordMerger +{ + /** + * Unique strategy id identifying this custom merger. The test table's + * {@code hoodie.record.merge.strategy.id} is set to this value so the reader resolves this implementation. + */ + public static final String MERGE_STRATEGY_ID = "3c2d7e10-9a4b-4f81-bd6e-7f0a1c5e8d24"; + + /** Name of the column whose value decides the merge. */ + public static final String RANK_COLUMN = "merge_rank"; + + @Override + public BufferedRecord merge(BufferedRecord older, BufferedRecord newer, RecordContext recordContext, TypedProperties props) + throws IOException + { + // Deletes are passed through so tombstones still win; this test does not exercise deletes. + if (older == null || older.isDelete() || newer.isDelete()) { + return newer; + } + long olderRank = rankOf(older, recordContext); + long newerRank = rankOf(newer, recordContext); + // Keep the newer record on ties so the policy stays deterministic and matches the expected-state fold. + return newerRank >= olderRank ? newer : older; + } + + private static long rankOf(BufferedRecord record, RecordContext recordContext) + { + HoodieSchema schema = recordContext.getSchemaFromBufferRecord(record); + Object value = recordContext.getValue(record.getRecord(), schema, RANK_COLUMN); + return ((Number) value).longValue(); + } + + @Override + public String[] getMandatoryFieldsForMerging(HoodieSchema dataSchema, HoodieTableConfig cfg, TypedProperties properties) + { + // Declare merge_rank (read in merge()) as mandatory, on top of the default key/ordering fields, so the + // reader includes it in the read schema even when a query does not project it. + LinkedHashSet fields = new LinkedHashSet<>( + Arrays.asList(HoodieRecordMerger.super.getMandatoryFieldsForMerging(dataSchema, cfg, properties))); + fields.add(RANK_COLUMN); + return fields.toArray(new String[0]); + } + + /** + * Merging depends only on {@code merge_rank}, which is declared mandatory above, so it works on projected + * records. Without this override the file-group reader reads ALL table columns for merging, which the Trino + * connector does not support -- it can only resolve columns in the projection plus the merger's declared + * mandatory fields. + */ + @Override + public boolean isProjectionCompatible() + { + return true; + } + + @Override + public HoodieRecord.HoodieRecordType getRecordType() + { + return HoodieRecord.HoodieRecordType.AVRO; + } + + @Override + public String getMergingStrategy() + { + return MERGE_STRATEGY_ID; + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java new file mode 100644 index 0000000000000..89db0e7dffbeb --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +/** + * Test-only merger that is identical to {@link KeyBasedTestRecordMerger} except that it reports + * {@code isProjectionCompatible() == false}, which makes the file-group reader request the FULL table schema + * instead of the connector's projection. + *

    + * The Trino connector can only resolve columns that are in the read projection (plus Hudi meta columns), so a + * data column outside the projection is rejected with a {@code NOT_SUPPORTED} error. This merger exists purely + * to exercise that guard; it shares {@link KeyBasedTestRecordMerger#MERGE_STRATEGY_ID} so it resolves against + * the same test table. + */ +public class NonProjectionCompatibleTestRecordMerger + extends KeyBasedTestRecordMerger +{ + @Override + public boolean isProjectionCompatible() + { + return false; + } +} diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java similarity index 95% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java index 10d1b9873ad35..47902849e0d96 100644 --- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java @@ -19,8 +19,6 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.hdfs.HdfsContext; -import io.trino.hdfs.HdfsEnvironment; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; @@ -45,6 +43,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; import org.apache.hudi.client.common.HoodieJavaEngineContext; import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; import org.apache.hudi.common.config.HoodieMetadataConfig; @@ -54,7 +53,6 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.marker.MarkerType; -import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieIndexConfig; @@ -64,12 +62,10 @@ import org.intellij.lang.annotations.Language; import java.io.IOException; -import java.time.Instant; import java.time.LocalDate; import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; @@ -90,9 +86,7 @@ import static io.trino.metastore.HiveType.HIVE_INT; import static io.trino.metastore.HiveType.HIVE_LONG; import static io.trino.metastore.HiveType.HIVE_STRING; -import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; -import static io.trino.testing.TestingConnectorSession.SESSION; import static java.lang.String.format; import static java.nio.file.Files.createTempDirectory; import static java.util.Collections.unmodifiableList; @@ -112,7 +106,6 @@ public class TpchHudiTablesInitializer new Column("_hoodie_record_key", HIVE_STRING, Optional.empty(), Map.of()), new Column("_hoodie_partition_path", HIVE_STRING, Optional.empty(), Map.of()), new Column("_hoodie_file_name", HIVE_STRING, Optional.empty(), Map.of())); - private static final HdfsContext CONTEXT = new HdfsContext(SESSION); private final List> tpchTables; @@ -156,7 +149,7 @@ public void initializeTables(QueryRunner queryRunner, Location externalLocation, public void load(TpchTable tpchTables, QueryRunner queryRunner, java.nio.file.Path tableDirectory) { - try (HoodieJavaWriteClient writeClient = createWriteClient(tpchTables, HDFS_ENVIRONMENT, new Path(tableDirectory.toUri()))) { + try (HoodieJavaWriteClient writeClient = createWriteClient(tpchTables, new Path(tableDirectory.toUri()))) { RecordConverter recordConverter = createRecordConverter(tpchTables); @Language("SQL") String sql = generateScanSql(TPCH_TINY, tpchTables); @@ -168,9 +161,9 @@ public void load(TpchTable tpchTables, QueryRunner queryRunner, java.nio.file .map(MaterializedRow::getFields) .map(recordConverter::toRecord) .collect(Collectors.toList()); - String timestamp = HoodieInstantTimeGenerator.formatDate(Date.from(Instant.now())); - writeClient.startCommitWithTime(timestamp); - writeClient.insert(records, timestamp); + String instantTime = writeClient.startCommit(); + List writeStatuses = writeClient.insert(records, instantTime); + writeClient.commit(instantTime, writeStatuses); } } @@ -211,10 +204,10 @@ private static Table createTableDefinition(String schemaName, TpchTable table .build(); } - private static HoodieJavaWriteClient createWriteClient(TpchTable table, HdfsEnvironment hdfsEnvironment, Path tablePath) + private static HoodieJavaWriteClient createWriteClient(TpchTable table, Path tablePath) { Schema schema = createAvroSchema(table); - Configuration conf = hdfsEnvironment.getConfiguration(CONTEXT, tablePath); + Configuration conf = new Configuration(); try { HoodieTableMetaClient.newTableBuilder() diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/TypeInfoHelper.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/TypeInfoHelper.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/TypeInfoHelper.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/testing/TypeInfoHelper.java diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationAssertions.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationAssertions.java new file mode 100644 index 0000000000000..546db8446a9e5 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationAssertions.java @@ -0,0 +1,176 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.util; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import io.airlift.log.Logger; +import io.trino.plugin.hudi.util.FileOperationUtils.FileOperation; +import io.trino.testing.QueryRunner; +import org.intellij.lang.annotations.Language; + +import java.util.Comparator; +import java.util.function.Supplier; + +import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.getCacheOperationSpans; +import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.getFileLocation; +import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.isTrinoSchemaOrPermissions; +import static io.trino.testing.MultisetAssertions.assertMultisetsEqual; +import static java.util.stream.Collectors.toCollection; + +public final class FileOperationAssertions +{ + private static final Logger log = Logger.get(FileOperationAssertions.class); + + private FileOperationAssertions() {} + + /** + * Asserts that file system accesses match expected operations. + * This version uses manual filtering for Input/InputFile operations. + * On assertion failure, logs a detailed comparison at WARN level to aid debugging. + */ + public static void assertFileSystemAccesses( + QueryRunner queryRunner, + @Language("SQL") String query, + Multiset expectedCacheAccesses) + throws InterruptedException + { + queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); + // Async table-stats computation can outlive the synchronous query and emit spans into + // the exporter after execute returns. A fixed Thread.sleep races with this: when stats + // from query N is still running while query N+1's measurement happens, spans leak + // across the boundary and counts get scrambled. Poll until the span set is stable for + // two consecutive reads. + Multiset actualCacheAccesses = waitForStableFileOperations(() -> getFileOperations(queryRunner)); + try { + assertMultisetsEqual(actualCacheAccesses, expectedCacheAccesses); + } + catch (AssertionError e) { + logFileAccessDebugInfo(queryRunner, actualCacheAccesses, expectedCacheAccesses); + throw e; + } + } + + /** + * Asserts that file system accesses match expected operations for Alluxio cache tests. + * This version uses getCacheOperationSpans for filtering. + * On assertion failure, logs a detailed comparison at WARN level to aid debugging. + */ + public static void assertAlluxioFileSystemAccesses( + QueryRunner queryRunner, + @Language("SQL") String query, + Multiset expectedCacheAccesses) + throws InterruptedException + { + queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query); + // See assertFileSystemAccesses for the rationale behind polling instead of a fixed sleep. + Multiset actualCacheAccesses = waitForStableFileOperations(() -> getAlluxioFileOperations(queryRunner)); + try { + assertMultisetsEqual(actualCacheAccesses, expectedCacheAccesses); + } + catch (AssertionError e) { + logFileAccessDebugInfo(queryRunner, actualCacheAccesses, expectedCacheAccesses); + throw e; + } + } + + /** + * Returns the file-operation set once two consecutive reads (200ms apart) agree. Bounded by a + * 30-second ceiling so a runaway test fails loudly instead of hanging. + */ + private static Multiset waitForStableFileOperations(Supplier> reader) + throws InterruptedException + { + long deadlineMillis = System.currentTimeMillis() + 30_000L; + Multiset previous = null; + while (System.currentTimeMillis() < deadlineMillis) { + Thread.sleep(200L); + Multiset current = reader.get(); + if (previous != null && current.equals(previous)) { + return current; + } + previous = current; + } + return previous != null ? previous : reader.get(); + } + + /** + * Gets file operations from query runner spans using manual filtering. + */ + public static Multiset getFileOperations(QueryRunner queryRunner) + { + return queryRunner.getSpans().stream() + .filter(span -> span.getName().startsWith("Input.") || span.getName().startsWith("InputFile.") || span.getName().startsWith("FileSystemCache.")) + .filter(span -> !span.getName().startsWith("InputFile.newInput")) + .filter(span -> !span.getName().startsWith("InputFile.exists")) + .filter(span -> !isTrinoSchemaOrPermissions(getFileLocation(span))) + .map(FileOperation::create) + .collect(toCollection(HashMultiset::create)); + } + + /** + * Gets file operations for Alluxio cache tests using getCacheOperationSpans. + */ + public static Multiset getAlluxioFileOperations(QueryRunner queryRunner) + { + return getCacheOperationSpans(queryRunner) + .stream() + .filter(span -> !span.getName().startsWith("InputFile.exists")) + .map(FileOperation::create) + .collect(toCollection(HashMultiset::create)); + } + + private static void logFileAccessDebugInfo( + QueryRunner queryRunner, + Multiset actualCacheAccesses, + Multiset expectedCacheAccesses) + { + // Log all file paths accessed for debugging + log.warn("=== All File Paths Accessed ==="); + queryRunner.getSpans().stream() + .filter(span -> span.getName().equals("InputFile.lastModified") || span.getName().equals("InputFile.length")) + .forEach(span -> log.warn("%s: %s", span.getName(), getFileLocation(span))); + + // Log actual and expected cache accesses + log.warn("=== Actual Cache Accesses ==="); + logSortedMultiset(actualCacheAccesses); + + log.warn("=== Expected Cache Accesses ==="); + logSortedMultiset(expectedCacheAccesses); + + // Calculate and log differences + Multiset extraInActual = HashMultiset.create(actualCacheAccesses); + extraInActual.removeAll(expectedCacheAccesses); + + Multiset missingFromActual = HashMultiset.create(expectedCacheAccesses); + missingFromActual.removeAll(actualCacheAccesses); + + if (!extraInActual.isEmpty()) { + log.warn("=== Extra in Actual (not expected) ==="); + logSortedMultiset(extraInActual); + } + + if (!missingFromActual.isEmpty()) { + log.warn("=== Missing from Actual (expected but not found) ==="); + logSortedMultiset(missingFromActual); + } + } + + private static void logSortedMultiset(Multiset multiset) + { + multiset.entrySet().stream() + .sorted(Comparator.comparing(a -> a.getElement().toString())) + .forEach(entry -> log.warn("%s: %s", entry.getElement(), entry.getCount())); + } +} diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/FileOperationUtils.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationUtils.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/FileOperationUtils.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/util/FileOperationUtils.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsExtendedNullFilterTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsExtendedNullFilterTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsExtendedNullFilterTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsExtendedNullFilterTest.java diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsTest.java similarity index 100% rename from hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsTest.java rename to hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTupleDomainUtilsTest.java diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md similarity index 91% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md index 9883fe6db2a77..c8a1da4cf8f48 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v6_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md similarity index 91% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md index ade3e61723df0..26bb6c2a659bb 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_comprehensive_types_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md similarity index 72% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md index 39e06481bcd78..63fc089b0e7cc 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_table_with_field_names_in_caps.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_tbl.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_tbl.zip similarity index 74% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_pt_tbl.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_pt_tbl.zip index 2f2238b22339f2498d1d9893c20866dc7980e671..47c0532a9369de80b12e8d52e53a83460f7bcfc3 100644 GIT binary patch delta 918 zcmaDB{UVwtz?+$civa{wGbi$RFh%um4C`ZLQq7$Fj4_N!G;XpTQw)fj!*mu%ZO&$% z!U2@r%*a#B2$40Jyoy%?#Js`_R%;dKzuOsJUXjk~F2H#_WsRX*3# ze+?CmSRha>bXwc;rMa?nbUUJDGw~9+>4Ix zU4HB>$95b229c=~+*{W4Y~%V@)AYC9?ZQ?omu{|Uv(wWiK6IPgBXRw-%{$QzW@~lI zMPjddOB6ln-*>6RCN}2$vCBVGE;7rsaI4xi%(^_?q^{&w_RMRqecUs7#JKn7tYQ9P zQ0RO0o~=c|*X1m+PyQ|4nqn&Jur`(9@LZNpGalG|eQ_vYqw~^`UmvWjR(|=g+eF;! z^&g!-=UWezJ=T#s(fX)j$D05ffAyo$0W*bfu=c$;UnEkoEdJv=F1a}NdPY!!6agkj z4hCRy3zGE!C5+8;O$#Yt-BXNvY;q)-2!@bx}$yKO=L^jq zL$=p9f{tAiT^M1$aYahyrBLhOUw74yZVaEDsNyP6?gx zYZ4NPvTXuajC}X(jgJIhm2qg^TbFiPcEY6W1x#gkv~6~7U*~&tqryEcjrwBc z2l3t9cfSQRJzCawIcH7MYi&6 zf0c4%N9?l=VFz|?*>op(OWg#6Cnx0RUN13^lq!olWmCTLhe4&hq46^wP~Kys{2!QyC7=Wcf7Xfn90F;ed_J z;aas7vhzc%euU2AUg}#X_V2k<;`Tx@{wGd_HZf%y^OjG3q`Tq_=Nl&Xl5|Tho6GBe zyyudObFXIrB{dOXLgQcnric((4<=B`m?saRUns&-NC_zM<*J4Nlj7vVs<9xRl3Efd z^|YyFfT#~@O(3d7eG^DdLn9l+o2mhp`>z3(Yu4NZNrRSJ)|17wBqu9quz^)+m4K)O zU75-1+DW`9NpSKsZD(Y)zqDnT++-&|R280Vqb)gkf-Ki$KOH3wkRDzR&NGvA#F!?? vL#2f$Z__aXnfyvehUu0fR8)Slm97fZKBku{aJ9Os5|j7ns + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_field_names_in_caps.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md similarity index 60% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md index 9a7ccd9eaf575..b8e067165e0ed 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_table_with_multi_keys_and_field_names_in_caps.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md similarity index 78% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md index f38d43d149c2b..75239267a8065 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_custom_keygen_pt_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md similarity index 59% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md index f73eb1591343b..39b5fb127dfab 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_table_with_field_names_in_caps.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md similarity index 72% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md index 733443b8619dd..4112d12c5699b 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md similarity index 75% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md index 6d78d7eb6768f..5c727a0cf6342 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md similarity index 78% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md index 119c8ffdd8e7c..e8d65b1394f96 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Revision: 6f65998117a2d1228fc96d36053bd0d394499afe diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_multi_pt_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md similarity index 66% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md index db288b2dfc233..e6b2e02ee0dc9 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_extractable_partition_path.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_cow.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_cow.md similarity index 65% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_cow.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_cow.md index 26f65f3fcd78f..33ee239ad9af0 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_cow.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_cow.md @@ -1,3 +1,20 @@ + + # Hudi Test Resources ## Generating Hudi Resources diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_cow.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_cow.zip similarity index 62% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_cow.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_cow.zip index 019860e88cff39513a039345405a54b53806b879..55632f95ca9fdb24cebaedee821a3f20347729a4 100644 GIT binary patch delta 802 zcmcbkcTSHdz?+$civa|xy(aQ_F;#nQj9_JCX_=`XJvot4pJToFi3|{)yq-;x7bJT3 z?(VkZj0_AXC+}l4M@au=l&cq4k96=~I>m5W-$bKr>o{C@^|_{=yzJhk`S<MN? z8ch;{ox-Ugi$^XT{lB&5CB5zmn61`6-Kefk zett^SkHueFudHMK#PqfG$TH1`F1P1t>#01icVfzHe*Tf=KD$>fQ(5Dp&FPv&<;&te z-WRml&p(a(_SD{J)wNs8Gq{hnDL#5~VlwLn@z>sS>SmpLezIg*imgJ7#MWEVY%i{M zB*f&Ew6>KhEL*BS^{C}kF}~2;8Sva)d9_BmBA{jOPb zn3}oX^9b-M{&BtHS8U4S{h(wZ0!#!P45D$H-!i8%F^R@a4q+<+Qkze)buogtVH~?b zTwzXqE?_d5>>?ZnqUH$4LsH9sVOth(O4%qPjh0d}L|oxgAd}9C$S_@Io}3^iJ-MDu za>W F001AiI|l#& delta 784 zcmX@7cSnyWz?+$civa|z<0kTWFXwy3NlbQ>^ z?Dy{y*7ut}agRiC>ARiJ>*lQf?N!8Y^w91k`_Hqb(_)`L{+VEL*6s4cmODOdixdBC zF8?a(Yjn7DQvc+?$wl`2@2Y*}a#`6>vG2me-76dJ-K(umC^!E0`1kXl$KO?R&TF4F zC*)35{ZmmX_vv2EuMcT)Go5ZVa+t=oWywC)0&T9XY`xl>Tyzg!ighW|Y-zn9UAU-W z(&g(WTHf`(mrvxR-ty4Wj#m;>d<6R=Ej_82!~gi?3Zi@U8sy@mqpm$* zUZgWMJ+8H&e*GiE(tBE)yG*mzUS7Wa-$@_#{k(BAZq=6@n3q_bn6!J^hv(hglOHSH zG^>c4H>usBS3N=8psHMK@t5>ZqW@(NUXHFVO5AM~xB8j%8r#U`(!>rE>s=?Aj%;4- zJ*!^ASM6rT9NzB@8C>mKMIGYSaJ6gehSkS9vAo*VllJhIB>Vq6pKCJ)vrrhw&Uq(=(5CJ9$4u;U8&G(s8 znV3S0CcCkf0IAJ;*t!@&Tn~=jATBGXJ{K_2Og0k^15tg#@sPChQP_5JtC&1ml8F*= z1&M<69}tmYe#p!)`2(}*WD!v%kdTF_46_FtNXU3{m#7{{XctguA6Q6t@-H^7$pT_Z z@}N}sI_t*yaG(nlfLMn?0!T72023v{l18t|fnvdmAT=%e43V=r85p{jurWkI)qvD3 ZX=LYQm@LK_3$#~avXwX=o0%|38~_zIKz;xK diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_mor.md similarity index 74% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_mor.md index 90b168a74156c..3dd4b36f4d162 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_mor.md @@ -1,3 +1,20 @@ + + # Hudi Test Resources ## Generating Hudi Resources diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_stock_ticks_cow.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_stock_ticks_cow.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_stock_ticks_cow.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_stock_ticks_cow.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_stock_ticks_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_stock_ticks_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_stock_ticks_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_stock_ticks_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md similarity index 77% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md index 9e426b9fd3783..f7d9d9655feb9 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md similarity index 78% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md index f547b378067f3..1a26c6021884c 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_timestamp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md b/hudi-trino/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md similarity index 59% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md index a263f502e4155..222e3932177b3 100644 --- a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md +++ b/hudi-trino/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.md @@ -1,3 +1,20 @@ + + ## Create script Structure of table: diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.zip b/hudi-trino/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.zip rename to hudi-trino/src/test/resources/hudi-testing-data/hudi_trips_cow_v8.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/stock_ticks_cow.zip b/hudi-trino/src/test/resources/hudi-testing-data/stock_ticks_cow.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/stock_ticks_cow.zip rename to hudi-trino/src/test/resources/hudi-testing-data/stock_ticks_cow.zip diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/stock_ticks_mor.zip b/hudi-trino/src/test/resources/hudi-testing-data/stock_ticks_mor.zip similarity index 100% rename from hudi-trino-plugin/src/test/resources/hudi-testing-data/stock_ticks_mor.zip rename to hudi-trino/src/test/resources/hudi-testing-data/stock_ticks_mor.zip diff --git a/hudi-trino-plugin/src/test/resources/long_timestamp.parquet b/hudi-trino/src/test/resources/long_timestamp.parquet similarity index 100% rename from hudi-trino-plugin/src/test/resources/long_timestamp.parquet rename to hudi-trino/src/test/resources/long_timestamp.parquet diff --git a/packaging/hudi-trino-bundle/pom.xml b/packaging/hudi-trino-bundle/pom.xml deleted file mode 100644 index 083805d304903..0000000000000 --- a/packaging/hudi-trino-bundle/pom.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - - hudi - org.apache.hudi - 1.2.0 - ../../pom.xml - - 4.0.0 - hudi-trino-bundle - jar - - - true - ${project.parent.basedir} - true - - - - - - org.apache.rat - apache-rat-plugin - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - ${shadeSources} - ${project.build.directory}/dependency-reduced-pom.xml - - - - - - true - - - META-INF/LICENSE - target/classes/META-INF/LICENSE - - - - - - org.apache.hudi:hudi-hadoop-common - org.apache.hudi:hudi-common - org.apache.hudi:hudi-client-common - org.apache.hudi:hudi-java-client - org.apache.hudi:hudi-hadoop-mr - - - com.esotericsoftware:kryo-shaded - com.esotericsoftware:minlog - org.objenesis:objenesis - - org.apache.parquet:parquet-avro - org.apache.avro:avro - com.github.ben-manes.caffeine:caffeine - org.codehaus.jackson:* - com.yammer.metrics:metrics-core - commons-io:commons-io - com.google.protobuf:protobuf-java - org.openjdk.jol:jol-core - - - - - - com.esotericsoftware.kryo. - org.apache.hudi.com.esotericsoftware.kryo. - - - com.esotericsoftware.reflectasm. - org.apache.hudi.com.esotericsoftware.reflectasm. - - - com.esotericsoftware.minlog. - org.apache.hudi.com.esotericsoftware.minlog. - - - org.objenesis. - org.apache.hudi.org.objenesis. - - - - org.apache.parquet.avro. - org.apache.hudi.org.apache.parquet.avro. - - - org.apache.avro. - org.apache.hudi.org.apache.avro. - - - org.apache.commons.io. - org.apache.hudi.org.apache.commons.io. - - - com.yammer.metrics. - org.apache.hudi.com.yammer.metrics. - - - com.google.common. - ${trino.bundle.bootstrap.shade.prefix}com.google.common. - - - org.apache.commons.lang. - ${trino.bundle.bootstrap.shade.prefix}org.apache.commons.lang. - - - com.google.protobuf. - ${trino.bundle.bootstrap.shade.prefix}com.google.protobuf. - - - org.openjdk.jol. - org.apache.hudi.org.openjdk.jol. - - - false - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/services/javax.* - **/*.proto - - - - ${project.artifactId}-${project.version} - - - - - - - - src/main/resources - - - - - - - - org.apache.hudi - hudi-hadoop-mr-bundle - ${project.version} - - - org.apache.hudi - hudi-client-common - ${project.version} - - - guava - com.google.guava - - - - - org.apache.hudi - hudi-java-client - ${project.version} - - - - - com.esotericsoftware - kryo-shaded - ${kryo.shaded.version} - compile - - - - - org.apache.parquet - parquet-avro - ${trino.parquet.version} - compile - - - - - org.apache.avro - avro - ${avro.version} - compile - - - - - com.google.protobuf - protobuf-java - ${proto.version} - ${trino.bundle.bootstrap.scope} - - - - - - trino-shade-unbundle-bootstrap - - provided - - - - - diff --git a/packaging/hudi-trino-bundle/src/main/java/org/apache/hudi/trino/bundle/Main.java b/packaging/hudi-trino-bundle/src/main/java/org/apache/hudi/trino/bundle/Main.java deleted file mode 100644 index eec1ecf88a8d8..0000000000000 --- a/packaging/hudi-trino-bundle/src/main/java/org/apache/hudi/trino/bundle/Main.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.trino.bundle; - -import org.apache.hudi.common.util.ReflectionUtils; - -/** - * A simple main class to dump all classes loaded in current classpath - *

    - * This is a workaround for generating sources and javadoc jars for packaging modules. The maven plugins for generating - * javadoc and sources plugins do not generate corresponding jars if there are no source files. - *

    - * This class does not have anything to do with Hudi but is there to keep mvn javadocs/source plugin happy. - */ -public class Main { - - public static void main(String[] args) { - ReflectionUtils.getTopLevelClassesInClasspath(Main.class).forEach(System.out::println); - } -} diff --git a/pom.xml b/pom.xml index 2b354fa4437bd..184d3ffadc4d6 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,6 @@ packaging/hudi-utilities-bundle packaging/hudi-utilities-slim-bundle packaging/hudi-timeline-server-bundle - packaging/hudi-trino-bundle hudi-examples hudi-flink-datasource hudi-kafka-connect @@ -686,7 +685,6 @@ **/*.iml .mvn/** - hudi-trino-plugin/** @@ -2311,6 +2309,13 @@ packaging/hudi-metaserver-server-bundle + + + hudi-trino + + hudi-trino + + integration-tests diff --git a/scripts/release/validate_source_copyright.sh b/scripts/release/validate_source_copyright.sh index 1d1c6bf4506bb..67f32e2d9a7b8 100755 --- a/scripts/release/validate_source_copyright.sh +++ b/scripts/release/validate_source_copyright.sh @@ -47,10 +47,10 @@ echo -e "\t\tNotice file exists ? [OK]\n" ### Licensing Check echo "Performing custom Licensing Check " # --- -# Exclude the 'hudi-trino-plugin' directory. Its license checks are handled by airlift: +# Exclude the 'hudi-trino' directory. Its license checks are handled by airlift: # https://github.com/airlift/airbase/blob/823101482dbc60600d7862f0f5c93aded6190996/airbase/pom.xml#L1239 # --- -numfilesWithNoLicense=$(find . -path './hudi-trino-plugin' -prune -o -type f -iname '*' | grep -v './hudi-trino-plugin' | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.zip' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'banner.txt' | grep -v '.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)") +numfilesWithNoLicense=$(find . -path './hudi-trino' -prune -o -type f -iname '*' | grep -v './hudi-trino' | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.zip' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'banner.txt' | grep -v '.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)") # Check if the variable holding the list of files is non-empty if [ -n "$numfilesWithNoLicense" ]; then # If the list isn't empty, count the files and report the error diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh index 36e25ca6aea55..1e1ee7c4565c8 100755 --- a/scripts/release/validate_staged_bundles.sh +++ b/scripts/release/validate_staged_bundles.sh @@ -88,7 +88,7 @@ declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle" "hudi-cli-bundle_2.12" "hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle" "hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle" "hudi-presto-bundle" "hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12" -"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" "hudi-trino-bundle" +"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" "hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13" "hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13") diff --git a/style/checkstyle-suppressions.xml b/style/checkstyle-suppressions.xml index c31a9ff97dce8..39ccbf1aa460c 100644 --- a/style/checkstyle-suppressions.xml +++ b/style/checkstyle-suppressions.xml @@ -33,7 +33,7 @@ - - + + From db4710e3818bb7fd59478555620c643fa827a486 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Mon, 27 Jul 2026 01:35:41 -0700 Subject: [PATCH 208/255] feat(hive-sync): batch and parallelize HiveQL partition operations (#18984) * feat(hive-sync): batch and parallelize HiveQL partition operations Adds an opt-in pool that splits HiveQL partition DDL (add/update/touch/drop) into batches of `hoodie.datasource.hive_sync.batch_num` and dispatches them in parallel across a pool of single-thread workers. Each worker owns its own Hive `Driver` + `SessionState` (both thread-bound in Hive 2.x), so the fan-out is implemented as a fixed pool of dedicated single-thread executors rather than a shared thread pool. Table-level operations (create/alter table, last commit time, writer version) continue to use the single session `Driver`. Partition-phase SQL lists run through the pool only when `hoodie.datasource.hive_sync.batching.enabled` is set to true; default off, existing behavior unchanged. Hive 2.x's `ALTER PARTITION SET LOCATION` ignores db.table qualifiers and uses the connection's current database, so each worker is primed with the correct USE statement before any partition ALTER is dispatched. * fix(hive-sync): address review feedback on HiveQL partition batching - Gate TOUCH batch-splitting on hoodie.datasource.hive_sync.batching.enabled so the default (off) HiveQL path emits a single ALTER TABLE ... TOUCH statement as before, instead of always splitting into batch_num chunks. - Give each HiveDriverPool worker its own exclusively-owned SessionState instead of sharing one SessionState object across all workers. Concurrent Driver.run() calls against a shared session risked corrupting session- scoped state (current db, scratch dirs, txn/lock manager), and closing the shared session once per worker on teardown closed it multiple times while racing worker Driver.close() calls. Workers now bootstrap one at a time (each SessionState construction no longer races another). - Close the driver pool in HiveQueryDDLExecutor's constructor failure path, so a failed SessionState/Driver bootstrap doesn't leak the pool's worker threads, Drivers, and sessions (the pool is constructed by the caller before this constructor runs, and no one else can close it if we throw). - Fix testHiveQLTouchPartitionsWithBatching to drive touchPartitionsToTable directly; the previous resync-based version never reached the batched TOUCH path because incremental sync short-circuits with no new commit. - Make TestHiveDriverPool's cancel-on-first-error assertion race-free by parking pending tasks on a latch and asserting isCancelled() || !isDone(). * fix(hive-sync): isolate per-worker HiveConf; document ADD parallel dispatch - HiveDriverPool.DefaultDriverFactory now builds a per-worker HiveConf copy (new HiveConf(hiveConf)) instead of sharing one HiveConf instance across all workers, and passes that copy to both SessionState and Driver. Hive's QueryState/Driver mutate per-query keys (e.g. HIVEQUERYID) on the conf during run(), so a shared HiveConf let concurrent Driver.run() calls overwrite each other's query-scoped configuration even though each worker already had its own SessionState object. - Reworded HIVE_SYNC_BATCHING_ENABLED's doc to explicitly include ADD in the parallel dispatch scope. addPartitionsToTable routes through runSQLs, so ADD batches are dispatched across the pool same as TOUCH/ SET_LOCATION when the flag is on; only the batch size (not the fan-out) was already unchanged before this flag existed. * fix(hive-sync): abort dispatch on first error; scope TOUCH batching to parallel paths Two review fixes on the HiveQL partition batching path. 1. awaitAll did not actually cancel pending work on first error. The futures returned by dispatchAll belong to N independent single-thread executors, each draining its own queue. awaitAll blocked on Future.get() in submission order, so a failure on a fast worker went unobserved while the awaiting thread was parked on a slow worker's earlier future -- and the failed worker kept pulling and applying more partition DDL from its own queue. The advertised "cancel pending futures on first error" behavior did not hold. dispatchAll now returns a Dispatch handle carrying a shared abort flag. Each task checks the flag on entry and bails with CancellationException without touching its Driver; the first task to fail sets it. awaitAll blocks on a latch that trips on either all-settled or first-abort, then sweeps cancel(false) before walking the futures. Cancelling from the awaiting thread is inherently late here, so the in-task check is what bounds how much extra DDL a failed sync can apply. mayInterruptIfRunning=false is preserved, so in-flight statements still run to completion rather than leaving a Driver mid-statement. 2. hoodie.datasource.hive_sync.batching.enabled leaked into JDBC mode. QueryBasedDDLExecutor is also the base class for JDBCExecutor, which does not override runSQLs. With the flag on in JDBC mode, TOUCH was split into batch_num statements that then executed serially -- changing statement count and partial-application semantics for no benefit, and contradicting the documented "JDBC is unaffected" contract. The config read in constructPartitionAlterStatements is replaced by a getTouchBatchSize(int) hook. The base implementation returns the full partition count (one statement, the long-standing behavior); only HiveQueryDDLExecutor overrides it, and only when a driver pool is actually present. Keying on pool presence rather than on the config means the split can never take effect on a path that would just execute the batches serially. Tests: - awaitAllStopsLaterWorkerWhenEarlierFutureIsSlow: pins the interleaving the bug needs (slow statement on worker 0, fast failure on worker 1) and asserts the statement queued behind the failure never reaches a Driver. - awaitAllCancelsPendingFuturesOnFirstError: rewritten. The prior version asserted isCancelled() || !isDone() to hedge around the race; the abort flag makes the outcome deterministic, so it now asserts on what actually executed. - New TestQueryBasedDDLExecutorTouchBatching: asserts a serial executor emits a single TOUCH statement with the flag on, that its SQL is byte-identical with the flag on and off, and that a parallel-dispatch executor still splits. Both new pool tests were verified to fail on all surefire attempts against the prior logic and pass against the fix. Full hudi-hive-sync suite: 303/303. * fix(hive-sync): close driver pool if sync client construction fails HiveQueryDDLExecutor's own catch closes the pool, but it only covers throws from inside its try block. QueryBasedDDLExecutor's super(config) runs the PartitionValueExtractor reflection first, so a bad hoodie.datasource.hive_sync.partition_extractor_class throws before that try is entered -- the executor's catch never runs, HoodieHiveSyncClient's catch just rethrows, and the already-constructed pool leaks its worker threads and Drivers. Close the pool in HoodieHiveSyncClient's constructor catch, which covers every window between building the pool and handing ownership to the executor. close() is idempotent, so overlapping with the executor's own cleanup is harmless. (cherry picked from commit 483dd878b4ef1b3df1c6135b5daae3fd0274c892) --- .../hudi/hive/HiveSyncConfigHolder.java | 22 + .../hudi/hive/HoodieHiveSyncClient.java | 38 +- .../hudi/hive/ddl/HiveQueryDDLExecutor.java | 91 +++- .../hudi/hive/ddl/QueryBasedDDLExecutor.java | 70 ++- .../apache/hudi/hive/util/HiveDriverPool.java | 463 ++++++++++++++++++ .../apache/hudi/hive/TestHiveSyncTool.java | 107 ++++ ...estQueryBasedDDLExecutorTouchBatching.java | 192 ++++++++ .../hudi/hive/util/TestHiveDriverPool.java | 316 ++++++++++++ 8 files changed, 1281 insertions(+), 18 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestQueryBasedDDLExecutorTouchBatching.java create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfigHolder.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfigHolder.java index 418676d591627..14afa81ea30f7 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfigHolder.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfigHolder.java @@ -122,6 +122,28 @@ public class HiveSyncConfigHolder { .defaultValue(1000) .markAdvanced() .withDocumentation("The number of partitions one batch when synchronous partitions to hive."); + public static final ConfigProperty HIVE_SYNC_BATCHING_ENABLED = ConfigProperty + .key("hoodie.datasource.hive_sync.batching.enabled") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Only applies to HiveQL sync mode; has no effect in HMS or JDBC mode. When true, " + + "ADD, TOUCH, and SET_LOCATION partition statements are dispatched in parallel across a pool of " + + "Hive Driver workers, with ADD and TOUCH additionally split into batches of " + + "`hoodie.datasource.hive_sync.batch_num` partitions per statement (ADD was already batched " + + "before this flag existed; only its dispatch becomes parallel here). SET_LOCATION remains one " + + "statement per partition, as Hive SQL has no multi-partition form. DROP remains serial. " + + "Table-level statements (create/alter table, last commit time, writer version) continue to run " + + "on the single session Driver. Default off; the default HiveQL path is unchanged unless " + + "explicitly opted in."); + public static final ConfigProperty HIVE_SYNC_BATCHING_THREADS = ConfigProperty + .key("hoodie.datasource.hive_sync.batching.threads") + .defaultValue(4) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Pool size (number of Hive Driver workers) and worker-thread count for parallel " + + "HiveQL partition dispatch when `hoodie.datasource.hive_sync.batching.enabled` is true. " + + "Ignored otherwise."); public static final ConfigProperty HIVE_SYNC_MODE = ConfigProperty .key("hoodie.datasource.hive_sync.mode") .noDefaultValue() diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java index 3de4077fd8dab..44968ca3bc37f 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java @@ -38,6 +38,7 @@ import org.apache.hudi.hive.ddl.HiveSyncMode; import org.apache.hudi.hive.ddl.JDBCBasedMetadataOperator; import org.apache.hudi.hive.ddl.JDBCExecutor; +import org.apache.hudi.hive.util.HiveDriverPool; import org.apache.hudi.hive.util.IMetaStoreClientUtil; import org.apache.hudi.hive.util.PartitionFilterGenerator; import org.apache.hudi.sync.common.HoodieSyncClient; @@ -66,6 +67,8 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getInputFormatClassName; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getOutputFormatClassName; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getSerDeClassName; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_ENABLED; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_THREADS; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_MODE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_USE_SPARK_CATALOG; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USE_JDBC; @@ -86,6 +89,10 @@ public class HoodieHiveSyncClient extends HoodieSyncClient { private final Map initialTableByName = new HashMap<>(); DDLExecutor ddlExecutor; private IMetaStoreClient client; + // Present only when HIVE_SYNC_BATCHING_ENABLED and sync mode is HIVEQL (explicit + // or legacy default). Owned by HiveQueryDDLExecutor; this field is kept for + // reference only — close() is delegated through ddlExecutor.close(). + private Option partitionDriverPool = Option.empty(); /** * JDBC-based metadata operator, lazily initialized on first Thrift @@ -124,7 +131,8 @@ public HoodieHiveSyncClient(HiveSyncConfig config, HoodieTableMetaClient metaCli ddlExecutor = new HMSDDLExecutor(config, this.client); break; case HIVEQL: - ddlExecutor = new HiveQueryDDLExecutor(config, this.client); + this.partitionDriverPool = maybeBuildHiveDriverPool(config); + ddlExecutor = new HiveQueryDDLExecutor(config, this.client, this.partitionDriverPool); break; case JDBC: JDBCExecutor jdbcExecutor = new JDBCExecutor(config); @@ -142,14 +150,32 @@ public HoodieHiveSyncClient(HiveSyncConfig config, HoodieTableMetaClient metaCli jdbcMetadataOperator = new JDBCBasedMetadataOperator( jdbcExecutor.getConnection(), databaseName); } else { - ddlExecutor = new HiveQueryDDLExecutor(config, this.client); + this.partitionDriverPool = maybeBuildHiveDriverPool(config); + ddlExecutor = new HiveQueryDDLExecutor(config, this.client, this.partitionDriverPool); } } } catch (Exception e) { + // The pool owns live daemon threads and Hive Drivers, and is built before the + // executor that would otherwise own its lifecycle. Any throw between those two + // points would leak it -- notably QueryBasedDDLExecutor's super(config), which + // runs the PartitionValueExtractor reflection before HiveQueryDDLExecutor's own + // try block is even entered. Closing here covers every such window; the pool's + // close() is idempotent, so overlapping with the executor's cleanup is harmless. + closePartitionDriverPoolQuietly(); throw new HoodieHiveSyncException("Failed to create HiveMetaStoreClient", e); } } + private void closePartitionDriverPoolQuietly() { + partitionDriverPool.ifPresent(pool -> { + try { + pool.close(); + } catch (Exception e) { + log.warn("Error closing HiveDriverPool during failed sync client construction", e); + } + }); + } + /** * Returns true if Thrift API was detected as incompatible and JDBC * fallback is available. When true, metadata operations should use @@ -201,6 +227,14 @@ private IMetaStoreClient createMetaStoreClient(HiveSyncConfig config) { } } + private Option maybeBuildHiveDriverPool(HiveSyncConfig config) { + if (!config.getBooleanOrDefault(HIVE_SYNC_BATCHING_ENABLED)) { + return Option.empty(); + } + int size = config.getIntOrDefault(HIVE_SYNC_BATCHING_THREADS); + return Option.of(new HiveDriverPool(config, size)); + } + private Table getInitialTable(String table) { return initialTableByName.computeIfAbsent(table, t -> { try { diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java index 25434d29eb3ff..c853313182ed1 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java @@ -19,8 +19,10 @@ package org.apache.hudi.hive.ddl; import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.Option; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveSyncException; +import org.apache.hudi.hive.util.HiveDriverPool; import org.apache.hudi.hive.util.HivePartitionUtil; import lombok.extern.slf4j.Slf4j; @@ -41,6 +43,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; import static org.apache.hudi.sync.common.util.TableUtils.tableId; /** @@ -52,10 +55,20 @@ public class HiveQueryDDLExecutor extends QueryBasedDDLExecutor { private final IMetaStoreClient metaStoreClient; private SessionState sessionState; private Driver hiveDriver; + // When present, partition-phase SQL lists fan out across this pool; table-level SQL + // (createTable, schema evolution, single-statement runSQL callers) always uses the + // session `hiveDriver` above. See HiveDriverPool javadoc. + private final Option driverPool; public HiveQueryDDLExecutor(HiveSyncConfig config, IMetaStoreClient metaStoreClient) { + this(config, metaStoreClient, Option.empty()); + } + + public HiveQueryDDLExecutor(HiveSyncConfig config, IMetaStoreClient metaStoreClient, + Option driverPool) { super(config); this.metaStoreClient = metaStoreClient; + this.driverPool = driverPool; try { this.sessionState = new SessionState(config.getHiveConf(), UserGroupInformation.getCurrentUser().getShortUserName()); @@ -73,6 +86,15 @@ public HiveQueryDDLExecutor(HiveSyncConfig config, IMetaStoreClient metaStoreCli if (this.hiveDriver != null) { this.hiveDriver.close(); } + // driverPool (if present) was already constructed by the caller before this + // ctor ran; since we're about to throw, no one else will call close() on it. + driverPool.ifPresent(pool -> { + try { + pool.close(); + } catch (Exception poolCloseException) { + log.error("Error while closing HiveDriverPool", poolCloseException); + } + }); throw new HoodieHiveSyncException("Failed to create HiveQueryDDL object", e); } } @@ -82,19 +104,74 @@ public void runSQL(String sql) { updateHiveSQLs(Collections.singletonList(sql)); } + /** + * Partition-phase SQL fan-out. When the driver pool is present, any leading + * {@code USE database} statements are run on every worker (Hive 2.x's + * ALTER PARTITION SET LOCATION ignores db.table qualifiers and uses the + * connection's current database, so each worker needs to USE the right db + * before any partition ALTER). The remaining statements are then dispatched + * round-robin across the pool. Falls through to the sequential path on the + * session Driver when no pool is configured. + */ + @Override + protected void runSQLs(List sqls) { + if (sqls.isEmpty()) { + return; + } + if (!driverPool.isPresent()) { + updateHiveSQLs(sqls); + return; + } + HiveDriverPool pool = driverPool.get(); + int useStatementCount = 0; + while (useStatementCount < sqls.size() && isUseStatement(sqls.get(useStatementCount))) { + useStatementCount++; + } + if (useStatementCount > 0) { + List setupStatements = sqls.subList(0, useStatementCount); + pool.runOnEachWorker(setupStatements); + } + List partitionStatements = sqls.subList(useStatementCount, sqls.size()); + if (partitionStatements.isEmpty()) { + return; + } + pool.awaitAll(pool.dispatchAll(partitionStatements)); + } + + /** + * Splits TOUCH into batches of {@code HIVE_BATCH_SYNC_PARTITION_NUM} only when a + * driver pool is actually present — i.e. only when {@link #runSQLs(List)} will + * dispatch those batches in parallel. Keyed on pool presence rather than on the + * {@code batching.enabled} config so the split can never take effect on a path + * that would just execute the batches serially (the base class, and therefore + * {@code JDBCExecutor}, always emits one statement). + */ + @Override + protected int getTouchBatchSize(int partitionCount) { + return driverPool.isPresent() + ? config.getIntOrDefault(HIVE_BATCH_SYNC_PARTITION_NUM) : partitionCount; + } + + // Strict 4-char prefix match on "USE ". Internal callers (constructPartitionAlterStatements) + // always emit the USE statement without leading whitespace; do not call with externally + // supplied SQL that might be padded. + private static boolean isUseStatement(String sql) { + return sql != null && sql.regionMatches(true, 0, "USE ", 0, 4); + } + private List updateHiveSQLs(List sqls) { List responses = new ArrayList<>(); + HoodieTimer timer = HoodieTimer.start(); try { for (String sql : sqls) { if (hiveDriver != null) { - HoodieTimer timer = HoodieTimer.start(); responses.add(hiveDriver.run(sql)); - log.info("Time taken to execute [{}]: {} ms", sql, timer.endTimer()); } } } catch (Exception e) { throw new HoodieHiveSyncException("Failed in executing SQL", e); } + log.info("Executed {} SQL statements sequentially in {} ms", sqls.size(), timer.endTimer()); return responses; } @@ -149,6 +226,16 @@ public void dropPartitionsToTable(String tableName, List partitionsToDro @Override public void close() { + // Close the pool first so the worker threads stop dispatching against their + // Drivers before we tear down anything else. The pool's close() runs + // Driver/SessionState cleanup on each worker's own thread. + driverPool.ifPresent(pool -> { + try { + pool.close(); + } catch (Exception e) { + log.warn("Error closing HiveDriverPool", e); + } + }); if (metaStoreClient != null) { Hive.closeCurrent(); } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java index 78b8e684e49a4..3bcbbe1841dfa 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.PartitionPathEncodeUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; @@ -75,6 +76,35 @@ public QueryBasedDDLExecutor(HiveSyncConfig config) { */ public abstract void runSQL(String sql); + /** + * Runs a list of SQL statements. The default implementation executes them + * sequentially via {@link #runSQL(String)}. Subclasses that can parallelize + * (e.g. {@link HiveQueryDDLExecutor} with a driver pool) override this hook + * to fan the list out across workers. The contract requires that the list + * has no positional dependencies — callers must fully qualify table names + * with {@code `db`.`tbl`} so any statement can run on any worker. + */ + protected void runSQLs(List sqls) { + for (String sql : sqls) { + runSQL(sql); + } + } + + /** + * Number of partitions to pack into a single {@code ALTER TABLE ... TOUCH} statement. + * + *

    The base implementation returns {@code partitionCount}, i.e. one statement + * covering every partition — the long-standing behavior, and the only correct choice + * when {@link #runSQLs(List)} executes the list serially. Splitting a TOUCH into + * several statements changes failure semantics (a mid-list failure leaves some + * partitions touched and some not), so it is only worth doing when the resulting + * statements are actually dispatched in parallel. Subclasses that parallelize + * override this; see {@link HiveQueryDDLExecutor}. + */ + protected int getTouchBatchSize(int partitionCount) { + return partitionCount; + } + @Override public void createDatabase(String databaseName) { runSQL("create database if not exists " + databaseName); @@ -120,7 +150,7 @@ public void addPartitionsToTable(String tableName, List partitionsToAdd) } log.info("Adding partitions {} to table {}", partitionsToAdd.size(), tableName); List sqls = constructAddPartitions(tableName, partitionsToAdd); - sqls.stream().forEach(sql -> runSQL(sql)); + runSQLs(sqls); } @Override @@ -131,9 +161,7 @@ public void updatePartitionsToTable(String tableName, List changedPartit } log.info("Changing partitions {} on {}", changedPartitions.size(), tableName); List sqls = constructPartitionAlterStatements(tableName, changedPartitions, PartitionAlterType.SET_LOCATION); - for (String sql : sqls) { - runSQL(sql); - } + runSQLs(sqls); } @Override @@ -216,29 +244,43 @@ public void touchPartitionsToTable(String tableName, List touchPartition } log.info("Touching partitions {} on {}", touchPartitions.size(), tableName); List sqls = constructPartitionAlterStatements(tableName, touchPartitions, PartitionAlterType.TOUCH); - for (String sql : sqls) { - runSQL(sql); - } + runSQLs(sqls); } /** * Builds SQL statements to either touch partitions or set their location. - * TOUCH: one ALTER TABLE ... TOUCH PARTITION (p1) PARTITION (p2) ... - * SET_LOCATION: one ALTER TABLE ... PARTITION (p) SET LOCATION '...' per partition. + * + *

    The first element of the returned list is always a {@code USE database} + * statement. Hive 2.x's ALTER PARTITION ... SET LOCATION does not respect the + * {@code db.table} qualifier (silently routes to the connection's current + * database), so the {@code USE} is load-bearing. Parallel execution paths must + * run this statement on every worker before fanning out the rest. + * + *

    TOUCH: one {@code ALTER TABLE ... TOUCH PARTITION (p1) ...} statement per + * batch of {@link #getTouchBatchSize(int)} partitions. The base implementation + * returns the full partition count, i.e. a single statement covering everything, + * matching pre-batching behavior. Only subclasses that actually dispatch the + * resulting statements in parallel override it to split. + * + *

    SET_LOCATION: one {@code ALTER TABLE ... PARTITION (p) SET LOCATION '...'} + * per partition (Hive SQL does not support multi-partition SET LOCATION in one + * statement). */ private List constructPartitionAlterStatements(String tableName, List partitions, PartitionAlterType alterType) { List result = new ArrayList<>(); - // Hive 2.x doesn't like db.table name for operations, hence we need to change to using the database first String useDatabase = "USE " + HIVE_ESCAPE_CHARACTER + databaseName + HIVE_ESCAPE_CHARACTER; result.add(useDatabase); String alterTablePrefix = "ALTER TABLE " + HIVE_ESCAPE_CHARACTER + tableName + HIVE_ESCAPE_CHARACTER; + int batchSyncPartitionNum = getTouchBatchSize(partitions.size()); switch (alterType) { case TOUCH: - String alterTable = alterTablePrefix + " TOUCH"; - for (String partition : partitions) { - alterTable += " PARTITION (" + getPartitionClause(partition) + ")"; + for (List batch : CollectionUtils.batches(partitions, batchSyncPartitionNum)) { + StringBuilder alterTable = new StringBuilder(alterTablePrefix).append(" TOUCH"); + for (String partition : batch) { + alterTable.append(" PARTITION (").append(getPartitionClause(partition)).append(")"); + } + result.add(alterTable.toString()); } - result.add(alterTable); break; case SET_LOCATION: for (String partition : partitions) { diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java new file mode 100644 index 0000000000000..5950bf442313d --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive.util; + +import org.apache.hudi.common.util.VisibleForTesting; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveSyncException; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; + +/** + * Pool of Hive {@link Driver} + {@link SessionState} pairs for parallel HiveQL DDL. + * + *

    Hive's {@code SessionState.start(state)} binds state to the calling thread's + * thread-local, and {@code Driver} reads from that thread-local during {@code run()}. + * A Driver constructed on one thread cannot be safely used from another. This pool + * solves that by giving each slot its own dedicated worker thread (a single-thread + * executor) — the Driver and SessionState are built on that thread by a bootstrap + * task, and all subsequent SQL for that slot runs on the same thread. + * + *

    Usage contract: use this pool only for partition-row DDL statements that + * are independent of each other and freely shuffleable across workers. Table-level + * statements (createTable, schema evolution, USE database) must continue to run on + * the session {@code Driver} held by {@code HiveQueryDDLExecutor} on the sync driver + * thread. The pool is gated behind {@code hoodie.datasource.hive_sync.batching.enabled} + * and is constructed only for HiveQL sync mode. + */ +public class HiveDriverPool implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(HiveDriverPool.class); + + // Per-worker Driver construction has to be fast in practice (a few hundred ms + // for the SessionState + Driver init). A 60s ceiling per worker leaves plenty of + // headroom for a slow JVM warm-up but bounds the failure mode if the metastore + // is unreachable or Hive hangs during init. + private static final long BOOTSTRAP_TIMEOUT_SECONDS = 60; + + private final List workers; + private final int size; + private volatile boolean closed; + + public HiveDriverPool(HiveSyncConfig config, int size) { + this(config, size, new DefaultDriverFactory(config)); + } + + // Package-private for tests: accepts a DriverFactory so unit tests can inject + // mock Driver instances without standing up a real Hive instance. + HiveDriverPool(HiveSyncConfig config, int size, DriverFactory factory) { + if (size < 1) { + throw new IllegalArgumentException("Pool size must be >= 1, got " + size); + } + this.size = size; + this.workers = new ArrayList<>(size); + String databaseName = config.getStringOrDefault(META_SYNC_DATABASE_NAME); + PoolThreadFactory threadFactory = new PoolThreadFactory(); + try { + // Bootstrap workers one at a time (not concurrently): each worker builds its + // own exclusively-owned SessionState, and constructing several SessionStates + // in parallel risks racing on shared scratch-dir creation. This only affects + // one-time pool startup cost, not per-statement dispatch latency. + for (int i = 0; i < size; i++) { + Worker worker = new Worker(threadFactory); + workers.add(worker); + worker.executor.submit(() -> { + worker.driver = factory.newDriver(databaseName); + worker.sessionState = SessionState.get(); + return null; + }).get(BOOTSTRAP_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } catch (Exception e) { + tearDown(); + throw new HoodieException("Failed to construct HiveDriverPool of size " + size, e); + } + LOG.info("Initialized HiveDriverPool with {} workers", size); + } + + /** + * Runs each given SQL on every worker, in order. Used for setup statements + * (e.g. {@code USE database}) that must establish per-thread session context + * before any partition statement runs. Blocks until all workers have completed + * the setup. Throws on first error. + */ + public void runOnEachWorker(List setupSqls) { + if (closed) { + throw new IllegalStateException("Cannot dispatch to a closed HiveDriverPool"); + } + if (setupSqls.isEmpty()) { + return; + } + Dispatch dispatch = new Dispatch(workers.size()); + for (Worker worker : workers) { + dispatch.add(worker.executor.submit(() -> { + if (dispatch.aborted()) { + throw new CancellationException("Skipped after an earlier setup statement failed"); + } + try { + for (String sql : setupSqls) { + worker.driver.run(sql); + } + } catch (Throwable t) { + dispatch.abort(); + throw t; + } finally { + dispatch.taskSettled(); + } + return null; + })); + } + dispatch.sealed(); + awaitAll(dispatch); + } + + /** + * Dispatches each SQL string to a worker (round-robin) and returns a handle to the + * in-flight batch — this method does not block. The caller is responsible for + * awaiting completion via {@link #awaitAll(Dispatch)} and collecting errors. SQL text + * is intentionally not logged per-statement here: batched TOUCH/ADD statements can + * be many kilobytes, and N parallel workers would multiply the log volume. See + * {@link #awaitAll(Dispatch)} for the per-call summary log. + * + *

    Statements are spread round-robin across workers, so worker w owns + * indices {@code w, w + N, w + 2N, ...}. Each worker drains its own queue + * independently, which is why abort has to be observed by the tasks themselves + * rather than by the awaiting thread — see {@link Dispatch}. + */ + public Dispatch dispatchAll(List sqls) { + if (closed) { + throw new IllegalStateException("Cannot dispatch to a closed HiveDriverPool"); + } + Dispatch dispatch = new Dispatch(sqls.size()); + for (int i = 0; i < sqls.size(); i++) { + String sql = sqls.get(i); + Worker worker = workers.get(i % workers.size()); + dispatch.add(worker.executor.submit(() -> { + // Abort check inside the task: a worker can dequeue this statement while the + // awaiting thread is still parked on some other worker's slower statement, so + // Future.cancel() alone cannot stop it in time. Checking here means no + // statement starts after a sibling has already failed. + if (dispatch.aborted()) { + throw new CancellationException("Skipped after an earlier statement failed"); + } + try { + worker.driver.run(sql); + } catch (Throwable t) { + dispatch.abort(); + throw t; + } finally { + dispatch.taskSettled(); + } + return null; + })); + } + dispatch.sealed(); + return dispatch; + } + + /** + * Awaits the dispatched batch and throws the first error encountered. Errors are + * observed in completion order, not submission order: the awaiting thread + * blocks until every task has settled (or the batch has aborted), so a failure on a + * fast worker cancels the queues of all other workers even while a slow worker is + * still mid-statement. Errors that finished before cancellation are logged at WARN. + * Callers do not need per-statement results (Hive's Driver.run side-effects the + * metastore), so this method is void. + */ + public void awaitAll(Dispatch dispatch) { + long start = System.currentTimeMillis(); + // Block until either every task settled or one of them aborted the batch. Only + // then walk the futures — by that point no un-started task can still begin, so + // the walk order no longer affects how much extra DDL gets applied. + dispatch.awaitSettledOrAborted(); + int cancelled = dispatch.cancelPending(); + + Exception firstError = null; + int completed = 0; + for (Future f : dispatch.futures()) { + try { + f.get(); + completed++; + } catch (CancellationException ce) { + // Either we cancelled it before it started, or the task itself observed the + // abort flag and bailed. Not a new failure; just note it for the summary. + cancelled++; + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + if (firstError == null) { + firstError = ie; + } + } catch (ExecutionException ee) { + Exception cause = unwrap(ee); + if (cause instanceof CancellationException) { + cancelled++; + } else if (firstError == null) { + firstError = cause; + } else { + LOG.warn("Additional SQL batch failed (suppressed in favor of first error)", cause); + } + } + } + if (firstError != null) { + throw new HoodieHiveSyncException("Failed in executing SQL", firstError); + } + LOG.info("Completed {} SQL statements ({} cancelled) in {} ms across {} workers", + completed, cancelled, System.currentTimeMillis() - start, size); + } + + /** + * Handle to one {@link #dispatchAll(List)} batch: the submitted futures plus the + * shared abort flag the tasks consult before running. + * + *

    The flag exists because the futures belong to N independent single-thread + * executors. Cancelling from the awaiting thread is inherently late — a worker can + * pull its next statement off its own queue at any moment — so each task also + * re-checks {@link #aborted()} on entry. That is what actually bounds how much extra + * partition DDL a failed sync can apply. + */ + public static final class Dispatch { + private final List> futures; + private final int total; + private final AtomicInteger settled = new AtomicInteger(0); + private final AtomicBoolean aborted = new AtomicBoolean(false); + private final CountDownLatch done = new CountDownLatch(1); + private volatile boolean sealed; + + private Dispatch(int total) { + this.total = total; + this.futures = new ArrayList<>(total); + } + + private void add(Future future) { + futures.add(future); + } + + // Called once submission finishes. A task that settles before the last submit + // would otherwise see settled < total and never trip the latch, so re-check here. + private void sealed() { + sealed = true; + signalIfComplete(); + } + + private boolean aborted() { + return aborted.get(); + } + + private void abort() { + aborted.set(true); + done.countDown(); + } + + private void taskSettled() { + settled.incrementAndGet(); + signalIfComplete(); + } + + private void signalIfComplete() { + if (sealed && settled.get() >= total) { + done.countDown(); + } + } + + private void awaitSettledOrAborted() { + if (total == 0) { + return; + } + try { + done.await(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + aborted.set(true); + } + } + + // mayInterruptIfRunning=false: the worker thread is bound to a Hive Driver whose + // state we don't want to corrupt mid-statement. Cancel only tasks that haven't + // started; in-flight statements run to completion. + private int cancelPending() { + int cancelled = 0; + for (Future f : futures) { + if (f.cancel(false)) { + cancelled++; + } + } + return cancelled; + } + + private List> futures() { + return futures; + } + + @VisibleForTesting + public int size() { + return futures.size(); + } + + @VisibleForTesting + public Future futureAt(int index) { + return futures.get(index); + } + } + + private static Exception unwrap(ExecutionException ee) { + Throwable cause = ee.getCause(); + return (cause instanceof Exception) ? (Exception) cause : ee; + } + + public int size() { + return size; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + tearDown(); + } + + private void tearDown() { + // Close each worker's own Driver and SessionState on its own thread, then shut + // the executor down. Each worker owns an exclusive SessionState (see + // DefaultDriverFactory), so there is no cross-worker close ordering to worry + // about here — closing worker i never affects worker j. + for (Worker worker : workers) { + try { + worker.executor.submit(() -> { + if (worker.driver != null) { + try { + worker.driver.close(); + } catch (Exception e) { + LOG.warn("Error closing pooled Driver", e); + } + } + if (worker.sessionState != null) { + try { + worker.sessionState.close(); + } catch (Exception e) { + LOG.warn("Error closing pooled SessionState", e); + } + } + return null; + }).get(30, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.warn("Error during pool worker shutdown", e); + } + worker.executor.shutdown(); + try { + if (!worker.executor.awaitTermination(10, TimeUnit.SECONDS)) { + worker.executor.shutdownNow(); + } + } catch (InterruptedException ie) { + worker.executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + workers.clear(); + } + + /** + * Per-slot state: a single-thread executor and the Driver + SessionState bound to + * its thread. Both are volatile because they are written by the bootstrap task and + * read by subsequent dispatch/teardown tasks on the same executor. + */ + private static final class Worker { + final ExecutorService executor; + volatile Driver driver; + volatile SessionState sessionState; + + Worker(ThreadFactory threadFactory) { + this.executor = Executors.newSingleThreadExecutor(threadFactory); + } + } + + @FunctionalInterface + interface DriverFactory { + Driver newDriver(String databaseName) throws Exception; + } + + /** + * Builds a real Hive {@link Driver} on the calling thread, backed by a + * {@link SessionState} that is exclusively owned by that thread (not shared with + * any other worker). Hive's session-scoped state (current database, scratch + * directories, and the transaction/lock manager under {@code DbTxnManager}) is + * mutated by {@code Driver.run()} and is not safe for concurrent use from multiple + * threads, so each worker must have its own instance. Bootstrap of all workers is + * done sequentially by the pool constructor specifically so these constructions + * don't race each other (e.g. on scratch-dir creation). + * + *

    Each worker also gets its own {@link HiveConf} copy. {@code SessionState} + * retains whatever {@code HiveConf} it's given, and {@code QueryState}/{@code Driver} + * mutate per-query keys on that conf during {@code run()} (e.g. {@code HIVEQUERYID}). + * Sharing one {@code HiveConf} instance across workers would let concurrent + * {@code Driver.run()} calls overwrite each other's query-scoped configuration even + * though each worker has its own {@code SessionState} object. + */ + private static final class DefaultDriverFactory implements DriverFactory { + private final HiveConf hiveConf; + + DefaultDriverFactory(HiveSyncConfig config) { + this.hiveConf = config.getHiveConf(); + } + + @Override + public Driver newDriver(String databaseName) throws Exception { + HiveConf workerConf = new HiveConf(hiveConf); + SessionState sessionState = new SessionState(workerConf, + UserGroupInformation.getCurrentUser().getShortUserName()); + sessionState.setCurrentDatabase(databaseName); + SessionState.start(sessionState); + return new Driver(workerConf); + } + } + + private static final class PoolThreadFactory implements ThreadFactory { + private static final AtomicInteger POOL_ID = new AtomicInteger(0); + private final AtomicInteger threadId = new AtomicInteger(0); + private final String namePrefix = "hudi-hive-driver-pool-" + POOL_ID.incrementAndGet() + "-"; + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r, namePrefix + threadId.incrementAndGet()); + t.setDaemon(true); + return t; + } + } +} diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index 9a7dceef0e4ec..fc4b47a4b10d7 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -96,9 +96,12 @@ import static org.apache.hudi.hive.HiveSyncConfig.HIVE_SYNC_FILTER_PUSHDOWN_ENABLED; import static org.apache.hudi.hive.HiveSyncConfig.RECREATE_HIVE_TABLE_ON_ERROR; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_ENABLED; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_THREADS; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_COMMENT; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_MODE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_TABLE_STRATEGY; @@ -329,6 +332,110 @@ public void testDropUpperCasePartitionWithHMS() throws Exception { "Table partitions should match the number of partitions we wrote"); } + /** + * Exercises HiveQL sync with parallel partition batching enabled. Routes through + * the HiveDriverPool — each worker thread owns a Driver+SessionState pair, and + * the SQL list (qualified with `db`.`tbl`) is fanned out across them. + */ + @Test + public void testHiveQLSyncWithBatchingEnabled() throws Exception { + hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), HiveSyncMode.HIVEQL.name()); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_ENABLED.key(), "true"); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_THREADS.key(), "3"); + hiveSyncProps.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key(), "3"); + + int partitionCount = 10; + HiveTestUtil.createCOWTable("100", partitionCount, true); + + reInitHiveSyncClient(); + assertFalse(hiveClient.tableExists(HiveTestUtil.TABLE_NAME), + "Table should not exist before initial sync"); + reSyncHiveTable(); + assertEquals(partitionCount, hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).size(), + "All partitions should be added under parallel HiveQL batching"); + + // Add more partitions, then sync again to exercise the parallel update path. + HiveTestUtil.addCOWPartition("2050/01/01", true, true, "101"); + HiveTestUtil.addCOWPartition("2050/01/02", true, true, "102"); + HiveTestUtil.addCOWPartition("2050/01/03", true, true, "103"); + HiveTestUtil.addCOWPartition("2050/01/04", true, true, "104"); + reInitHiveSyncClient(); + reSyncHiveTable(); + assertEquals(partitionCount + 4, hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).size(), + "Incremental add via parallel HiveQL batching should sync the new partitions"); + } + + /** + * Exercises the SET_LOCATION path in HiveQL mode with batching on. SET_LOCATION + * emits one ALTER PARTITION ... SET LOCATION statement per partition (Hive SQL + * has no multi-partition SET LOCATION), so this is the fan-out path most likely + * to exercise concurrent ALTER PARTITION calls against the same table. + */ + @Test + public void testHiveQLSetLocationWithBatching() throws Exception { + hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), HiveSyncMode.HIVEQL.name()); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_ENABLED.key(), "true"); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_THREADS.key(), "3"); + hiveSyncProps.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key(), "2"); + + int partitionCount = 6; + HiveTestUtil.createCOWTable("100", partitionCount, true); + reInitHiveSyncClient(); + reSyncHiveTable(); + assertEquals(partitionCount, hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).size()); + + // Drive the SET_LOCATION path by directly calling updatePartitionsToTable with + // existing partition paths. Each partition produces its own ALTER ... SET LOCATION + // statement, fanned out across the 3 workers in the pool. + List existingPartitions = hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).stream() + .map(p -> getRelativePartitionPath(new Path(basePath), new Path(p.getStorageLocation()))) + .collect(Collectors.toList()); + hiveClient.updatePartitionsToTable(HiveTestUtil.TABLE_NAME, existingPartitions); + + List after = hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME); + assertEquals(partitionCount, after.size(), + "Parallel SET_LOCATION must not change the partition set"); + Set relativePaths = after.stream() + .map(p -> getRelativePartitionPath(new Path(basePath), new Path(p.getStorageLocation()))) + .collect(Collectors.toSet()); + assertEquals(partitionCount, relativePaths.size(), + "Each partition should resolve to a unique relative path after parallel SET_LOCATION"); + assertTrue(relativePaths.containsAll(existingPartitions), + "All original partition paths should still be present after parallel SET_LOCATION"); + } + + /** + * Exercises the TOUCH path in HiveQL mode with batching on. Verifies that + * splitting one giant ALTER TABLE TOUCH PARTITION(...)... into multiple smaller + * statements does not break partition visibility downstream. + */ + @Test + public void testHiveQLTouchPartitionsWithBatching() throws Exception { + hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), HiveSyncMode.HIVEQL.name()); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_ENABLED.key(), "true"); + hiveSyncProps.setProperty(HIVE_SYNC_BATCHING_THREADS.key(), "2"); + hiveSyncProps.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key(), "2"); + hiveSyncProps.setProperty(META_SYNC_TOUCH_PARTITIONS_ENABLED.key(), "true"); + + int partitionCount = 6; + HiveTestUtil.createCOWTable("100", partitionCount, true); + reInitHiveSyncClient(); + reSyncHiveTable(); + assertEquals(partitionCount, hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).size()); + + // Drive the TOUCH path directly by calling touchPartitionsToTable with existing + // partition paths. Partitions are batched into groups of HIVE_BATCH_SYNC_PARTITION_NUM + // and each batch's ALTER ... TOUCH statement is fanned out across the 2 workers in + // the pool. + List existingPartitions = hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).stream() + .map(p -> getRelativePartitionPath(new Path(basePath), new Path(p.getStorageLocation()))) + .collect(Collectors.toList()); + hiveClient.touchPartitionsToTable(HiveTestUtil.TABLE_NAME, existingPartitions); + + assertEquals(partitionCount, hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME).size(), + "TOUCH batching must not change the partition set"); + } + @ParameterizedTest @MethodSource({"syncModeAndSchemaFromCommitMetadata"}) public void testBasicSync(boolean useSchemaFromCommitMetadata, String syncMode, String enablePushDown) throws Exception { diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestQueryBasedDDLExecutorTouchBatching.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestQueryBasedDDLExecutorTouchBatching.java new file mode 100644 index 0000000000000..73d23859dbfa9 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/ddl/TestQueryBasedDDLExecutorTouchBatching.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive.ddl; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.hive.HiveSyncConfig; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_ENABLED; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Verifies which execution paths TOUCH batching applies to. + * + *

    {@code hoodie.datasource.hive_sync.batching.enabled} only makes sense where the + * resulting statements are dispatched in parallel. {@link QueryBasedDDLExecutor} is + * also the base class for {@link JDBCExecutor}, which executes the list serially — so + * splitting there would change statement count and partial-application semantics for + * no benefit. The split is therefore driven by {@link QueryBasedDDLExecutor#getTouchBatchSize(int)}, + * which only {@link HiveQueryDDLExecutor} overrides (and only when a driver pool is + * actually present). + */ +class TestQueryBasedDDLExecutorTouchBatching { + + private static final String TABLE_NAME = "tbl"; + private static final int PARTITION_COUNT = 5; + private static final int BATCH_SIZE = 2; + + /** + * Captures the SQL handed to the executor instead of running it. Uses the base + * class's serial {@code runSQLs}, exactly as {@link JDBCExecutor} does. + * + *

    {@code parallelBatchSize} stands in for a subclass that dispatches in parallel: + * when set, {@link #getTouchBatchSize(int)} returns it, mimicking + * {@link HiveQueryDDLExecutor} with a driver pool present. When unset, the base-class + * default applies — the JDBC-mode shape. + */ + private static final class RecordingExecutor extends QueryBasedDDLExecutor { + private final List executed = new ArrayList<>(); + private final Integer parallelBatchSize; + + RecordingExecutor(HiveSyncConfig config) { + this(config, null); + } + + RecordingExecutor(HiveSyncConfig config, Integer parallelBatchSize) { + super(config); + this.parallelBatchSize = parallelBatchSize; + } + + @Override + protected int getTouchBatchSize(int partitionCount) { + return parallelBatchSize != null ? parallelBatchSize : super.getTouchBatchSize(partitionCount); + } + + @Override + public void runSQL(String sql) { + executed.add(sql); + } + + @Override + public Map getTableSchema(String tableName) { + return Collections.emptyMap(); + } + + @Override + public void dropPartitionsToTable(String tableName, List partitionsToDrop) { + // not exercised here + } + + @Override + public void close() { + // no resources held + } + } + + private static HiveSyncConfig configWithBatching(boolean batchingEnabled) { + TypedProperties props = new TypedProperties(); + props.setProperty(META_SYNC_DATABASE_NAME.key(), "db"); + props.setProperty(META_SYNC_BASE_PATH.key(), "file:///tmp/base"); + props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "dt"); + props.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key(), String.valueOf(BATCH_SIZE)); + props.setProperty(HIVE_SYNC_BATCHING_ENABLED.key(), String.valueOf(batchingEnabled)); + return new HiveSyncConfig(props); + } + + private static List partitions() { + return IntStream.range(0, PARTITION_COUNT) + .mapToObj(i -> "2026-01-0" + (i + 1)) + .collect(Collectors.toList()); + } + + private static List touchStatements(RecordingExecutor executor) { + // constructPartitionAlterStatements always emits a leading `USE db`; the TOUCH + // statements are everything after it. + return executor.executed.stream() + .filter(sql -> sql.contains(" TOUCH ")) + .collect(Collectors.toList()); + } + + /** + * Given a serial executor (the JDBC-mode shape) with batching enabled, when TOUCH is + * issued for more partitions than the batch size, then a single TOUCH statement is + * still emitted — the flag must not reach non-parallel execution paths. + */ + @Test + void serialExecutorEmitsSingleTouchStatementEvenWithBatchingEnabled() { + RecordingExecutor executor = new RecordingExecutor(configWithBatching(true)); + + executor.touchPartitionsToTable(TABLE_NAME, partitions()); + + List touches = touchStatements(executor); + assertEquals(1, touches.size(), + "Serial executors (e.g. JDBC mode) must emit one TOUCH statement regardless of " + + "hoodie.datasource.hive_sync.batching.enabled"); + assertEquals(PARTITION_COUNT, countPartitionClauses(touches.get(0)), + "The single statement must still cover every partition"); + } + + /** + * Given the same executor with batching disabled, when TOUCH is issued, then the SQL + * is byte-identical to the enabled case — pinning that the flag is a no-op here. + */ + @Test + void serialExecutorTouchSqlIsIdenticalWithAndWithoutBatchingFlag() { + RecordingExecutor withFlag = new RecordingExecutor(configWithBatching(true)); + RecordingExecutor withoutFlag = new RecordingExecutor(configWithBatching(false)); + + withFlag.touchPartitionsToTable(TABLE_NAME, partitions()); + withoutFlag.touchPartitionsToTable(TABLE_NAME, partitions()); + + assertEquals(withoutFlag.executed, withFlag.executed, + "Enabling the batching flag must not change JDBC-mode TOUCH SQL shape"); + } + + /** + * Given an executor that reports a parallel-dispatch batch size (the HiveQL-with-pool + * shape), when TOUCH is issued, then partitions are split across multiple statements. + * This pins that the base-class default is the only thing suppressing the split. + */ + @Test + void parallelExecutorSplitsTouchIntoBatches() { + RecordingExecutor executor = new RecordingExecutor(configWithBatching(true), BATCH_SIZE); + + executor.touchPartitionsToTable(TABLE_NAME, partitions()); + + List touches = touchStatements(executor); + // 5 partitions at 2 per batch -> 3 statements (2, 2, 1). + assertEquals(3, touches.size()); + assertEquals(PARTITION_COUNT, + touches.stream().mapToInt(TestQueryBasedDDLExecutorTouchBatching::countPartitionClauses).sum(), + "Batching must not drop or duplicate partitions"); + } + + private static int countPartitionClauses(String sql) { + int count = 0; + int idx = sql.indexOf("PARTITION ("); + while (idx >= 0) { + count++; + idx = sql.indexOf("PARTITION (", idx + 1); + } + return count; + } +} diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java new file mode 100644 index 0000000000000..fd58b7023aa19 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive.util; + +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveSyncException; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.junit.jupiter.api.Test; +import org.mockito.invocation.InvocationOnMock; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Unit tests for {@link HiveDriverPool} that exercise bootstrap, dispatch, error + * propagation, and close semantics without standing up a real Hive instance. + */ +class TestHiveDriverPool { + + private static HiveSyncConfig configWithEmptyHiveConf() { + HiveSyncConfig config = mock(HiveSyncConfig.class); + doAnswer(inv -> new HiveConf()).when(config).getHiveConf(); + doAnswer(inv -> "default").when(config).getStringOrDefault( + org.mockito.ArgumentMatchers.any()); + return config; + } + + @Test + void bootstrapBuildsOneDriverPerSlot() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + AtomicInteger built = new AtomicInteger(); + HiveDriverPool.DriverFactory factory = (db) -> { + built.incrementAndGet(); + return mock(Driver.class); + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 3, factory)) { + assertEquals(3, pool.size()); + assertEquals(3, built.get(), "One Driver per slot should be constructed eagerly"); + } + } + + @Test + void bootstrapFailurePropagatesAndTearsDown() { + HiveSyncConfig config = configWithEmptyHiveConf(); + AtomicInteger calls = new AtomicInteger(); + HiveDriverPool.DriverFactory factory = (db) -> { + int n = calls.incrementAndGet(); + if (n == 2) { + throw new RuntimeException("simulated driver build failure"); + } + return mock(Driver.class); + }; + HoodieException ex = assertThrows(HoodieException.class, + () -> new HiveDriverPool(config, 3, factory)); + assertTrue(ex.getMessage().contains("Failed to construct HiveDriverPool")); + } + + @Test + void runAllDispatchesEachSqlAcrossWorkers() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + // Each worker counts how many SQLs it received and remembers the thread. + ConcurrentHashMap> seenThreadsByDriver = new ConcurrentHashMap<>(); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + seenThreadsByDriver.put(d, ConcurrentHashMap.newKeySet()); + doAnswer((InvocationOnMock inv) -> { + seenThreadsByDriver.get(d).add(Thread.currentThread().getName()); + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 2, factory)) { + List sqls = Arrays.asList("SELECT 1", "SELECT 2", "SELECT 3", "SELECT 4"); + HiveDriverPool.Dispatch futures = pool.dispatchAll(sqls); + pool.awaitAll(futures); + assertEquals(2, seenThreadsByDriver.size(), "Expected exactly 2 worker Drivers"); + int totalCalls = seenThreadsByDriver.values().stream().mapToInt(Set::size).sum(); + assertTrue(totalCalls >= 1, "At least one worker should have logged a thread"); + // Each Driver should have been invoked exactly twice (round-robin with 4 sqls, 2 workers). + for (Driver d : seenThreadsByDriver.keySet()) { + verify(d, times(2)).run(anyString()); + } + } + } + + @Test + void awaitAllThrowsFirstError() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + doAnswer(inv -> { + String sql = inv.getArgument(0); + if (sql.equals("FAIL")) { + throw new RuntimeException("boom: " + sql); + } + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 2, factory)) { + HiveDriverPool.Dispatch futures = pool.dispatchAll(Arrays.asList("OK", "FAIL", "OK")); + HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, + () -> pool.awaitAll(futures)); + assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + } + } + + @Test + void concurrentDispatchBoundedByPoolSize() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + AtomicInteger inFlight = new AtomicInteger(); + AtomicInteger maxInFlight = new AtomicInteger(); + CountDownLatch hold = new CountDownLatch(1); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + doAnswer(inv -> { + int now = inFlight.incrementAndGet(); + maxInFlight.updateAndGet(prev -> Math.max(prev, now)); + hold.await(2, TimeUnit.SECONDS); + inFlight.decrementAndGet(); + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 2, factory)) { + // 5 SQLs against pool of size 2 → max in-flight should be 2. + HiveDriverPool.Dispatch futures = pool.dispatchAll(Arrays.asList("a", "b", "c", "d", "e")); + // Release after a short wait so all SQLs progress. + Thread.sleep(150); + hold.countDown(); + pool.awaitAll(futures); + assertTrue(maxInFlight.get() <= 2, + "Max concurrent dispatches must not exceed pool size, observed " + maxInFlight.get()); + assertTrue(maxInFlight.get() >= 1, "Sanity: at least one dispatch ran"); + } + } + + @Test + void closeIsIdempotentAndPreventsFurtherDispatch() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + HiveDriverPool.DriverFactory factory = (db) -> mock(Driver.class); + HiveDriverPool pool = new HiveDriverPool(config, 2, factory); + pool.close(); + pool.close(); + assertThrows(IllegalStateException.class, + () -> pool.dispatchAll(Arrays.asList("anything"))); + } + + @Test + void invalidSizeRejected() { + HiveSyncConfig config = configWithEmptyHiveConf(); + HiveDriverPool.DriverFactory factory = (db) -> mock(Driver.class); + assertThrows(IllegalArgumentException.class, + () -> new HiveDriverPool(config, 0, factory)); + } + + /** + * runOnEachWorker must execute the setup SQL on every worker (each on its bound + * thread) before {@code dispatchAll()} fans the partition statements out. Without this, + * Hive 2.x's SET LOCATION would silently route to the wrong database on the workers + * that never saw the leading USE statement. + */ + @Test + void runOnEachWorkerRunsSetupOnEveryWorker() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + ConcurrentHashMap> sqlsByDriver = new ConcurrentHashMap<>(); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + sqlsByDriver.put(d, java.util.Collections.synchronizedList(new java.util.ArrayList<>())); + doAnswer((InvocationOnMock inv) -> { + sqlsByDriver.get(d).add(inv.getArgument(0)); + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 3, factory)) { + pool.runOnEachWorker(Arrays.asList("USE `db1`")); + HiveDriverPool.Dispatch futures = pool.dispatchAll(Arrays.asList("ALTER 1", "ALTER 2", "ALTER 3")); + pool.awaitAll(futures); + + assertEquals(3, sqlsByDriver.size(), "Expected one Driver per worker"); + for (Map.Entry> e : sqlsByDriver.entrySet()) { + List seen = e.getValue(); + assertTrue(!seen.isEmpty() && seen.get(0).equals("USE `db1`"), + "Each worker must see USE first; saw " + seen); + } + } + } + + /** + * Given a single-worker pool where the first statement fails, when awaitAll runs, + * then it throws the original cause and neither queued statement is ever executed. + * + *

    Deterministic: statements queued behind the failure observe the batch's abort + * flag on entry and bail out without touching the Driver, so it does not matter + * whether the worker dequeues them before or after awaitAll's cancel() sweep. + */ + @Test + void awaitAllCancelsPendingFuturesOnFirstError() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + List executed = Collections.synchronizedList(new ArrayList<>()); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + doAnswer(inv -> { + String sql = inv.getArgument(0); + executed.add(sql); + if (sql.equals("FAIL")) { + throw new RuntimeException("boom"); + } + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 1, factory)) { + HiveDriverPool.Dispatch dispatch = pool.dispatchAll(Arrays.asList("FAIL", "PENDING_A", "PENDING_B")); + + HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, + () -> pool.awaitAll(dispatch)); + + assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + assertEquals(Collections.singletonList("FAIL"), executed, + "Statements queued behind the failure must never reach the Driver"); + } + } + + /** + * Regression for the in-order-await bug: a slow statement on worker 0 must not let + * worker 1 keep applying partition DDL after worker 1 has already failed. + * + *

    Given two workers, statements are dispatched round-robin — worker 0 gets + * {@code SLOW} and worker 1 gets {@code FAIL} then {@code AFTER_FAIL}. When awaitAll + * blocks in submission order, it parks on SLOW's future while worker 1 races ahead + * and runs AFTER_FAIL. Then AFTER_FAIL must never execute: the abort flag is set by + * FAIL before worker 1 can dequeue its next statement. + * + *

    SLOW is released only after the batch has aborted, which pins the interleaving + * the bug needs — without that, SLOW could finish first and mask the race. + */ + @Test + void awaitAllStopsLaterWorkerWhenEarlierFutureIsSlow() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + List executed = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch failed = new CountDownLatch(1); + CountDownLatch releaseSlow = new CountDownLatch(1); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + doAnswer(inv -> { + String sql = inv.getArgument(0); + executed.add(sql); + if (sql.equals("FAIL")) { + failed.countDown(); + throw new RuntimeException("boom"); + } + if (sql.equals("SLOW")) { + // Hold worker 0 until worker 1 has failed, so awaitAll is definitely still + // parked on future 0 at the moment worker 1 would pick up AFTER_FAIL. + releaseSlow.await(5, TimeUnit.SECONDS); + } + return null; + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 2, factory)) { + // Round-robin over 2 workers: index 0 -> worker 0, indices 1 and 2 -> worker 1. + HiveDriverPool.Dispatch dispatch = + pool.dispatchAll(Arrays.asList("SLOW", "FAIL", "AFTER_FAIL")); + assertTrue(failed.await(5, TimeUnit.SECONDS), "FAIL must have run"); + releaseSlow.countDown(); + + HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, + () -> pool.awaitAll(dispatch)); + + assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + assertFalse(executed.contains("AFTER_FAIL"), + "Statement queued behind a failure on the same worker must not be applied, " + + "even while an earlier future on another worker is still running"); + } + } +} From 56b933050216ae4e503dab09f20825947d0bb220 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Mon, 27 Jul 2026 20:31:32 +0700 Subject: [PATCH 209/255] fix(hive-sync): drop the unreachable HMS lock timeout-recovery path (#19371) * fix(hive-sync): drop the unreachable HMS lock timeout-recovery path * addressed review comments: stub checkLock and pin the abandoned late grant --------- Co-authored-by: Vova Kolmakov (cherry picked from commit 4a5d5b0e0410b1e2313fb7ad42b049bc6e36e417) --- .../lock/HiveMetastoreBasedLockProvider.java | 18 +--- ...astoreBasedLockProviderAcquireTimeout.java | 90 +++++++++++++++++++ 2 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderAcquireTimeout.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java index 91d4062ecc3bf..a10302e1beddd 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java @@ -215,29 +215,15 @@ private boolean isLockAcquired() { private void acquireLockInternal(long time, TimeUnit unit, LockComponent lockComponent) throws InterruptedException, ExecutionException, TimeoutException, TException { - LockRequest lockRequest = null; lockLostRemotely = false; try { // TODO : FIX:Using the parameterized constructor throws MethodNotFound final LockRequestBuilder builder = new LockRequestBuilder(); - lockRequest = builder.addLockComponent(lockComponent).setUser(System.getProperty("user.name")).build(); + final LockRequest lockRequest = builder.addLockComponent(lockComponent).setUser(System.getProperty("user.name")).build(); lockRequest.setUserIsSet(true); - final LockRequest lockRequestFinal = lockRequest; - this.lock = executor.submit(() -> hiveClient.lock(lockRequestFinal)) + this.lock = executor.submit(() -> hiveClient.lock(lockRequest)) .get(time, unit); scheduleHeartbeat(); - } catch (InterruptedException | TimeoutException e) { - if (this.lock == null || this.lock.getState() != LockState.ACQUIRED) { - LockResponse lockResponse = this.hiveClient.checkLock(lockRequest.getTxnid()); - if (lockResponse.getState() == LockState.ACQUIRED) { - this.lock = lockResponse; - // The lock was granted server-side even though the client timed out waiting on the - // future; it still needs a heartbeat, otherwise a long-running commit lets HMS expire it. - scheduleHeartbeat(); - } else { - throw e; - } - } } finally { // it is better to release WAITING lock, otherwise hive lock will hang forever // Snapshot the lock: the heartbeat thread clears it as soon as the metastore reports it gone. diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderAcquireTimeout.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderAcquireTimeout.java new file mode 100644 index 0000000000000..750e8406da1b0 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/transaction/lock/TestHiveMetastoreBasedLockProviderAcquireTimeout.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.transaction.lock; + +import org.apache.hudi.exception.HoodieLockException; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchLockException; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for what {@link HiveMetastoreBasedLockProvider} reports when the metastore does not + * answer a lock request in time, with a mocked {@link IMetaStoreClient} and no live metastore or + * ZooKeeper. + */ +class TestHiveMetastoreBasedLockProviderAcquireTimeout extends HiveMetastoreBasedLockProviderTestBase { + + private static final long LOCK_ID = 42L; + private static final long ACQUIRE_TIMEOUT_MS = 200L; + + @Test + void acquireTimeoutIsReportedAsATimeout() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + CountDownLatch metastoreAnswers = new CountDownLatch(1); + CountDownLatch lockReturned = new CountDownLatch(1); + when(client.lock(any())).thenAnswer(invocation -> { + metastoreAnswers.await(); + lockReturned.countDown(); + return acquiredLock(LOCK_ID); + }); + // What a real metastore answers for the txn id 0 that the timed-out request carried. Never + // reached now, it is here so that restoring the removed lookup fails this test on the cause of + // the exception rather than on a bare NPE from an unstubbed call. + when(client.checkLock(anyLong())).thenThrow(new NoSuchLockException("No such lock 0")); + + HiveMetastoreBasedLockProvider provider = new HiveMetastoreBasedLockProvider(lockConfiguration, client); + try { + HoodieLockException thrown = assertThrows(HoodieLockException.class, + () -> provider.tryLock(ACQUIRE_TIMEOUT_MS, TimeUnit.MILLISECONDS)); + + // The metastore never answered, and that is what the writer has to be told. Looking the lock + // up afterwards cannot help: the request that timed out never returned a lock id. + assertInstanceOf(TimeoutException.class, thrown.getCause()); + verify(client, never()).checkLock(anyLong()); + assertNull(provider.getLock()); + } finally { + metastoreAnswers.countDown(); + provider.close(); + } + + // A lock granted after the client gave up is abandoned, not released: the timed-out get() never + // assigned it, so neither the acquire path nor close() knows of a lock to unlock or heartbeat. + // It stays held at the metastore until hive.txn.timeout reaps it. + assertTrue(lockReturned.await(30, TimeUnit.SECONDS)); + verify(client, never()).unlock(anyLong()); + verify(client, never()).heartbeat(anyLong(), anyLong()); + } +} From 43fd161a02bc69d6d492ce4ebcb44110520f477b Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Tue, 28 Jul 2026 17:59:12 +0800 Subject: [PATCH 210/255] fix(metadata): correct index definition lookup and improve mdt read coverage (#19359) The production fix applies unchanged: getIndexDefinitions(indexType, sourceField, metaClient) was being called with its first two arguments swapped for both the secondary-index and expression-index lookups, and this branch had the identical bug. The now-unused isIndexDefinitionPresentForColumn is removed with it. Test-side adaptations for release-1.2.1: - Package moves: HoodieIOFactory / HoodieFileReaderFactory (core.io.storage -> io.storage), Types (common.schema.internal -> internal.schema), HoodieStorageUtils (common.util -> storage), HoodieColumnRangeMetadata and ValueMetadata (metadata.stats -> stats), HoodieAvroUtils (common.avro -> avro), and Expression / Predicate (common.expression -> expression). - Dropped TestHoodieTableMetadataUtil#testFilePartitionRecordConversionHandles- DeletesAndAppends. It builds Map>, but FileInfoAndPartition arrives with a001ea87da7b (#18348, "feat(index): Add Indexer abstraction and refactor metadata table init") and does not exist here; convertFilesToFilesPartitionRecords still takes Map> on this branch. - testBloomAndColumnStatsConversionFast: pass a default HoodieMetadataConfig to convertFilesToColumnStatsRecords. This branch's signature still carries that parameter after the meta client; master dropped it. - TestJavaHoodieBackedMetadata: removed .withMetadataIndexPartitionStats(true). That builder method does not exist here, where partition stats are driven by the column stats config; the preceding .withMetadataIndexColumnStats(true) already covers it. - TestMetadataPartitionType: added the java.util.stream.Collectors import that did not carry over. (cherry picked from commit 98462b48e15b52fcbd47c4bf8bf3a3dda79020ae) --- .../client/TestJavaHoodieBackedMetadata.java | 115 ++++ .../hudi/metadata/MetadataPartitionType.java | 9 +- .../TestBaseFileRecordParsingUtils.java | 90 +++ ...tHoodieBackedTableMetadataDataCleanup.java | 389 +++++++++++- .../metadata/TestHoodieTableMetadataUtil.java | 555 +++++++++++++++++- .../metadata/TestMetadataPartitionType.java | 78 +++ .../hudi/metadata/TestBaseTableMetadata.java | 299 ++++++++++ .../TestFileSystemBackedTableMetadata.java | 36 ++ .../metadata/TestHoodieMetadataPayload.java | 63 ++ 9 files changed, 1624 insertions(+), 10 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index 57086019a8d1b..36f92e621d149 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -45,6 +45,7 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; +import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.TableServiceType; @@ -77,6 +78,7 @@ import org.apache.hudi.common.util.JsonUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.hash.PartitionIndexID; import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieCleanConfig; @@ -99,6 +101,7 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil; import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter; import org.apache.hudi.metadata.MetadataPartitionType; +import org.apache.hudi.metadata.RawKey; import org.apache.hudi.metrics.Metrics; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; @@ -164,6 +167,8 @@ import static org.apache.hudi.metadata.HoodieTableMetadataUtil.deleteMetadataTable; import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS; import static org.apache.hudi.metadata.MetadataPartitionType.FILES; +import static org.apache.hudi.metadata.MetadataPartitionType.PARTITION_STATS; +import static org.apache.hudi.metadata.MetadataPartitionType.RECORD_INDEX; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -1322,6 +1327,116 @@ public void testReadRecordIndexLocationsByBucketId() throws Exception { } } + @Test + public void testMetadataReadRoundTrip() throws Exception { + this.tableType = COPY_ON_WRITE; + initPath(); + initFileSystem(basePath, storageConf); + storage.createDirectory(new StoragePath(basePath)); + initMetaClient(tableType); + initTestDataGenerator(); + metadataTableBasePath = getMetadataTableBasePath(basePath); + + HoodieJavaEngineContext engineContext = new HoodieJavaEngineContext(storageConf); + HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withMetadataIndexColumnStats(true) + .withColumnStatsIndexForColumns(HoodieRecord.RECORD_KEY_METADATA_FIELD) + .withEnableGlobalRecordLevelIndex(true) + .withRecordIndexFileGroupCount(3, 3) + .build()) + .build(); + + try (HoodieJavaWriteClient client = new HoodieJavaWriteClient(engineContext, writeConfig)) { + String instantTime = client.startCommit(); + List records = dataGen.generateInserts(instantTime, 30); + Map recordKeyToPartition = records.stream() + .collect(Collectors.toMap(HoodieRecord::getRecordKey, HoodieRecord::getPartitionPath)); + List writeStatuses = client.insert(records, instantTime); + client.commit(instantTime, writeStatuses); + assertNoWriteErrors(writeStatuses); + + metaClient = HoodieTableMetaClient.reload(metaClient); + assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(FILES)); + assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(COLUMN_STATS)); + assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(PARTITION_STATS)); + assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(RECORD_INDEX)); + + try (HoodieTableMetadata tableMetadata = metadata(client); + HoodieTableMetadata fileSystemMetadata = new FileSystemBackedTableMetadata( + engineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath)) { + List expectedPartitions = fileSystemMetadata.getAllPartitionPaths(); + List actualPartitions = tableMetadata.getAllPartitionPaths(); + Collections.sort(expectedPartitions); + Collections.sort(actualPartitions); + assertEquals(expectedPartitions, actualPartitions); + assertFalse(actualPartitions.isEmpty()); + + List> partitionAndFileNames = new ArrayList<>(); + for (String partition : actualPartitions) { + StoragePath partitionPath = partition.isEmpty() + ? new StoragePath(basePath) + : new StoragePath(basePath, partition); + List expectedFileNames = fileSystemMetadata.getAllFilesInPartition(partitionPath).stream() + .map(pathInfo -> pathInfo.getPath().getName()) + .sorted() + .collect(Collectors.toList()); + List actualFileNames = tableMetadata.getAllFilesInPartition(partitionPath).stream() + .map(pathInfo -> pathInfo.getPath().getName()) + .sorted() + .collect(Collectors.toList()); + assertEquals(expectedFileNames, actualFileNames); + assertFalse(actualFileNames.isEmpty()); + actualFileNames.forEach(fileName -> partitionAndFileNames.add(Pair.of(partition, fileName))); + } + + Map, HoodieMetadataColumnStats> columnStats = + tableMetadata.getColumnStats(partitionAndFileNames, HoodieRecord.RECORD_KEY_METADATA_FIELD); + assertEquals(partitionAndFileNames.size(), columnStats.size()); + columnStats.values().forEach(stats -> { + assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, stats.getColumnName().toString()); + assertFalse(stats.getIsDeleted()); + assertNotNull(stats.getMinValue()); + assertNotNull(stats.getMaxValue()); + }); + + Map partitionStatsKeyToPartition = actualPartitions.stream() + .collect(Collectors.toMap( + partition -> HoodieTableMetadataUtil.getPartitionStatsIndexKey( + partition, HoodieRecord.RECORD_KEY_METADATA_FIELD), + partition -> partition)); + List partitionStatsKeys = partitionStatsKeyToPartition.keySet().stream() + .map(key -> (RawKey) () -> key) + .collect(Collectors.toList()); + List> partitionStats = tableMetadata.getRecordsByKeyPrefixes( + HoodieListData.eager(partitionStatsKeys), PARTITION_STATS.getPartitionPath(), true).collectAsList(); + assertEquals(actualPartitions.size(), partitionStats.size()); + partitionStats.forEach(record -> { + assertTrue(partitionStatsKeyToPartition.containsKey(record.getRecordKey())); + assertTrue(record.getData().getColumnStatMetadata().isPresent()); + HoodieMetadataColumnStats stats = record.getData().getColumnStatMetadata().get(); + assertEquals(partitionStatsKeyToPartition.get(record.getRecordKey()), stats.getFileName().toString()); + assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, stats.getColumnName().toString()); + assertFalse(stats.getIsDeleted()); + assertNotNull(stats.getMinValue()); + assertNotNull(stats.getMaxValue()); + }); + + List> recordLocations = tableMetadata + .readRecordIndexLocationsWithKeys(HoodieListData.eager(new ArrayList<>(recordKeyToPartition.keySet()))) + .collectAsList(); + assertEquals(records.size(), recordLocations.size()); + assertEquals(recordKeyToPartition.keySet(), + recordLocations.stream().map(Pair::getLeft).collect(Collectors.toSet())); + recordLocations.forEach(entry -> { + assertEquals(recordKeyToPartition.get(entry.getLeft()), entry.getRight().getPartitionPath()); + assertNotNull(entry.getRight().getFileId()); + }); + } + } + } + @Test public void testReadRecordIndexLocationsByBucketIdFailsWhenRecordIndexDisabled() throws Exception { init(HoodieTableType.COPY_ON_WRITE); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java index 1e12d05b1c229..655fd4e1d138b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java @@ -510,7 +510,7 @@ public static boolean isNewSecondaryIndexDefinitionRequired(HoodieMetadataConfig return false; } // check the index definition already exists or not for this column - List indexDefinitions = getIndexDefinitions(secondaryIndexColumn, PARTITION_NAME_SECONDARY_INDEX, dataMetaClient); + List indexDefinitions = getIndexDefinitions(PARTITION_NAME_SECONDARY_INDEX, secondaryIndexColumn, dataMetaClient); return indexDefinitions.isEmpty(); } @@ -531,7 +531,7 @@ public static boolean isNewExpressionIndexDefinitionRequired(HoodieMetadataConfi // get all index definitions for this column and index type // check if none of the index definitions has index function matching the expression - List indexDefinitions = getIndexDefinitions(expressionIndexColumn, PARTITION_NAME_EXPRESSION_INDEX, dataMetaClient); + List indexDefinitions = getIndexDefinitions(PARTITION_NAME_EXPRESSION_INDEX, expressionIndexColumn, dataMetaClient); return indexDefinitions.isEmpty() || indexDefinitions.stream().noneMatch(indexDefinition -> indexDefinition.getIndexFunction().equals(expressionIndexOptions.get(HoodieExpressionIndex.EXPRESSION_OPTION))); } @@ -549,11 +549,6 @@ private static List getIndexDefinitions(String indexType, return indexDefinitions; } - private static boolean isIndexDefinitionPresentForColumn(String indexedColumn, String indexType, HoodieTableMetaClient dataMetaClient) { - return dataMetaClient.getIndexMetadata().isPresent() && dataMetaClient.getIndexMetadata().get().getIndexDefinitions().values().stream() - .anyMatch(indexDefinition -> indexDefinition.getSourceFields().contains(indexedColumn) && indexDefinition.getIndexType().equals(indexType)); - } - @Override public String toString() { return "Metadata partition {" diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java new file mode 100644 index 0000000000000..c5d75fd763853 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metadata; + +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.util.FileFormatUtils; +import org.apache.hudi.io.storage.HoodieIOFactory; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.DELETE; +import static org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.INSERT; +import static org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.UPDATE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +class TestBaseFileRecordParsingUtils { + + @Test + void testRecordKeyStatusClassificationAndSecondaryIndexKeys() { + HoodieStorage storage = mock(HoodieStorage.class); + HoodieIOFactory ioFactory = mock(HoodieIOFactory.class); + FileFormatUtils fileFormatUtils = mock(FileFormatUtils.class); + when(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET)).thenReturn(fileFormatUtils); + when(fileFormatUtils.readRowKeys(any(), any(StoragePath.class))).thenAnswer(invocation -> { + StoragePath path = invocation.getArgument(1); + return path.getName().equals("latest.parquet") + ? new HashSet<>(Arrays.asList("inserted", "updated")) + : new HashSet<>(Arrays.asList("updated", "deleted")); + }); + + try (MockedStatic ioFactoryMock = mockStatic(HoodieIOFactory.class)) { + ioFactoryMock.when(() -> HoodieIOFactory.getIOFactory(storage)).thenReturn(ioFactory); + + Map> statuses = + BaseFileRecordParsingUtils.getRecordKeyStatuses( + "/table", "partition", "latest.parquet", "previous.parquet", storage, + EnumSet.allOf(BaseFileRecordParsingUtils.RecordStatus.class)); + assertEquals(Collections.singletonList("inserted"), statuses.get(INSERT)); + assertEquals(Collections.singletonList("updated"), statuses.get(UPDATE)); + assertEquals(Collections.singletonList("deleted"), statuses.get(DELETE)); + + assertTrue(BaseFileRecordParsingUtils.getRecordKeyStatuses( + "/table", "partition", "latest.parquet", null, storage, EnumSet.of(UPDATE, DELETE)).isEmpty()); + assertEquals( + new HashSet<>(Arrays.asList("inserted", "updated")), + new HashSet<>(BaseFileRecordParsingUtils.getRecordKeyStatuses( + "/table", "partition", "latest.parquet", null, storage, EnumSet.of(INSERT)).get(INSERT))); + + HoodieWriteStat writeStat = mock(HoodieWriteStat.class); + when(writeStat.getPath()).thenReturn("partition/latest.parquet"); + when(writeStat.getPartitionPath()).thenReturn("partition"); + when(writeStat.getPrevBaseFile()).thenReturn("previous.parquet"); + List changedKeys = + BaseFileRecordParsingUtils.getRecordKeysDeletedOrUpdated("/table", writeStat, storage); + assertEquals(new HashSet<>(Arrays.asList("updated", "deleted")), new HashSet<>(changedKeys)); + } + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java index 03d013f4c586c..4ab59d65ccec6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java @@ -18,23 +18,56 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.model.HoodieMetadataRecord; +import org.apache.hudi.common.config.HoodieConfig; +import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.data.HoodiePairData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.expression.Expression; +import org.apache.hudi.common.function.SerializableFunction; +import org.apache.hudi.common.function.SerializableFunctionUnchecked; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.internal.schema.Types; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.io.storage.HoodieFileReaderFactory; +import org.apache.hudi.io.storage.HoodieIOFactory; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePathInfo; +import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; +import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -239,4 +272,358 @@ public void testCleanupManagerPropagatesExceptions() throws NoSuchFieldException // Verify cleanup manager was called verify(mockCleanupManager).ensureDataCleanupOnException(any()); } -} \ No newline at end of file + + @Test + public void testSecondaryIndexUnsupportedVersion() { + HoodieData keys = HoodieListData.eager(Collections.singletonList("key")); + HoodieIndexVersion unsupportedVersion = mock(HoodieIndexVersion.class); + try (MockedStatic mockedUtil = mockStatic(HoodieTableMetadataUtil.class)) { + mockedUtil.when(() -> HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any())) + .thenReturn(unsupportedVersion); + + when(mockMetadata.readSecondaryIndexLocationsWithKeys(keys, "secondary_index_test")).thenCallRealMethod(); + when(mockMetadata.readSecondaryIndexLocations(keys, "secondary_index_test")).thenCallRealMethod(); + when(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(keys, "secondary_index_test")).thenCallRealMethod(); + + assertThrows(IllegalArgumentException.class, + () -> mockMetadata.readSecondaryIndexLocationsWithKeys(keys, "secondary_index_test")); + assertThrows(IllegalArgumentException.class, + () -> mockMetadata.readSecondaryIndexLocations(keys, "secondary_index_test")); + assertThrows(IllegalArgumentException.class, + () -> mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(keys, "secondary_index_test")); + } + } + + @Test + public void testSecondaryIndexEmptyAndMissingPartition() { + HoodieData emptyKeys = HoodieListData.eager(Collections.emptyList()); + String partitionName = "secondary_index_test"; + when(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys, partitionName)).thenCallRealMethod(); + + try (MockedStatic mockedUtil = mockStatic(HoodieTableMetadataUtil.class)) { + mockedUtil.when(() -> HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any())) + .thenReturn(HoodieIndexVersion.V1); + assertTrue(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys, partitionName) + .collectAsList().isEmpty()); + + mockedUtil.when(() -> HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any())) + .thenReturn(HoodieIndexVersion.V2); + assertTrue(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys, partitionName) + .collectAsList().isEmpty()); + + HoodieData keys = HoodieListData.eager(Collections.singletonList("key")); + mockedUtil.when(() -> HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any())) + .thenReturn(HoodieIndexVersion.V1); + when(mockTableConfig.getMetadataPartitions()).thenReturn(Collections.emptySet()); + when(mockMetadata.readSecondaryIndexLocationsWithKeys(keys, partitionName)).thenCallRealMethod(); + assertThrows(IllegalStateException.class, + () -> mockMetadata.readSecondaryIndexLocationsWithKeys(keys, partitionName)); + } + } + + @Test + public void testEmptyRecordIndexAndMetadataTimeline() throws Exception { + Field fileSliceMapField = HoodieBackedTableMetadata.class.getDeclaredField("partitionFileSliceMap"); + fileSliceMapField.setAccessible(true); + Map> fileSliceMap = new HashMap<>(); + fileSliceMap.put(MetadataPartitionType.RECORD_INDEX.getPartitionPath(), Collections.emptyList()); + fileSliceMapField.set(mockMetadata, fileSliceMap); + + when(mockMetadata.readRecordIndexLocations( + org.mockito.ArgumentMatchers., List>>any())) + .thenCallRealMethod(); + assertTrue(mockMetadata.readRecordIndexLocations(slices -> slices).collectAsList().isEmpty()); + + when(mockMetadata.getSyncedInstantTime()).thenCallRealMethod(); + when(mockMetadata.getLatestCompactionTime()).thenCallRealMethod(); + assertFalse(mockMetadata.getSyncedInstantTime().isPresent()); + assertFalse(mockMetadata.getLatestCompactionTime().isPresent()); + } + + @Test + public void testPartitionFilterFallbackAndBucketValidation() throws Exception { + List selectedPartitions = Arrays.asList("year=2025", "year=2026"); + Expression expression = mock(Expression.class); + when(expression.accept(any())).thenReturn(expression); + when(mockMetadata.getPartitionPathWithPathPrefixes(any())).thenReturn(selectedPartitions); + when(mockMetadata.getPartitionPathWithPathPrefixUsingFilterExpression(any(), any(), any())) + .thenCallRealMethod(); + + assertEquals(selectedPartitions, mockMetadata.getPartitionPathWithPathPrefixUsingFilterExpression( + Collections.singletonList("year="), mock(Types.RecordType.class), expression)); + + Field partitionedMapField = + HoodieBackedTableMetadata.class.getDeclaredField("partitionedRLIFileSliceMap"); + partitionedMapField.setAccessible(true); + partitionedMapField.set(mockMetadata, new HashMap<>()); + when(mockMetadata.getBucketizedFileGroupsForPartitionedRLI(any())).thenCallRealMethod(); + assertThrows(IllegalArgumentException.class, + () -> mockMetadata.getBucketizedFileGroupsForPartitionedRLI(MetadataPartitionType.FILES)); + + when(mockMetadata.getFilegroupsForPartition(MetadataPartitionType.RECORD_INDEX)) + .thenReturn(Collections.emptyList()); + assertTrue(mockMetadata.getBucketizedFileGroupsForPartitionedRLI( + MetadataPartitionType.RECORD_INDEX).isEmpty()); + + FileSlice nonPartitionedSlice = mock(FileSlice.class); + when(nonPartitionedSlice.getFileId()).thenReturn("record-index-0000"); + when(mockMetadata.getFilegroupsForPartition(MetadataPartitionType.RECORD_INDEX)) + .thenReturn(Collections.singletonList(nonPartitionedSlice)); + assertThrows(IllegalArgumentException.class, + () -> mockMetadata.getBucketizedFileGroupsForPartitionedRLI( + MetadataPartitionType.RECORD_INDEX)); + } + + @Test + public void testPartitionedRecordIndexLookupGuards() throws Exception { + Method lookupMethod = HoodieBackedTableMetadata.class.getDeclaredMethod( + "lookupIndexRecords", HoodieData.class, String.class, List.class, Option.class); + lookupMethod.setAccessible(true); + FileSlice partitionedSlice = mock(FileSlice.class); + when(partitionedSlice.getFileId()).thenReturn("record-index-partition-x-0000"); + List slices = Collections.singletonList(partitionedSlice); + + HoodieData emptyResult = (HoodieData) lookupMethod.invoke( + mockMetadata, + HoodieListData.eager(Collections.emptyList()), + MetadataPartitionType.RECORD_INDEX.getPartitionPath(), + slices, + Option.empty()); + assertTrue(emptyResult.collectAsList().isEmpty()); + + InvocationTargetException exception = assertThrows( + InvocationTargetException.class, + () -> lookupMethod.invoke( + mockMetadata, + HoodieListData.eager(Collections.singletonList("key")), + MetadataPartitionType.RECORD_INDEX.getPartitionPath(), + slices, + Option.empty())); + assertTrue(exception.getCause() instanceof IllegalArgumentException); + } + + @Test + public void testSecondaryIndexEmptyIteratorPath() throws Exception { + Method method = HoodieBackedTableMetadata.class.getDeclaredMethod( + "readSliceAndFilterByKeys", String.class, List.class, FileSlice.class); + method.setAccessible(true); + Object iterator = method.invoke( + mockMetadata, + MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test", + Collections.emptyList(), + mock(FileSlice.class)); + assertFalse(((org.apache.hudi.common.util.collection.ClosableIterator) iterator).hasNext()); + } + + @Test + public void testInitializationFailureDisablesMetadata() throws Exception { + Field initializedField = BaseTableMetadata.class.getDeclaredField("isMetadataTableInitialized"); + initializedField.setAccessible(true); + initializedField.set(mockMetadata, true); + Field metadataBasePathField = + HoodieBackedTableMetadata.class.getDeclaredField("metadataBasePath"); + metadataBasePathField.setAccessible(true); + metadataBasePathField.set(mockMetadata, "/table/.hoodie/metadata"); + when(mockMetadata.getStorage()).thenReturn(mock(HoodieStorage.class)); + + HoodieTableMetaClient.Builder builder = mock(HoodieTableMetaClient.Builder.class); + when(builder.setStorage(any())).thenReturn(builder); + when(builder.setBasePath(anyString())).thenReturn(builder); + when(builder.build()).thenThrow(new HoodieException("initialization failed")); + + try (MockedStatic metaClientStatic = + mockStatic(HoodieTableMetaClient.class)) { + metaClientStatic.when(HoodieTableMetaClient::builder).thenReturn(builder); + Method initMethod = HoodieBackedTableMetadata.class.getDeclaredMethod("initIfNeeded"); + initMethod.setAccessible(true); + initMethod.invoke(mockMetadata); + } + + assertFalse(mockMetadata.isMetadataTableInitialized()); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void testSecondaryIndexRecordMapping() throws Exception { + prepareFileSliceRead(false); + HoodieRecord metadataRecord = + HoodieMetadataPayload.createPartitionFilesRecord( + "key", Collections.emptyMap(), Collections.emptyList()); + IndexedRecord indexedRecord = + (IndexedRecord) metadataRecord.getData().getInsertValue( + HoodieMetadataRecord.getClassSchema()).get(); + + HoodieFileGroupReader.HoodieFileGroupReaderBuilder builder = + mock(HoodieFileGroupReader.HoodieFileGroupReaderBuilder.class); + HoodieFileGroupReader fileGroupReader = mock(HoodieFileGroupReader.class); + when(builder.withReaderContext(any())).thenReturn(builder); + when(builder.withHoodieTableMetaClient(any())).thenReturn(builder); + when(builder.withLatestCommitTime(anyString())).thenReturn(builder); + when(builder.withBaseFileOption(any())).thenReturn(builder); + when(builder.withLogFiles(any())).thenReturn(builder); + when(builder.withPartitionPath(anyString())).thenReturn(builder); + when(builder.withDataSchema(any())).thenReturn(builder); + when(builder.withRequestedSchema(any())).thenReturn(builder); + when(builder.withProps(any())).thenReturn(builder); + when(builder.withRecordBufferLoader(any())).thenReturn(builder); + when(builder.build()).thenReturn(fileGroupReader); + when(fileGroupReader.getClosableIterator()).thenReturn( + ClosableIterator.wrap(Collections.singletonList(indexedRecord).iterator())); + + FileSlice fileSlice = mock(FileSlice.class); + when(fileSlice.getPartitionPath()).thenReturn( + MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test"); + when(fileSlice.getBaseFile()).thenReturn(Option.empty()); + when(fileSlice.getLogFiles()).thenReturn(Stream.empty()); + + try (MockedStatic readerStatic = + mockStatic(HoodieFileGroupReader.class)) { + readerStatic.when(HoodieFileGroupReader::builder).thenReturn(builder); + Method method = HoodieBackedTableMetadata.class.getDeclaredMethod( + "readSliceAndFilterByKeys", String.class, List.class, FileSlice.class); + method.setAccessible(true); + ClosableIterator>> iterator = + (ClosableIterator>>) method.invoke( + mockMetadata, + MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test", + Collections.singletonList("key"), + fileSlice); + + assertTrue(iterator.hasNext()); + assertEquals("key", iterator.next().getLeft()); + assertFalse(iterator.hasNext()); + iterator.close(); + + when(fileGroupReader.getClosableIterator()) + .thenThrow(new IOException("iterator failed")); + InvocationTargetException exception = assertThrows( + InvocationTargetException.class, + () -> method.invoke( + mockMetadata, + MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test", + Collections.singletonList("key"), + fileSlice)); + assertTrue(exception.getCause() instanceof org.apache.hudi.exception.HoodieIOException); + + Method scanMethod = HoodieBackedTableMetadata.class.getDeclaredMethod( + "scanRecordsItr", FileSlice.class, SerializableFunctionUnchecked.class); + scanMethod.setAccessible(true); + InvocationTargetException scanException = assertThrows( + InvocationTargetException.class, + () -> scanMethod.invoke( + mockMetadata, + fileSlice, + (SerializableFunctionUnchecked>) record -> null)); + assertTrue(scanException.getCause() instanceof org.apache.hudi.exception.HoodieIOException); + } + } + + @Test + public void testReusableReaderIOExceptionIsWrapped() throws Exception { + prepareFileSliceRead(true); + HoodieStorage storage = mock(HoodieStorage.class); + when(mockMetadata.getStorage()).thenReturn(storage); + + HoodieBaseFile baseFile = mock(HoodieBaseFile.class); + when(baseFile.getPathInfo()).thenReturn(mock(StoragePathInfo.class)); + FileSlice fileSlice = mock(FileSlice.class); + when(fileSlice.getPartitionPath()).thenReturn(MetadataPartitionType.FILES.getPartitionPath()); + when(fileSlice.getFileGroupId()).thenReturn( + new org.apache.hudi.common.model.HoodieFileGroupId( + MetadataPartitionType.FILES.getPartitionPath(), "file-id")); + when(fileSlice.getBaseFile()).thenReturn(Option.of(baseFile)); + + HoodieIOFactory ioFactory = mock(HoodieIOFactory.class); + HoodieFileReaderFactory readerFactory = mock(HoodieFileReaderFactory.class); + when(ioFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)) + .thenReturn(readerFactory); + when(readerFactory.getFileReader( + any(HoodieConfig.class), + any(StoragePathInfo.class), + any(HoodieFileFormat.class), + any(Option.class))) + .thenThrow(new IOException("reader failed")); + + try (MockedStatic ioFactoryStatic = + mockStatic(HoodieIOFactory.class)) { + ioFactoryStatic.when(() -> HoodieIOFactory.getIOFactory(storage)).thenReturn(ioFactory); + Method method = HoodieBackedTableMetadata.class.getDeclaredMethod( + "readSliceWithFilter", + org.apache.hudi.expression.Predicate.class, + FileSlice.class); + method.setAccessible(true); + InvocationTargetException exception = assertThrows( + InvocationTargetException.class, + () -> method.invoke( + mockMetadata, + mock(org.apache.hudi.expression.Predicate.class), + fileSlice)); + assertTrue(exception.getCause() instanceof org.apache.hudi.exception.HoodieIOException); + } + } + + @Test + public void testEmptyShardReturnsEmptyIterator() throws Exception { + HoodieEngineContext engineContext = mock(HoodieEngineContext.class); + HoodieData emptyKeys = HoodieListData.eager(Collections.emptyList()); + HoodieData> emptyResult = + HoodieListData.eager(Collections.emptyList()); + when(mockMetadata.getEngineContext()).thenReturn(engineContext); + when(engineContext.parallelize( + any(List.class), org.mockito.ArgumentMatchers.eq(1))).thenReturn(emptyKeys); + when(engineContext.mapGroupsByKey(any(), any(), any(), org.mockito.ArgumentMatchers.eq(true))) + .thenAnswer(invocation -> { + SerializableFunction, + java.util.Iterator>> processFunction = + invocation.getArgument(1); + assertFalse(processFunction.apply(Collections.emptyIterator()).hasNext()); + return emptyResult; + }); + + Method method = HoodieBackedTableMetadata.class.getDeclaredMethod( + "lookupIndexRecords", HoodieData.class, String.class, List.class, + Option.class); + method.setAccessible(true); + Object result = method.invoke( + mockMetadata, + emptyKeys, + MetadataPartitionType.FILES.getPartitionPath(), + Arrays.asList(mock(FileSlice.class), mock(FileSlice.class)), + Option.empty()); + + assertEquals(emptyResult, result); + } + + private void prepareFileSliceRead(boolean reuse) throws Exception { + HoodieTableMetaClient metadataMetaClient = mock(HoodieTableMetaClient.class); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class); + when(metadataMetaClient.getActiveTimeline()).thenReturn(timeline); + when(timeline.filterCompletedInstants()).thenReturn(timeline); + when(timeline.lastInstant()).thenReturn(Option.empty()); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.populateMetaFields()).thenReturn(true); + when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET); + when(metadataMetaClient.getTableConfig()).thenReturn(tableConfig); + + Field metadataMetaClientField = + HoodieBackedTableMetadata.class.getDeclaredField("metadataMetaClient"); + metadataMetaClientField.setAccessible(true); + metadataMetaClientField.set(mockMetadata, metadataMetaClient); + Field validInstantsField = + HoodieBackedTableMetadata.class.getDeclaredField("validInstantTimestamps"); + validInstantsField.setAccessible(true); + validInstantsField.set(mockMetadata, Collections.singleton("001")); + Field reuseField = HoodieBackedTableMetadata.class.getDeclaredField("reuse"); + reuseField.setAccessible(true); + reuseField.set(mockMetadata, reuse); + Field metadataConfigField = BaseTableMetadata.class.getDeclaredField("metadataConfig"); + metadataConfigField.setAccessible(true); + metadataConfigField.set( + mockMetadata, HoodieMetadataConfig.newBuilder().enable(true).build()); + Field storageConfField = + AbstractHoodieTableMetadata.class.getDeclaredField("storageConf"); + storageConfField.setAccessible(true); + storageConfField.set(mockMetadata, mock(StorageConfiguration.class)); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java index cd35f382dedba..b75cfbfb9de96 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java @@ -19,25 +19,65 @@ package org.apache.hudi.metadata; import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.avro.model.HoodieInstantInfo; +import org.apache.hudi.avro.model.HoodieMetadataColumnStats; import org.apache.hudi.avro.model.HoodieMetadataRecord; +import org.apache.hudi.avro.model.HoodieRollbackPlan; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.data.HoodiePairData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.engine.HoodieReaderContext; +import org.apache.hudi.common.engine.ReaderContextFactory; import org.apache.hudi.common.function.SerializableBiFunction; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; +import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.TableSchemaResolver; +import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.InstantGenerator; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; - +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.stats.HoodieColumnRangeMetadata; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.HoodieStorageUtils; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; + +import org.apache.avro.AvroTypeException; +import org.apache.avro.LogicalTypes; import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Test; - +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -45,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.stream.Stream; import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS; import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS; @@ -52,8 +93,17 @@ import static org.apache.hudi.metadata.SecondaryIndexKeyUtils.constructSecondaryIndexKey; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class TestHoodieTableMetadataUtil { @@ -432,4 +482,505 @@ void testGetLocationFromRecordIndexInfoFormatsInstantConsistently() { TimeZone.setDefault(originalTimeZone); } } + + @Test + void testColumnStatsValueValidation() { + assertFalse(HoodieTableMetadataUtil.getColumnStatsValueAsString(null).isPresent()); + assertThrows(HoodieNotSupportedException.class, + () -> HoodieTableMetadataUtil.getColumnStatsValueAsString(new Object())); + } + + @Test + void testWritePartitionPathsIncludeNonPartitionedTableIdentifier() { + HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); + commitMetadata.addWriteStat("", new HoodieWriteStat()); + commitMetadata.addWriteStat("year=2026", new HoodieWriteStat()); + + assertEquals( + new java.util.HashSet<>(Arrays.asList("", "year=2026")), + HoodieTableMetadataUtil.getWritePartitionPaths(Collections.singletonList(commitMetadata))); + } + + @Test + void testDecimalUpcastValidation() { + assertThrows(AvroTypeException.class, + () -> HoodieTableMetadataUtil.tryUpcastDecimal( + new BigDecimal("1.23"), LogicalTypes.decimal(5, 1))); + assertThrows(AvroTypeException.class, + () -> HoodieTableMetadataUtil.tryUpcastDecimal( + new BigDecimal("123"), LogicalTypes.decimal(3, 1))); + assertThrows(AvroTypeException.class, + () -> HoodieTableMetadataUtil.tryUpcastDecimal( + new BigDecimal("1234"), LogicalTypes.decimal(3, 0))); + } + + @Test + void testComparableCoercion() { + assertNull(HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.INT), null)); + assertEquals(1, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.INT), true)); + assertEquals(0L, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.LONG), false)); + assertEquals(1.5f, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.FLOAT), 1.5d)); + assertEquals(2.5d, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.DOUBLE), 2.5f)); + assertEquals(1.0f, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.FLOAT), true)); + assertEquals(0.0d, HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.DOUBLE), false)); + assertNull(HoodieTableMetadataUtil.coerceToComparable( + HoodieSchema.create(HoodieSchemaType.NULL), "ignored")); + } + + @Test + void testFileGroupCountBoundsAndInflightWriteStatusTracking() { + assertEquals(10, HoodieTableMetadataUtil.estimateFileGroupCount( + MetadataPartitionType.RECORD_INDEX, () -> 10_000L, 1, 1, 10, 1.0f, 100)); + assertEquals(5, HoodieTableMetadataUtil.estimateFileGroupCount( + MetadataPartitionType.RECORD_INDEX, () -> 500L, 1, 2, 10, 1.0f, 100)); + + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(tableConfig.isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX)).thenReturn(false); + when(tableConfig.getMetadataPartitionsInflight()) + .thenReturn(Collections.singleton(MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + + assertTrue(HoodieTableMetadataUtil.getMetadataPartitionsNeedingWriteStatusTracking( + HoodieMetadataConfig.newBuilder().enable(false).build(), metaClient)); + } + + @Test + @SuppressWarnings("deprecation") + void testGenerateKeyPrefixesMatchesRawKeyEncoding() { + List columns = Arrays.asList("c1", "c2"); + assertEquals( + HoodieTableMetadataUtil.generateColumnStatsKeys(columns, "partition").stream() + .map(ColumnStatsIndexPrefixRawKey::encode) + .collect(java.util.stream.Collectors.toList()), + HoodieTableMetadataUtil.generateKeyPrefixes(columns, "partition")); + } + + @Test + void testCollectColumnRangeMetadata() { + HoodieSchema recordSchema = mock(HoodieSchema.class); + StorageConfiguration storageConfig = mock(StorageConfiguration.class); + when(storageConfig.getString( + org.apache.hudi.common.config.HoodieStorageConfig.WRITE_UTC_TIMEZONE.key(), + org.apache.hudi.common.config.HoodieStorageConfig.WRITE_UTC_TIMEZONE.defaultValue().toString())) + .thenReturn("UTC"); + + HoodieRecord record = mock(HoodieRecord.class); + when(record.getRecordType()).thenReturn(HoodieRecordType.FLINK); + when(record.getColumnValueAsJava( + org.mockito.ArgumentMatchers.eq(recordSchema), + org.mockito.ArgumentMatchers.eq("id"), + org.mockito.ArgumentMatchers.any())) + .thenReturn(7); + + HoodieSchemaField idField = HoodieSchemaField.of( + "id", HoodieSchema.create(HoodieSchemaType.INT), null, null); + HoodieSchemaField unsupportedField = HoodieSchemaField.of( + "attributes", + HoodieSchema.createMap(HoodieSchema.create(HoodieSchemaType.STRING)), + null, + null); + Map> stats = + HoodieTableMetadataUtil.collectColumnRangeMetadata( + Collections.singletonList(record).iterator(), + Arrays.asList(Pair.of("id", idField), Pair.of("attributes", unsupportedField)), + "file.parquet", + recordSchema, + storageConfig, + HoodieIndexVersion.V1); + + assertEquals(7, stats.get("id").getMinValue()); + assertEquals(7, stats.get("id").getMaxValue()); + assertNull(stats.get("attributes").getMinValue()); + } + + @Test + void testBloomAndColumnStatsConversionFast() { + HoodieLocalEngineContext engineContext = + new HoodieLocalEngineContext(mock(StorageConfiguration.class)); + Map> deletedFiles = new HashMap<>(); + deletedFiles.put("p1", Arrays.asList("file.log.1", "file_1-0-1_001.parquet")); + + HoodieData records = HoodieTableMetadataUtil.convertFilesToBloomFilterRecords( + engineContext, + deletedFiles, + Collections.emptyMap(), + "001", + mock(HoodieTableMetaClient.class), + 2, + "SIMPLE"); + assertEquals(1, records.collectAsList().size()); + + // release-1.2.1 takes an extra HoodieMetadataConfig after the meta client; master dropped + // that parameter, so pass a default config here. + assertTrue(HoodieTableMetadataUtil.convertFilesToColumnStatsRecords( + engineContext, + Collections.emptyMap(), + Collections.emptyMap(), + mock(HoodieTableMetaClient.class), + HoodieMetadataConfig.newBuilder().build(), + 1, + 1024, + Collections.singletonList("id")).collectAsList().isEmpty()); + } + + @Test + void testFilesPartitionAvailability() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(tableConfig.getMetadataPartitions()) + .thenReturn(Collections.singleton(HoodieTableMetadataUtil.PARTITION_NAME_FILES)); + + assertTrue(HoodieTableMetadataUtil.isFilesPartitionAvailable(metaClient)); + } + + @Test + void testMetadataTableDeletionOutcomes() throws Exception { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + HoodieStorage storage = mock(HoodieStorage.class); + when(metaClient.getBasePath()).thenReturn(new StoragePath("/table")); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(metaClient.getStorage()).thenReturn(storage); + + when(storage.exists(any(StoragePath.class))).thenReturn(false); + assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenThrow(new FileNotFoundException("missing")); + assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenThrow(new IOException("check failed")); + assertThrows(HoodieMetadataException.class, + () -> HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + when(storage.rename(any(StoragePath.class), any(StoragePath.class))).thenReturn(true); + assertTrue(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, true) + .contains(".metadata_")); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + when(storage.rename(any(StoragePath.class), any(StoragePath.class))) + .thenThrow(new IOException("rename failed")); + assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, true)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + org.mockito.Mockito.doThrow(new IOException("delete failed")) + .when(storage).deleteDirectory(any(StoragePath.class)); + assertThrows(HoodieMetadataException.class, + () -> HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null, false)); + } + + @Test + void testMetadataPartitionDeletionOutcomes() throws Exception { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + HoodieStorage storage = mock(HoodieStorage.class); + when(metaClient.getBasePath()).thenReturn(new StoragePath("/table")); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(metaClient.getStorage()).thenReturn(storage); + String partition = MetadataPartitionType.COLUMN_STATS.getPartitionPath(); + + when(storage.exists(any(StoragePath.class))).thenReturn(false); + assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, MetadataPartitionType.FILES.getPartitionPath(), false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenThrow(new FileNotFoundException("missing")); + assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, partition, false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenThrow(new IOException("check failed")); + assertThrows(HoodieMetadataException.class, + () -> HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, partition, false)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + when(storage.rename(any(StoragePath.class), any(StoragePath.class))).thenReturn(true); + assertTrue(HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, partition, true).contains(".metadata_")); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + when(storage.rename(any(StoragePath.class), any(StoragePath.class))) + .thenThrow(new IOException("rename failed")); + assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, partition, true)); + + reset(storage); + when(storage.exists(any(StoragePath.class))).thenReturn(true); + org.mockito.Mockito.doThrow(new IOException("delete failed")) + .when(storage).deleteDirectory(any(StoragePath.class)); + assertThrows(HoodieMetadataException.class, + () -> HoodieTableMetadataUtil.deleteMetadataTablePartition( + metaClient, null, partition, false)); + } + + @Test + void testRecordKeyReadSchemaFailureIsWrapped() { + HoodieEngineContext engineContext = mock(HoodieEngineContext.class); + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + when(metaClient.getBasePath()).thenReturn(new StoragePath("/table")); + when(metaClient.getStorageConf()).thenReturn(mock(StorageConfiguration.class)); + + assertThrows(org.apache.hudi.exception.HoodieException.class, + () -> HoodieTableMetadataUtil.readRecordKeysFromFileSlices( + engineContext, + Collections.singletonList(Pair.of("partition", mock(FileSlice.class))), + 1, + "test", + metaClient, + false)); + } + + @Test + @SuppressWarnings("unchecked") + void testPartitionStatsConversionFailureIsWrapped() { + HoodiePairData>> pairData = + mock(HoodiePairData.class); + when(pairData.flatMapValues(any())).thenThrow(new RuntimeException("conversion failed")); + + assertThrows(org.apache.hudi.exception.HoodieException.class, + () -> HoodieTableMetadataUtil.convertMetadataToPartitionStatsRecords( + pairData, + mock(HoodieTableMetaClient.class), + Collections.emptyMap(), + HoodieIndexVersion.V1)); + } + + @Test + void testMergeColumnStatsTombstoneWins() { + HoodieMetadataColumnStats previous = HoodieMetadataColumnStats.newBuilder() + .setColumnName("column") + .setIsDeleted(false) + .build(); + HoodieMetadataColumnStats tombstone = HoodieMetadataColumnStats.newBuilder() + .setColumnName("column") + .setIsDeleted(true) + .build(); + + assertEquals(tombstone, HoodieTableMetadataUtil.mergeColumnStatsRecords(previous, tombstone)); + assertEquals(previous, HoodieTableMetadataUtil.mergeColumnStatsRecords(tombstone, previous)); + } + + @Test + void testFileSliceAndSchemaResolutionEdge() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTimeline timeline = mock(HoodieTimeline.class); + when(metaClient.getCommitsTimeline()).thenReturn(timeline); + when(timeline.filterCompletedInstants()).thenReturn(timeline); + when(timeline.countInstants()).thenReturn(1); + when(metaClient.getBasePath()).thenReturn(new StoragePath("/table")); + assertThrows(org.apache.hudi.exception.HoodieException.class, + () -> HoodieTableMetadataUtil.tryResolveSchemaForTable(metaClient)); + + HoodieTableFileSystemView fileSystemView = mock(HoodieTableFileSystemView.class); + HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); + when(metaClient.getActiveTimeline()).thenReturn(activeTimeline); + when(activeTimeline.filterCompletedInstants()).thenReturn(activeTimeline); + when(activeTimeline.lastInstant()).thenReturn(Option.empty()); + assertTrue(HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices( + metaClient, fileSystemView, "files").isEmpty()); + } + + @Test + void testEmptyLogInputsAvoidReaderConstruction() { + Pair, java.util.Set> changes = + HoodieTableMetadataUtil.getRevivedAndDeletedKeysFromMergedLogs( + mock(HoodieTableMetaClient.class), + "001", + Collections.singletonList("previous.log"), + Option.empty(), + Collections.singletonList("current.log"), + "partition", + mock(org.apache.hudi.common.engine.HoodieReaderContext.class)); + + assertTrue(changes.getLeft().isEmpty()); + assertTrue(changes.getRight().isEmpty()); + } + + @Test + void testRollbackPlanFallbackAndReadFailure() throws Exception { + Method method = HoodieTableMetadataUtil.class.getDeclaredMethod( + "getRollbackedCommits", + HoodieInstant.class, + HoodieActiveTimeline.class, + InstantGenerator.class); + method.setAccessible(true); + + HoodieInstant completed = mock(HoodieInstant.class); + HoodieInstant requested = mock(HoodieInstant.class); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class); + InstantGenerator instantGenerator = mock(InstantGenerator.class); + HoodieRollbackPlan rollbackPlan = mock(HoodieRollbackPlan.class); + HoodieInstantInfo instantInfo = mock(HoodieInstantInfo.class); + when(completed.getAction()).thenReturn(HoodieTimeline.ROLLBACK_ACTION); + when(completed.requestedTime()).thenReturn("002"); + when(timeline.readRollbackMetadata(completed)).thenThrow(new IOException("empty rollback")); + when(instantGenerator.createNewInstant( + HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, "002")) + .thenReturn(requested); + when(timeline.readRollbackPlan(requested)).thenReturn(rollbackPlan); + when(rollbackPlan.getInstantToRollback()).thenReturn(instantInfo); + when(instantInfo.getCommitTime()).thenReturn("001"); + assertEquals(Collections.singletonList("001"), method.invoke( + null, completed, timeline, instantGenerator)); + + when(completed.getAction()).thenReturn(HoodieTimeline.RESTORE_ACTION); + when(timeline.readRestoreMetadata(completed)).thenThrow(new IOException("broken restore")); + InvocationTargetException exception = assertThrows( + InvocationTargetException.class, + () -> method.invoke(null, completed, timeline, instantGenerator)); + assertTrue(exception.getCause() instanceof HoodieMetadataException); + } + + @Test + void testMetadataPartitionExistenceFailureIsWrapped() throws Exception { + HoodieEngineContext context = mock(HoodieEngineContext.class); + StorageConfiguration storageConfiguration = mock(StorageConfiguration.class); + HoodieStorage storage = mock(HoodieStorage.class); + org.mockito.Mockito.doReturn(storageConfiguration).when(context).getStorageConf(); + when(storage.exists(any(StoragePath.class))).thenThrow(new IOException("failed")); + + try (MockedStatic storageUtils = mockStatic(HoodieStorageUtils.class)) { + storageUtils.when(() -> HoodieStorageUtils.getStorage(any(String.class), any())) + .thenReturn(storage); + assertThrows(org.apache.hudi.exception.HoodieIOException.class, + () -> HoodieTableMetadataUtil.metadataPartitionExists( + "/table", context, MetadataPartitionType.FILES.getPartitionPath())); + } + } + + @Test + @SuppressWarnings("unchecked") + void testDeletedFileStatsCreateStubs() throws Exception { + Method method = HoodieTableMetadataUtil.class.getDeclaredMethod( + "getFileStatsRangeMetadata", + String.class, + String.class, + HoodieTableMetaClient.class, + List.class, + boolean.class, + int.class, + HoodieIndexVersion.class); + method.setAccessible(true); + + List> stats = + (List>) method.invoke( + null, + "partition", + "file.parquet", + mock(HoodieTableMetaClient.class), + Arrays.asList("c1", "c2"), + true, + 1024, + HoodieIndexVersion.V1); + assertEquals(2, stats.size()); + } + + @Test + void testCommitPartitionExtraction() throws Exception { + HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); + commitMetadata.addWriteStat("", new HoodieWriteStat()); + commitMetadata.addWriteStat("partition", new HoodieWriteStat()); + Method method = HoodieTableMetadataUtil.class.getDeclaredMethod( + "getPartitionsAdded", HoodieCommitMetadata.class); + method.setAccessible(true); + + assertEquals( + new java.util.HashSet<>(Arrays.asList(HoodieTableMetadata.NON_PARTITIONED_NAME, "partition")), + new java.util.HashSet<>((List) method.invoke(null, commitMetadata))); + } + + @Test + void testInflightFileSliceViewIsClosedWhenCreatedInternally() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableFileSystemView fileSystemView = mock(HoodieTableFileSystemView.class); + when(fileSystemView.getLatestFileSlicesIncludingInflight("files")) + .thenReturn(Stream.empty()); + + try (MockedStatic util = + mockStatic(HoodieTableMetadataUtil.class, org.mockito.Answers.CALLS_REAL_METHODS)) { + util.when(() -> HoodieTableMetadataUtil.getFileSystemViewForMetadataTable(metaClient)) + .thenReturn(fileSystemView); + assertTrue(HoodieTableMetadataUtil.getPartitionLatestFileSlicesIncludingInflight( + metaClient, Option.empty(), "files").isEmpty()); + } + + verify(fileSystemView).close(); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + void testLogOnlyFileSliceRecordKeys() throws Exception { + HoodieEngineContext engineContext = mock(HoodieEngineContext.class); + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieSchema schema = mock(HoodieSchema.class); + FileSlice fileSlice = mock(FileSlice.class); + List> slices = + Collections.singletonList(Pair.of("partition", fileSlice)); + when(metaClient.getBasePath()).thenReturn(new StoragePath("/table")); + StorageConfiguration storageConfiguration = mock(StorageConfiguration.class); + when(storageConfiguration.getEnum( + any(String.class), any(ExternalSpillableMap.DiskMapType.class))) + .thenAnswer(invocation -> invocation.getArgument(1)); + org.mockito.Mockito.doReturn(storageConfiguration).when(metaClient).getStorageConf(); + HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class); + when(metaClient.getActiveTimeline()).thenReturn(timeline); + when(timeline.filterCompletedInstants()).thenReturn(timeline); + when(timeline.lastInstant()).thenReturn(Option.empty()); + when(fileSlice.getBaseFile()).thenReturn(Option.empty()); + when(fileSlice.getLogFiles()).thenReturn(Stream.empty()); + when(fileSlice.getPartitionPath()).thenReturn("partition"); + when(fileSlice.getFileId()).thenReturn("file-id"); + when(fileSlice.getBaseInstantTime()).thenReturn("20240101000000000"); + org.mockito.Mockito.doReturn(HoodieListData.eager(slices)) + .when(engineContext).parallelize(anyList(), anyInt()); + + ReaderContextFactory readerContextFactory = mock(ReaderContextFactory.class); + HoodieReaderContext readerContext = mock(HoodieReaderContext.class); + org.mockito.Mockito.doReturn(readerContextFactory) + .when(engineContext).getReaderContextFactory(metaClient); + when(readerContextFactory.getContext()).thenReturn(readerContext); + + HoodieFileGroupReader.HoodieFileGroupReaderBuilder builder = + mock(HoodieFileGroupReader.HoodieFileGroupReaderBuilder.class); + HoodieFileGroupReader fileGroupReader = mock(HoodieFileGroupReader.class); + when(builder.withReaderContext(any())).thenReturn(builder); + when(builder.withHoodieTableMetaClient(any())).thenReturn(builder); + when(builder.withBaseFileOption(any())).thenReturn(builder); + when(builder.withLogFiles(any())).thenReturn(builder); + when(builder.withPartitionPath(any())).thenReturn(builder); + when(builder.withDataSchema(any())).thenReturn(builder); + when(builder.withRequestedSchema(any())).thenReturn(builder); + when(builder.withLatestCommitTime(any())).thenReturn(builder); + when(builder.withProps(any())).thenReturn(builder); + when(builder.build()).thenReturn(fileGroupReader); + when(fileGroupReader.getClosableKeyIterator()) + .thenReturn(ClosableIterator.wrap(Collections.emptyIterator())); + + try (MockedConstruction ignored = + mockConstruction(TableSchemaResolver.class, + (resolver, context) -> when(resolver.getTableSchema()).thenReturn(schema)); + MockedStatic readerStatic = + mockStatic(HoodieFileGroupReader.class)) { + readerStatic.when(HoodieFileGroupReader::builder).thenReturn(builder); + assertTrue(HoodieTableMetadataUtil.readRecordKeysFromFileSlices( + engineContext, slices, 1, "test", metaClient, false).collectAsList().isEmpty()); + } + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java index 2ccddf81add3c..062b4b967d8d0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java @@ -19,6 +19,8 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.model.HoodieMetadataBloomFilter; +import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; @@ -28,6 +30,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -35,11 +39,13 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -312,4 +318,76 @@ public void testIsExpressionOrSecondaryIndex() { } } } + + @Test + public void testProjectedPayloadConstruction() { + HoodieMetadataPayload payload = new HoodieMetadataPayload(Option.empty()); + Schema projectedSchema = Schema.createRecord("ProjectedMetadataRecord", null, null, false); + projectedSchema.setFields(Collections.emptyList()); + GenericData.Record projectedRecord = new GenericData.Record(projectedSchema); + + MetadataPartitionType.FILES.constructMetadataPayload(payload, projectedRecord); + MetadataPartitionType.COLUMN_STATS.constructMetadataPayload(payload, projectedRecord); + assertThrows(UnsupportedOperationException.class, + () -> MetadataPartitionType.EXPRESSION_INDEX.constructMetadataPayload(payload, projectedRecord)); + + GenericData.Record invalidBloomFilterRecord = + new GenericData.Record(HoodieMetadataRecord.getClassSchema()); + assertThrows(IllegalArgumentException.class, + () -> MetadataPartitionType.BLOOM_FILTERS.constructMetadataPayload(payload, invalidBloomFilterRecord)); + assertThrows(IllegalArgumentException.class, () -> MetadataPartitionType.get(Integer.MAX_VALUE)); + } + + @Test + public void testBloomFilterCombinationAndAllPartitionsEnablement() { + HoodieMetadataPayload older = new HoodieMetadataPayload("key", + new HoodieMetadataBloomFilter("SIMPLE", "1", ByteBuffer.wrap(new byte[] {1}), false)); + HoodieMetadataPayload newer = new HoodieMetadataPayload("key", + new HoodieMetadataBloomFilter("SIMPLE", "2", ByteBuffer.wrap(new byte[] {2}), false)); + + HoodieMetadataPayload combined = MetadataPartitionType.BLOOM_FILTERS.combineMetadataPayloads(older, newer); + + assertEquals(newer.getBloomFilterMetadata(), combined.getBloomFilterMetadata()); + assertTrue(MetadataPartitionType.ALL_PARTITIONS.isMetadataPartitionEnabled( + HoodieMetadataConfig.newBuilder().enable(true).build(), mock(HoodieTableConfig.class))); + } + + @Test + public void testNewIndexDefinitionChecks() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieIndexDefinition secondaryIndex = createIndexDefinition( + MetadataPartitionType.SECONDARY_INDEX, "existing", HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX, + null, Collections.singletonList("secondary_col"), null); + HoodieIndexDefinition expressionIndex = createIndexDefinition( + MetadataPartitionType.EXPRESSION_INDEX, "existing", HoodieTableMetadataUtil.PARTITION_NAME_EXPRESSION_INDEX, + "lower", Collections.singletonList("expression_col"), null); + HoodieIndexMetadata indexMetadata = new HoodieIndexMetadata(createIndexDefinitions(secondaryIndex, expressionIndex)); + when(metaClient.getIndexMetadata()).thenReturn(Option.of(indexMetadata)); + + HoodieMetadataConfig secondaryConfig = HoodieMetadataConfig.newBuilder() + .withSecondaryIndexForColumn("secondary_col") + .build(); + assertFalse(MetadataPartitionType.isNewSecondaryIndexDefinitionRequired(secondaryConfig, metaClient)); + + HoodieMetadataConfig expressionConfig = HoodieMetadataConfig.newBuilder() + .withExpressionIndexColumn("expression_col") + .withExpressionIndexOptions(Collections.singletonMap("expr", "lower")) + .build(); + assertFalse(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(expressionConfig, metaClient)); + + HoodieMetadataConfig differentExpressionConfig = HoodieMetadataConfig.newBuilder() + .withExpressionIndexColumn("expression_col") + .withExpressionIndexOptions(Collections.singletonMap("expr", "upper")) + .build(); + assertTrue(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(differentExpressionConfig, metaClient)); + + HoodieMetadataConfig noExpressionConfig = HoodieMetadataConfig.newBuilder() + .withExpressionIndexColumn("expression_col") + .build(); + assertFalse(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(noExpressionConfig, metaClient)); + } + + private static Map createIndexDefinitions(HoodieIndexDefinition... definitions) { + return Arrays.stream(definitions).collect(Collectors.toMap(HoodieIndexDefinition::getIndexName, definition -> definition)); + } } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java new file mode 100644 index 0000000000000..94a3ffe8dd9a2 --- /dev/null +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metadata; + +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.data.HoodieListData; +import org.apache.hudi.common.data.HoodieListPairData; +import org.apache.hudi.common.data.HoodiePairData; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.expression.Expression; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordGlobalLocation; +import org.apache.hudi.internal.schema.Types; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestBaseTableMetadata { + + @TempDir + Path tempDir; + + private HoodieTableMetaClient metaClient; + private String basePath; + + @BeforeEach + void setUp() throws Exception { + basePath = tempDir.toString(); + metaClient = HoodieTestUtils.init(basePath); + } + + @Test + void testReadFailuresAreWrappedWithMetadataContext() { + TestingTableMetadata metadata = newMetadata(); + metadata.failSingleReads = true; + assertThrows(HoodieMetadataException.class, metadata::getAllPartitionPaths); + + metadata.failSingleReads = false; + metadata.failBulkReads = true; + assertThrows(HoodieMetadataException.class, + () -> metadata.getAllFilesInPartitions(Collections.singletonList(basePath))); + } + + @Test + void testDisabledIndexesAndEmptyBloomFilterLookup() { + TestingTableMetadata metadata = newMetadata(); + assertFalse(metadata.getBloomFilter("", "file.parquet", MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isPresent()); + assertTrue(metadata.getBloomFilters( + Collections.singletonList(Pair.of("", "file.parquet")), + MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isEmpty()); + assertTrue(metadata.getColumnStats( + Collections.singletonList(Pair.of("", "file.parquet")), + Collections.singletonList("column")).isEmpty()); + + metaClient.getTableConfig().setMetadataPartitionState( + metaClient, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath(), true); + metadata = newMetadata(); + assertFalse(metadata.getBloomFilter( + "", "file.parquet", MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isPresent()); + assertTrue(metadata.getBloomFilters( + Collections.emptyList(), MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isEmpty()); + } + + @Test + void testPayloadFailuresAndMissingColumnStats() { + HoodieMetadataPayload payload = new HoodieMetadataPayload( + "bad", MetadataPartitionType.FILES.getRecordType(), Collections.emptyMap()) { + @Override + public List getFileList( + HoodieStorage storage, StoragePath partitionPath) { + throw new HoodieException("corrupt payload"); + } + }; + + TestingTableMetadata firstMetadata = newMetadata(); + firstMetadata.singleRecord = Option.of(payload); + assertThrows(HoodieException.class, + () -> firstMetadata.getAllFilesInPartition(new StoragePath(basePath))); + + HoodieMetadataPayload inconsistentPayload = + HoodieMetadataPayload.createPartitionListRecord( + Collections.singletonList("ghost"), true).getData(); + firstMetadata.singleRecord = Option.of(inconsistentPayload); + assertThrows(HoodieMetadataException.class, firstMetadata::getAllPartitionPaths); + + metaClient.getTableConfig().setMetadataPartitionState( + metaClient, MetadataPartitionType.COLUMN_STATS.getPartitionPath(), true); + TestingTableMetadata metadata = newMetadata(); + HoodieMetadataPayload missingColumnStats = + HoodieMetadataPayload.createPartitionFilesRecord( + "", Collections.singletonMap("file.parquet", 1L), + Collections.emptyList()).getData(); + metadata.pairRecords = HoodieListPairData.eager( + Collections.singletonList(Pair.of("missing-key", missingColumnStats))); + assertTrue(metadata.getColumnStats( + Collections.singletonList(Pair.of("", "file.parquet")), + Collections.singletonList("column")).isEmpty()); + } + + @Test + void testProtectedReadContextAccessors() { + TestingTableMetadata metadata = newMetadata(); + assertNotNull(metadata.storageConfiguration()); + assertEquals("00000000000000", metadata.latestDataInstant()); + } + + @Test + void testHoodieBackedMetadataStaysDisabledWithoutMetadataTable() { + HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata( + null, + metaClient.getStorage(), + HoodieMetadataConfig.newBuilder().enable(false).build(), + basePath); + + assertFalse(metadata.isMetadataTableInitialized()); + assertFalse(metadata.getSyncedInstantTime().isPresent()); + assertFalse(metadata.getLatestCompactionTime().isPresent()); + metadata.close(); + } + + private TestingTableMetadata newMetadata() { + return new TestingTableMetadata( + null, metaClient.getStorage(), + HoodieMetadataConfig.newBuilder() + .enable(true) + .ignoreSpuriousDeletes(false) + .build(), + basePath); + } + + private static class TestingTableMetadata extends BaseTableMetadata { + private boolean failSingleReads; + private boolean failBulkReads; + private Option singleRecord = Option.empty(); + private HoodiePairData pairRecords = + HoodieListPairData.eager(Collections.emptyList()); + + TestingTableMetadata(HoodieEngineContext engineContext, + HoodieStorage storage, + HoodieMetadataConfig metadataConfig, + String dataBasePath) { + super(engineContext, storage, metadataConfig, dataBasePath); + isMetadataTableInitialized = true; + } + + @Override + protected Option readFilesIndexRecords(String key, String partitionName) { + if (failSingleReads) { + throw new HoodieException("single read failed"); + } + return singleRecord; + } + + @Override + public List getPartitionPathWithPathPrefixUsingFilterExpression( + List relativePathPrefixes, + Types.RecordType partitionFields, + Expression expression) { + return Collections.emptyList(); + } + + @Override + public List getPartitionPathWithPathPrefixes(List relativePathPrefixes) { + return Collections.emptyList(); + } + + @Override + public HoodiePairData readIndexRecordsWithKeys( + HoodieData rawKeys, String partitionName) { + if (failBulkReads) { + throw new HoodieException("bulk read failed"); + } + return pairRecords; + } + + @Override + protected HoodiePairData readIndexRecordsWithKeys( + HoodieData rawKeys, + String partitionName, + Option dataTablePartition) { + return readIndexRecordsWithKeys(rawKeys, partitionName); + } + + @Override + public HoodiePairData readSecondaryIndexDataTableRecordKeysWithKeys( + HoodieData keys, String partitionName) { + return HoodieListPairData.eager(Collections.emptyList()); + } + + @Override + public HoodiePairData readSecondaryIndexLocationsWithKeys( + HoodieData secondaryKeys, String partitionName) { + return HoodieListPairData.eager(Collections.emptyList()); + } + + @Override + public HoodiePairData readRecordIndexLocationsWithKeys( + HoodieData recordKeys) { + return HoodieListPairData.eager(Collections.emptyList()); + } + + @Override + public HoodiePairData readRecordIndexLocationsWithKeys( + HoodieData recordKeys, Option dataTablePartition) { + return HoodieListPairData.eager(Collections.emptyList()); + } + + @Override + public HoodieData> getRecordsByKeyPrefixes( + HoodieData rawKeys, + String partitionName, + boolean shouldLoadInMemory) { + return HoodieListData.eager(Collections.emptyList()); + } + + @Override + public Map, List> listPartitions( + List> partitionPathList) { + return Collections.emptyMap(); + } + + @Override + public Option getSyncedInstantTime() { + return Option.empty(); + } + + @Override + public Option getLatestCompactionTime() { + return Option.empty(); + } + + @Override + public void reset() { + } + + @Override + public void close() { + } + + @Override + public int getNumFileGroupsForPartition(MetadataPartitionType partition) { + return 0; + } + + @Override + public Map> getBucketizedFileGroupsForPartitionedRLI( + MetadataPartitionType partition) { + return Collections.emptyMap(); + } + + StorageConfiguration storageConfiguration() { + return getStorageConf(); + } + + String latestDataInstant() { + return getLatestDataInstantTime(); + } + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java index 48a2851c0924d..30a26b65c9c9b 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java @@ -18,10 +18,13 @@ package org.apache.hudi.metadata; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestTable; +import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; @@ -256,4 +259,37 @@ public void testMultiLevelEmptyPartitionTable() throws Exception { } } + @Test + public void testMetadataIndexOperationsAreUnsupported() { + HoodieLocalEngineContext localEngineContext = + new HoodieLocalEngineContext(metaClient.getStorageConf()); + FileSystemBackedTableMetadata metadata = new FileSystemBackedTableMetadata( + localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath); + + Assertions.assertThrows(UnsupportedOperationException.class, metadata::getSyncedInstantTime); + Assertions.assertThrows(UnsupportedOperationException.class, metadata::getLatestCompactionTime); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getBloomFilter("", "file.parquet", MetadataPartitionType.BLOOM_FILTERS.getPartitionPath())); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getBloomFilters(Collections.emptyList(), MetadataPartitionType.BLOOM_FILTERS.getPartitionPath())); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getColumnStats(Collections.emptyList(), "column")); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getColumnStats(Collections.emptyList(), Collections.singletonList("column"))); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getRecordsByKeyPrefixes( + HoodieListData.eager(Collections.emptyList()), MetadataPartitionType.FILES.getPartitionPath(), false)); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(Collections.emptyList()))); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(Collections.emptyList()), Option.empty())); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.readSecondaryIndexLocationsWithKeys( + HoodieListData.eager(Collections.emptyList()), MetadataPartitionType.SECONDARY_INDEX.getPartitionPath())); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getNumFileGroupsForPartition(MetadataPartitionType.FILES)); + Assertions.assertThrows(HoodieMetadataException.class, + () -> metadata.getBucketizedFileGroupsForPartitionedRLI(MetadataPartitionType.RECORD_INDEX)); + } + } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java index f928b46b7c9c2..204467f9d21fc 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java @@ -18,17 +18,24 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.stats.HoodieColumnRangeMetadata; import org.apache.hudi.stats.ValueMetadata; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -325,6 +332,62 @@ public void testSecondaryIndexPayloadMerging() { assertEquals(newSecondaryIndexRecord.getData(), combinedSecondaryIndexRecord.get().getData()); } + @Test + public void testPayloadAccessorsAndObjectMethods() { + HoodieMetadataPayload emptyPayload = new HoodieMetadataPayload(Option.empty()); + assertFalse(emptyPayload.getBloomFilterMetadata().isPresent()); + assertFalse(emptyPayload.getColumnStatMetadata().isPresent()); + assertFalse(emptyPayload.equals("not-a-payload")); + emptyPayload.hashCode(); + + HoodieMetadataPayload secondaryIndexPayload = HoodieMetadataPayload.createSecondaryIndexRecord( + "record-key", "secondary-key", MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test", true).getData(); + assertTrue(secondaryIndexPayload.isSecondaryIndexDeleted()); + } + + @Test + public void testInvalidRecordIndexInputs() { + assertThrows(HoodieMetadataException.class, + () -> HoodieMetadataPayload.parseRecordIndexInstantTime("not-an-instant")); + assertThrows(HoodieMetadataException.class, + () -> HoodieMetadataPayload.createRecordIndexUpdate( + "record-key", PARTITION_NAME, "not-a-uuid", "20240101000000000", 0)); + } + + @Test + public void testProjectedInsertValueIncludesBloomFilter() throws IOException { + HoodieMetadataPayload bloomFilterPayload = HoodieMetadataPayload.createBloomFilterMetadataRecord( + PARTITION_NAME, "file-id_1-0-1_20240101000000000.parquet", "20240101000000000", "SIMPLE", + ByteBuffer.wrap("bloom-data".getBytes()), false).getData(); + Schema projectedSchema = HoodieSchemaUtils.addMetadataFields( + HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())).toAvroSchema(); + + IndexedRecord projectedRecord = bloomFilterPayload.getInsertValue(projectedSchema).get(); + + assertEquals(bloomFilterPayload.getBloomFilterMetadata().get(), + ((GenericRecord) projectedRecord).get("BloomFilterMetadata")); + } + + @Test + public void testPayloadToStringForIndexedRecordTypes() { + HoodieMetadataPayload filesPayload = HoodieMetadataPayload.createPartitionFilesRecord( + PARTITION_NAME, Collections.singletonMap("file.parquet", 10L), Collections.singletonList("old.parquet")).getData(); + assertTrue(filesPayload.toString().contains("creations=[file.parquet]")); + assertTrue(filesPayload.toString().contains("deletions=[old.parquet]")); + + HoodieMetadataPayload bloomFilterPayload = HoodieMetadataPayload.createBloomFilterMetadataRecord( + PARTITION_NAME, "file-id_1-0-1_20240101000000000.parquet", "20240101000000000", "SIMPLE", + ByteBuffer.wrap("bloom-data".getBytes()), false).getData(); + assertTrue(bloomFilterPayload.toString().contains("BloomFilter")); + + HoodieColumnRangeMetadata columnRange = HoodieColumnRangeMetadata.create( + "file.parquet", "column", 1, 2, 0, 2, 10, 10, ValueMetadata.V1EmptyMetadata.get()); + HoodieMetadataPayload columnStatsPayload = + (HoodieMetadataPayload) HoodieMetadataPayload.createColumnStatsRecords( + PARTITION_NAME, Collections.singletonList(columnRange), false).findFirst().get().getData(); + assertTrue(columnStatsPayload.toString().contains("ColStats")); + } + @Test public void testConstructSecondaryIndexKey() { // Simple case From 71072a60b7d38baadc22e665dc0dcb63f09e072d Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Tue, 28 Jul 2026 18:04:06 +0800 Subject: [PATCH 211/255] fix(common): load single archived instant details (#19385) * Fix loading single archived compaction plan * Fix completed instant detail range (cherry picked from commit 527b0ec813084886edd3e52674af854cfc5ca621) --- .../timeline/TestArchivedTimelineV2.java | 69 +++++++++++++++++-- .../versioning/v2/ArchivedTimelineV2.java | 4 +- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java index 02ca10c102701..cd977c41a99af 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java @@ -26,8 +26,10 @@ import org.apache.hudi.common.engine.LocalTaskContextSupplier; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestTable; +import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; @@ -38,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; @@ -46,6 +49,8 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; /** @@ -77,6 +82,54 @@ public void testLoadingInstantsIncrementally() throws Exception { assertThat(archivedTimeline.firstInstant().map(HoodieInstant::requestedTime).orElse(""), is("10000011")); } + @Test + void testLoadCompactionDetailsForSingleInstant() { + String instantTime = "10000001"; + byte[] compactionPlan = {1, 2, 3}; + HoodieInstant completed = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, instantTime, "10000002"); + ActiveAction activeAction = new DummyActiveAction(completed, new byte[0]) { + @Override + public String getPendingAction() { + return HoodieTimeline.COMPACTION_ACTION; + } + + @Override + public Option getCompactionPlan(HoodieTableMetaClient metaClient) { + return Option.of(compactionPlan); + } + }; + createTimelineWriter().write( + Collections.singletonList(activeAction), Option.empty(), Option.empty()); + + HoodieArchivedTimeline archivedTimeline = metaClient.getArchivedTimeline(); + HoodieInstant archivedInstant = archivedTimeline.firstInstant().get(); + assertFalse(archivedTimeline.getInstantDetails(archivedInstant).isPresent()); + + archivedTimeline.loadCompactionDetailsInMemory(instantTime); + + assertArrayEquals(compactionPlan, archivedTimeline.getInstantDetails(archivedInstant).get()); + } + + @Test + void testLoadCompletedDetailsForSingleInstant() { + String instantTime = "10000001"; + byte[] commitMetadata = {1, 2, 3}; + HoodieInstant completed = INSTANT_GENERATOR.createNewInstant( + HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, instantTime, "10000002"); + createTimelineWriter().write( + Collections.singletonList(new DummyActiveAction(completed, commitMetadata)), + Option.empty(), Option.empty()); + + HoodieArchivedTimeline archivedTimeline = metaClient.getArchivedTimeline(); + HoodieInstant archivedInstant = archivedTimeline.firstInstant().get(); + assertFalse(archivedTimeline.getInstantDetails(archivedInstant).isPresent()); + + archivedTimeline.loadCompletedInstantDetailsInMemory(instantTime, instantTime); + + assertArrayEquals(commitMetadata, archivedTimeline.getInstantDetails(archivedInstant).get()); + } + @Test void getInstantReaderReferencesSelf() { HoodieArchivedTimeline timeline = TIMELINE_FACTORY.createArchivedTimeline(metaClient); @@ -88,12 +141,8 @@ void getInstantReaderReferencesSelf() { private void writeArchivedTimeline(int batchSize, long startTs) throws Exception { HoodieTestTable testTable = HoodieTestTable.of(this.metaClient); - HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(this.metaClient.getBasePath()) - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) - .withMarkersType("DIRECT") - .build(); + LSMTimelineWriter writer = createTimelineWriter(); HoodieEngineContext engineContext = new HoodieLocalEngineContext(getDefaultStorageConf()); - LSMTimelineWriter writer = LSMTimelineWriter.getInstance(writeConfig, new LocalTaskContextSupplier(), metaClient); List instantBuffer = new ArrayList<>(); for (int i = 1; i <= 50; i++) { long instantTimeTs = startTs + i; @@ -105,10 +154,18 @@ private void writeArchivedTimeline(int batchSize, long startTs) throws Exception instantBuffer.add(new DummyActiveAction(instant, serializedMetadata)); if (i % batchSize == 0) { // archive 10 instants each time - writer.write(instantBuffer, org.apache.hudi.common.util.Option.empty(), org.apache.hudi.common.util.Option.empty()); + writer.write(instantBuffer, Option.empty(), Option.empty()); writer.compactAndClean(engineContext); instantBuffer.clear(); } } } + + private LSMTimelineWriter createTimelineWriter() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(this.metaClient.getBasePath()) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + .withMarkersType("DIRECT") + .build(); + return LSMTimelineWriter.getInstance(writeConfig, new LocalTaskContextSupplier(), metaClient); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java index 9a181cf9bcbec..f88168ea985c4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java @@ -147,7 +147,7 @@ public void loadCompactionDetailsInMemory(String compactionInstantTime) { public void loadCompactionDetailsInMemory(String startTs, String endTs) { // load compactionPlan - List loadedInstants = loadInstants(new HoodieArchivedTimeline.TimeRangeFilter(startTs, endTs), HoodieArchivedTimeline.LoadMode.PLAN, + List loadedInstants = loadInstants(new HoodieArchivedTimeline.ClosedClosedTimeRangeFilter(startTs, endTs), HoodieArchivedTimeline.LoadMode.PLAN, record -> record.get(ACTION_ARCHIVED_META_FIELD).toString().equals(COMMIT_ACTION) && record.get(PLAN_ARCHIVED_META_FIELD) != null ); @@ -164,7 +164,7 @@ record -> record.get(ACTION_ARCHIVED_META_FIELD).toString().equals(COMMIT_ACTION @Override public void loadCompletedInstantDetailsInMemory(String startTs, String endTs) { - List loadedInstants = loadInstants(new HoodieArchivedTimeline.TimeRangeFilter(startTs, endTs), HoodieArchivedTimeline.LoadMode.METADATA); + List loadedInstants = loadInstants(new HoodieArchivedTimeline.ClosedClosedTimeRangeFilter(startTs, endTs), HoodieArchivedTimeline.LoadMode.METADATA); appendLoadedInstants(loadedInstants); } From 1e02ba584237110183f82d1be44c2dc664eff5b7 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Tue, 28 Jul 2026 05:13:34 -0700 Subject: [PATCH 212/255] fix(schema): gate timestamp-precision change behind a per-field verdict (#19029) Adapted for release-1.2.1: this branch keeps the pre-#19195 package layout, so the schema-internal classes live under org.apache.hudi.internal.schema rather than org.apache.hudi.common.schema.internal. Git relocated the four modified files and the new TestSchemaChangeUtils automatically; the remaining work was import paths. - BaseHoodieWriteClient, FileGroupReaderBasedMergeHandle, HoodieMergeHelper: each conflict rendered master's whole schema-internal import block, but the commit adds exactly one import to each (SchemaChangeUtils). Added it at this branch's path and left the existing imports alone. - AvroSchemaEvolutionUtils: added HoodieSchemaType, SchemaCompatibilityException and Type; HoodieCommonConfig and java.util.Map were already present here. - TestAvroSchemaEvolutionUtils: kept this branch's org.apache.hudi.avro .HoodieAvroUtils and internal.schema imports, adding only HoodieCommonConfig and SchemaCompatibilityException. - HoodieSchemaUtils.scala: added Type and SchemaChangeUtils; the other four imports in the conflict already exist here at internal.schema paths. - TestSchemaChangeUtils: package declaration corrected to org.apache.hudi.internal.schema.utils to match its relocated path. The fix itself, including the new TimestampLogicalTypeClassifier and the hoodie.write.timestamp.logical.type.overrides config, applies unchanged. (cherry picked from commit c1e2d0b0fcdf755547342e8856098080e7034e17) --- .../hudi/client/BaseHoodieWriteClient.java | 6 +- .../apache/hudi/config/HoodieWriteConfig.java | 6 + .../io/FileGroupReaderBasedMergeHandle.java | 5 +- .../action/commit/HoodieMergeHelper.java | 5 +- .../common/config/HoodieCommonConfig.java | 22 ++ .../util/TimestampLogicalTypeClassifier.java | 218 +++++++++++++++++ .../internal/schema/action/TableChanges.java | 16 +- .../utils/AvroSchemaEvolutionUtils.java | 135 ++++++++++- .../schema/utils/SchemaChangeUtils.java | 120 +++++++++- .../TestTimestampLogicalTypeClassifier.java | 143 +++++++++++ .../utils/TestAvroSchemaEvolutionUtils.java | 223 +++++++++++++++++- .../schema/utils/TestSchemaChangeUtils.java | 123 ++++++++++ .../org/apache/hudi/HoodieSchemaUtils.scala | 34 ++- .../TestHoodieDeltaStreamer.java | 97 +++++++- 14 files changed, 1114 insertions(+), 39 deletions(-) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/TimestampLogicalTypeClassifier.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestTimestampLogicalTypeClassifier.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 73f5de7b94ef4..07b08c44768fa 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -86,6 +86,7 @@ import org.apache.hudi.internal.schema.io.FileBasedInternalSchemaStorageManager; import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils; import org.apache.hudi.internal.schema.utils.InternalSchemaUtils; +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.hudi.metadata.HoodieTableMetadataUtil; @@ -369,7 +370,10 @@ private void saveInternalSchema(HoodieTable table, String instantTime, HoodieCom internalSchema = InternalSchemaUtils.searchSchema(Long.parseLong(instantTime), SerDeHelper.parseSchemas(historySchemaStr)); } - InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(schema, internalSchema, config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS)); + InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(schema, internalSchema, + config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + config.getStringOrDefault(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES))); if (evolvedSchema.equals(internalSchema)) { metadata.addMetadata(SerDeHelper.LATEST_SCHEMA, SerDeHelper.toJson(evolvedSchema)); //TODO save history schema by metaTable diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index f349f744a6ccc..317f548faaf80 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -74,6 +74,7 @@ import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; import org.apache.hudi.io.FileGroupReaderBasedMergeHandle; import org.apache.hudi.io.HoodieConcatHandle; import org.apache.hudi.keygen.SimpleAvroKeyGenerator; @@ -3864,6 +3865,11 @@ private void validate() { + "schedule inline compaction (%s) can be enabled. Both can't be set to true at the same time. %s, %s", HoodieCompactionConfig.INLINE_COMPACT.key(), HoodieCompactionConfig.SCHEDULE_INLINE_COMPACT.key(), inlineCompact, inlineCompactSchedule)); + // Parse-and-discard so a malformed 'field:type' entry fails at client build time rather + // than deep inside deduceWriterSchema on the first commit. Empty (default) is a no-op. + SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + writeConfig.getStringOrDefault(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES)); + int lookbackCommits = writeConfig.getInt(ROLLING_METADATA_TIMELINE_LOOKBACK_COMMITS); checkArgument(lookbackCommits >= 0, String.format("%s must be non-negative, but was %d", diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java index ef0d7e08dc31f..b9432b626cd51 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java @@ -51,6 +51,7 @@ import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils; +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; import org.apache.hudi.io.storage.HoodieFileWriterFactory; import org.apache.hudi.keygen.BaseKeyGenerator; @@ -258,7 +259,9 @@ public void doMerge() { boolean usePosition = config.getBooleanOrDefault(MERGE_USE_RECORD_POSITIONS); Option internalSchemaOption = SerDeHelper.fromJson(config.getInternalSchema()) .map(internalSchema -> AvroSchemaEvolutionUtils.reconcileSchema(writeSchemaWithMetaFields, internalSchema, - config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS))); + config.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + config.getStringOrDefault(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES)))); long maxMemoryPerCompaction = getMaxMemoryForMerge(); props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)); Option> logFilesStreamOpt = compactionOperation.map(op -> op.getDeltaFileNames().stream().map(logFileName -> diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java index 2f571e0d3e2ba..79bfa975e953c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java @@ -36,6 +36,7 @@ import org.apache.hudi.internal.schema.convert.InternalSchemaConverter; import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils; import org.apache.hudi.internal.schema.utils.InternalSchemaUtils; +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; import org.apache.hudi.io.HoodieWriteMergeHandle; import org.apache.hudi.io.storage.HoodieFileReader; @@ -171,7 +172,9 @@ private Option> composeSchemaEvolutionTrans if (querySchemaOpt.isPresent() && !baseFile.getBootstrapBaseFile().isPresent()) { // check implicitly add columns, and position reorder(spark sql may change cols order) InternalSchema querySchema = AvroSchemaEvolutionUtils.reconcileSchema(writerSchema, - querySchemaOpt.get(), writeConfig.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS)); + querySchemaOpt.get(), writeConfig.getBooleanOrDefault(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + writeConfig.getStringOrDefault(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES))); long commitInstantTime = Long.parseLong(baseFile.getCommitTime()); InternalSchema fileSchema = InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, metaClient); if (fileSchema.isEmptySchema() && writeConfig.getBoolean(HoodieCommonConfig.RECONCILE_SCHEMA)) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java index e488324af3412..13a3e2e6e480e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java @@ -83,6 +83,28 @@ public class HoodieCommonConfig extends HoodieConfig { + " operation will fail schema compatibility check. Set this option to true will make the missing " + " column be filled with null values to successfully complete the write operation."); + public static final ConfigProperty TIMESTAMP_LOGICAL_TYPE_OVERRIDES = ConfigProperty + .key("hoodie.write.timestamp.logical.type.overrides") + .defaultValue("") + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Per-field authority for the timestamp logical type, taking precedence over the " + + "auto-inferred schema. Comma-separated 'field:type' pairs, where type is one of timestamp-micros, " + + "timestamp-millis, local-timestamp-micros, local-timestamp-millis (case-insensitive). A field with an " + + "entry is pinned to that logical type: an incoming value of a different precision is coerced to it, and " + + "the change from the table's current type is permitted. A timestamp precision change with no entry for " + + "the field is rejected with an error, so an unverified micros/millis flip can never happen silently. " + + "An entry also attaches a local-timestamp logical type to a column that 0.x persisted as a bare long " + + "because its converter did not recognize the type. A UTC/local zone change is never authorized by " + + "this config, whatever the entry says, since no rescale can express it. " + + "Derive the value from the stored longs, never from the incoming schema: for instants after 1990 an " + + "epoch-millis value is around 1e12 while epoch-micros is around 1e15, so the two ranges do not " + + "overlap. TimestampLogicalTypeClassifier implements that verdict for inspection tooling to reuse. " + + "NOTE: this corrects the table schema only. Existing base files keep the old logical type; Hudi " + + "readers compensate for it, but external engines (Trino, Athena, BigQuery external, Spark-native " + + "parquet) keep misreading those files until they are rewritten under the corrected schema via " + + "clustering or compaction. Treat this as a one-time migration: set the override, then rewrite."); + public static final ConfigProperty SPILLABLE_DISK_MAP_TYPE = ConfigProperty .key("hoodie.common.spillable.diskmap.type") .defaultValue(ExternalSpillableMap.DiskMapType.BITCASK) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/TimestampLogicalTypeClassifier.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TimestampLogicalTypeClassifier.java new file mode 100644 index 0000000000000..efb2b6e5be2c4 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TimestampLogicalTypeClassifier.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util; + +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +/** + * Shared classifier for the timestamp logical-type drift introduced by Hudi 0.14.1 / 0.15.0 / 1.0.x. + * It decides, from three signals about a long-backed column, what the correct target timestamp + * logical type is, so a caller can suggest a value for + * {@code hoodie.write.timestamp.logical.type.overrides}. + * + *

    The three signals are: (1) the table schema logical type, (2) the base-file schema logical + * type, and (3) the shape of the raw stored {@code long} values. Signal (1) must be resolved as of + * the file's own commit instant, not the latest table schema, so that a file is classified within + * its own era. The classification here is pure; the I/O that gathers the signals is caller-specific + * (the OSS scanner reads through the storage abstraction, the data-plane tool reads parquet + * directly), but both must share this logic so their verdicts cannot drift apart. + * + *

    Status: this is the verdict logic only. The inspection scanner that samples base files + * and emits a ready-to-paste config value is not part of this repo yet, so an in-repo grep finds + * only tests today. It lives here rather than in the tool so that every consumer shares one + * definition of the verdict and they cannot drift apart. + * + *

    Do not rely on the enums or method signatures here as stable public API: this is internal to + * the timestamp-repair workflow. + */ +public class TimestampLogicalTypeClassifier { + + private TimestampLogicalTypeClassifier() { + } + + // Plausibility windows: an epoch instant in 1990-01-01 .. 2100-01-01, interpreted as millis vs + // micros. The two windows are ~1000x apart and do not overlap, so a single value fits at most one. + private static final long PLAUSIBLE_MILLIS_MIN = 631152000000L; // 1990-01-01 + private static final long PLAUSIBLE_MILLIS_MAX = 4102444800000L; // 2100-01-01 + private static final long PLAUSIBLE_MICROS_MIN = 631152000000000L; // 1990-01-01 + private static final long PLAUSIBLE_MICROS_MAX = 4102444800000000L; // 2100-01-01 + + /** The timestamp logical type of a long-backed column, or NONE / UNKNOWN. */ + public enum LogicalTimestampType { + NONE, + TIMESTAMP_MICROS, + TIMESTAMP_MILLIS, + LOCAL_TIMESTAMP_MICROS, + LOCAL_TIMESTAMP_MILLIS, + UNKNOWN + } + + /** The shape of a stored long value, judged against the plausibility windows. */ + public enum DataShape { + MICROS, + MILLIS, + AMBIGUOUS, + UNKNOWN + } + + /** The per-column verdict. */ + public enum Bucket { + /** No timestamp logical type and no timestamp-shaped data. Nothing to do, safe to upgrade. */ + UNAFFECTED, + /** Table, file, and values agree at the same precision. Correct, though it may still need a + * defensive pin if the ingestion source declares a different precision. */ + CORRECT, + /** + * Label says micros but the values are millis, or the symmetric inverse (label says millis but + * values are micros): the 0.14.1 drift. Both directions map to the same repair action — pin the + * field to whatever the values actually are — so they share this bucket. The observed + * production case is label_micros/values_millis; the symmetric case is included to be safe. + */ + LEGACY_0X_BUG, + /** Bare long, but the values are timestamp-shaped: the 0.x local-timestamp logical-type loss. */ + DROPPED_LOGICAL_TYPE, + /** The three signals disagree in some other way. */ + DIVERGENT, + /** The value shape cannot be judged confidently (near-epoch, sentinels, zeros, negatives). */ + AMBIGUOUS + } + + /** Classifies the logical type of a long-backed Avro field (the union is expected to be unwrapped). */ + public static LogicalTimestampType classifyAvroLogicalType(Schema longSchema) { + LogicalType lt = longSchema.getLogicalType(); + if (lt == null) { + return LogicalTimestampType.NONE; + } + if (lt instanceof LogicalTypes.TimestampMillis) { + return LogicalTimestampType.TIMESTAMP_MILLIS; + } + if (lt instanceof LogicalTypes.TimestampMicros) { + return LogicalTimestampType.TIMESTAMP_MICROS; + } + if (lt instanceof LogicalTypes.LocalTimestampMillis) { + return LogicalTimestampType.LOCAL_TIMESTAMP_MILLIS; + } + if (lt instanceof LogicalTypes.LocalTimestampMicros) { + return LogicalTimestampType.LOCAL_TIMESTAMP_MICROS; + } + return LogicalTimestampType.UNKNOWN; + } + + /** + * Judges a single raw long. Zeros, negatives, and sentinels (for example the year-9999 markers) + * fall outside both plausibility windows and are reported UNKNOWN so they never drive a verdict. + */ + public static DataShape classifyValueShape(long value) { + if (value <= 0) { + return DataShape.UNKNOWN; + } + boolean millisPlausible = value >= PLAUSIBLE_MILLIS_MIN && value < PLAUSIBLE_MILLIS_MAX; + boolean microsPlausible = value >= PLAUSIBLE_MICROS_MIN && value < PLAUSIBLE_MICROS_MAX; + if (millisPlausible && !microsPlausible) { + return DataShape.MILLIS; + } + if (microsPlausible && !millisPlausible) { + return DataShape.MICROS; + } + if (millisPlausible) { + // The windows do not overlap, so this is unreachable for a single value; kept for safety. + return DataShape.AMBIGUOUS; + } + return DataShape.UNKNOWN; + } + + /** Folds one sampled value's shape into the running per-column shape across many samples/files. */ + public static DataShape reduceShape(DataShape acc, DataShape sample) { + if (acc == null || acc == DataShape.UNKNOWN) { + return sample; + } + if (sample == DataShape.UNKNOWN || acc == sample) { + return acc; + } + return DataShape.AMBIGUOUS; + } + + /** + * Reconciles the three signals into a verdict. {@code tableType} must be the table logical type as + * of the inspected file's commit instant. + */ + public static Bucket classifyBucket(LogicalTimestampType tableType, LogicalTimestampType fileType, DataShape shape) { + boolean noLogicalType = tableType == LogicalTimestampType.NONE && fileType == LogicalTimestampType.NONE; + if (shape == DataShape.UNKNOWN) { + // Nothing timestamp-shaped was seen. Bare-long-everywhere is a plain non-timestamp column; + // anything else cannot be judged without a clearer value signal. + return noLogicalType ? Bucket.UNAFFECTED : Bucket.AMBIGUOUS; + } + if (shape == DataShape.AMBIGUOUS) { + return Bucket.AMBIGUOUS; + } + if (noLogicalType) { + // Bare long with timestamp-shaped data: 0.x dropped the local-timestamp logical type. + return Bucket.DROPPED_LOGICAL_TYPE; + } + boolean tableMicros = tableType == LogicalTimestampType.TIMESTAMP_MICROS || tableType == LogicalTimestampType.LOCAL_TIMESTAMP_MICROS; + boolean tableMillis = tableType == LogicalTimestampType.TIMESTAMP_MILLIS || tableType == LogicalTimestampType.LOCAL_TIMESTAMP_MILLIS; + boolean fileMicros = fileType == LogicalTimestampType.TIMESTAMP_MICROS || fileType == LogicalTimestampType.LOCAL_TIMESTAMP_MICROS; + boolean fileMillis = fileType == LogicalTimestampType.TIMESTAMP_MILLIS || fileType == LogicalTimestampType.LOCAL_TIMESTAMP_MILLIS; + if (tableMicros && fileMicros && shape == DataShape.MILLIS) { + return Bucket.LEGACY_0X_BUG; + } + if (tableMillis && fileMillis && shape == DataShape.MICROS) { + return Bucket.LEGACY_0X_BUG; + } + if (tableMicros && fileMicros && shape == DataShape.MICROS) { + // All three agree on micros. Genuinely correct; a source-declared millis is indistinguishable + // here and maps to the same action (keep micros), so it is not a separate bucket. + return Bucket.CORRECT; + } + if (tableMillis && fileMillis && shape == DataShape.MILLIS) { + return Bucket.CORRECT; + } + // Reached when the table and file logical types disagree (for example table_micros + + // file_millis) or the surviving cases where the three signals do not line up to CORRECT + // or LEGACY_0X_BUG. The operator must decide the correct override; no auto-suggestion. + return Bucket.DIVERGENT; + } + + /** + * The suggested {@code hoodie.write.timestamp.logical.type.overrides} token for a column, or empty + * when the operator must decide (ambiguous / divergent) or nothing is needed (unaffected). + * {@code local} selects the local-timestamp variant, carried from the table/file logical type. + */ + public static Option suggestedOverrideToken(Bucket bucket, DataShape shape, boolean local) { + switch (bucket) { + case CORRECT: + // Pin to the current precision so a differently-declared source cannot flip it. + return Option.of(token(shape, local)); + case LEGACY_0X_BUG: + case DROPPED_LOGICAL_TYPE: + // Repair to what the values actually are. + return Option.of(token(shape, local)); + default: + return Option.empty(); + } + } + + private static String token(DataShape shape, boolean local) { + String precision = shape == DataShape.MILLIS ? "millis" : "micros"; + return (local ? "local-timestamp-" : "timestamp-") + precision; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java index 05d9e0bfa3f34..17e4517b5fbd5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java @@ -47,13 +47,11 @@ public static class ColumnUpdateChange extends TableChange.BaseColumnChange { @Getter private final Map updates = new HashMap<>(); + private final boolean allowTimestampPrecisionEvolution; - private ColumnUpdateChange(InternalSchema schema) { - super(schema, false); - } - - private ColumnUpdateChange(InternalSchema schema, boolean caseSensitive) { + private ColumnUpdateChange(InternalSchema schema, boolean caseSensitive, boolean allowTimestampPrecisionEvolution) { super(schema, caseSensitive); + this.allowTimestampPrecisionEvolution = allowTimestampPrecisionEvolution; } @Override @@ -96,7 +94,7 @@ public ColumnUpdateChange updateColumnType(String name, Type newType) { throw new SchemaCompatibilityException(String.format("Cannot update type for column '%s' because it does not exist in the schema", name)); } - if (!SchemaChangeUtils.isTypeUpdateAllow(field.type(), newType)) { + if (!SchemaChangeUtils.isTypeUpdateAllow(field.type(), newType, allowTimestampPrecisionEvolution)) { throw new SchemaCompatibilityException(String.format( "Cannot update column '%s' from type '%s' to incompatible type '%s'.", name, field.type(), newType)); } @@ -232,11 +230,11 @@ protected Integer findIdByFullName(String fullName) { } public static ColumnUpdateChange get(InternalSchema schema) { - return new ColumnUpdateChange(schema); + return new ColumnUpdateChange(schema, false, false); } - public static ColumnUpdateChange get(InternalSchema schema, boolean caseSensitive) { - return new ColumnUpdateChange(schema, caseSensitive); + public static ColumnUpdateChange get(InternalSchema schema, boolean caseSensitive, boolean allowTimestampPrecisionEvolution) { + return new ColumnUpdateChange(schema, caseSensitive, allowTimestampPrecisionEvolution); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java index 6ed1722589fb3..fd6084b201a34 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java @@ -18,9 +18,13 @@ package org.apache.hudi.internal.schema.utils; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.exception.SchemaCompatibilityException; import org.apache.hudi.internal.schema.InternalSchema; +import org.apache.hudi.internal.schema.Type; import org.apache.hudi.internal.schema.action.TableChanges; import org.apache.hudi.internal.schema.action.TableChangesHelper; @@ -29,6 +33,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -62,7 +67,8 @@ public class AvroSchemaEvolutionUtils { * nullable in the result. Otherwise, no updates will be made to those fields. * @return reconcile Schema */ - public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, InternalSchema oldTableSchema, boolean makeMissingFieldsNullable) { + public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, InternalSchema oldTableSchema, + boolean makeMissingFieldsNullable, Map timestampLogicalTypeOverrides) { /* If incoming schema is null, we fall back on table schema. */ if (incomingSchema.isSchemaNull()) { return oldTableSchema; @@ -129,9 +135,22 @@ public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, Intern // do type evolution. InternalSchema internalSchemaAfterAddColumns = SchemaChangeUtils.applyTableChanges2Schema(oldTableSchema, addChange); - TableChanges.ColumnUpdateChange typeChange = TableChanges.ColumnUpdateChange.get(internalSchemaAfterAddColumns); + // The reconcile pre-validates timestamp precision changes per field below (against the explicit + // overrides), so the update change is constructed permissively; non-overridden precision changes + // are rejected here with an actionable error rather than deferred to the gate. + TableChanges.ColumnUpdateChange typeChange = TableChanges.ColumnUpdateChange.get( + internalSchemaAfterAddColumns, false, true); typeChangeColumns.stream().filter(f -> !inComingInternalSchema.findType(f).isNestedType()).forEach(col -> { - typeChange.updateColumnType(col, inComingInternalSchema.findType(col)); + Type tableType = oldTableSchema.findType(col); + Type incomingType = inComingInternalSchema.findType(col); + if (SchemaChangeUtils.isGatedTimestampChange(tableType, incomingType)) { + // Skip-if equals the *table* type: the reconcile is producing the new table schema starting + // from oldTableSchema, so a coerce-to-table-precision override needs no schema update — the + // writer coerces incoming values via rewriteRecordWithNewSchema. + applyTimestampOverrideOrThrow(col, tableType, incomingType, timestampLogicalTypeOverrides, tableType, typeChange); + } else { + typeChange.updateColumnType(col, incomingType); + } }); // relax existing columns to nullable when the incoming schema made them nullable (valid widening) @@ -162,8 +181,114 @@ public static InternalSchema reconcileSchema(HoodieSchema incomingSchema, Intern return evolvedSchema; } - public static HoodieSchema reconcileSchema(HoodieSchema incomingSchema, HoodieSchema oldTableSchema, boolean makeMissingFieldsNullable) { - return convert(reconcileSchema(incomingSchema, convert(oldTableSchema), makeMissingFieldsNullable), oldTableSchema.getFullName()); + public static HoodieSchema reconcileSchema(HoodieSchema incomingSchema, HoodieSchema oldTableSchema, boolean makeMissingFieldsNullable, + Map timestampLogicalTypeOverrides) { + return convert(reconcileSchema(incomingSchema, convert(oldTableSchema), makeMissingFieldsNullable, timestampLogicalTypeOverrides), oldTableSchema.getFullName()); + } + + /** + * Reconciles only the timestamp logical-type precision of {@code writerSchema} against + * {@code tableSchema}, independent of column add/drop/nullability reconciliation. This is the + * single guard that every writer-schema deduction path must apply, including the non-reconcile + * paths that otherwise validate via the logical-type-blind Avro reader/writer compatibility check + * and would let an unverified micros/millis flip through silently. + * + *

    For each field whose precision differs from the table: an override pins it (equal to the + * table type coerces the incoming values, a different type applies the authorized evolution), and + * a change with no override throws. A UTC/local zone change throws unconditionally, since no + * override authorizes one. Non-timestamp changes are left untouched here. + */ + public static HoodieSchema reconcileTimestampLogicalType(HoodieSchema writerSchema, HoodieSchema tableSchema, + Map timestampLogicalTypeOverrides) { + if (writerSchema == null || writerSchema.getType() != HoodieSchemaType.RECORD + || tableSchema == null || tableSchema.getType() != HoodieSchemaType.RECORD) { + return writerSchema; + } + InternalSchema writerInternal = convert(writerSchema); + InternalSchema tableInternal = convert(tableSchema); + List tableCols = tableInternal.getAllColsFullName(); + TableChanges.ColumnUpdateChange typeChange = TableChanges.ColumnUpdateChange.get(writerInternal, false, true); + boolean changed = false; + for (String col : writerInternal.getAllColsFullName()) { + if (!tableCols.contains(col)) { + continue; + } + Type writerType = writerInternal.findType(col); + Type tableType = tableInternal.findType(col); + if (writerType.isNestedType()) { + continue; + } + // A zone change is never authorizable, and this is the only guard on the default + // non-reconcile path -- the Avro reader/writer check that follows is logical-type-blind for + // two long-backed fields, so skipping here would let the flip through silently. + if (SchemaChangeUtils.isCrossZoneTimestampChange(tableType, writerType)) { + throw crossZoneTimestampChangeError(col, tableType, writerType); + } + if (!SchemaChangeUtils.isGatedTimestampChange(tableType, writerType)) { + continue; + } + // Skip-if equals the *writer* type: this method returns a modified writerSchema. When the + // override already matches the writer field, the writer schema is what we want; no update. + if (applyTimestampOverrideOrThrow(col, tableType, writerType, timestampLogicalTypeOverrides, writerType, typeChange)) { + changed = true; + } + } + if (!changed) { + return writerSchema; + } + return convert(SchemaChangeUtils.applyTableChanges2Schema(writerInternal, typeChange), writerSchema.getFullName()); + } + + /** + * Shared override-apply for a single field whose type is a gated timestamp precision change. + * Called from both {@link #reconcileSchema} and {@link #reconcileTimestampLogicalType} — those + * two paths differ only in which "current" schema they compare the override against (the table + * type vs. the writer type), so the caller passes that in as {@code skipIfEquals}. + * + * @param col the fully-qualified column name (for the error message) + * @param tableType the table's current type (for the error message) + * @param incomingType the writer/incoming type (for the error message) + * @param overrides the parsed per-field overrides map + * @param skipIfEquals compare the override against this; no schema update when equal + * @param typeChange the accumulator for schema updates + * @return {@code true} if the override was applied (schema will change), {@code false} otherwise + * @throws SchemaCompatibilityException when no override is present for this gated change + */ + private static boolean applyTimestampOverrideOrThrow(String col, Type tableType, Type incomingType, + Map overrides, Type skipIfEquals, + TableChanges.ColumnUpdateChange typeChange) { + Type overrideType = overrides.get(col); + if (overrideType == null) { + throw timestampPrecisionChangeError(col, tableType, incomingType); + } + if (overrideType.equals(skipIfEquals)) { + return false; + } + typeChange.updateColumnType(col, overrideType); + return true; + } + + private static SchemaCompatibilityException crossZoneTimestampChangeError(String col, Type from, Type to) { + return new SchemaCompatibilityException(String.format( + "Refusing to change the timestamp logical type of column '%s' from '%s' to '%s': this crosses the " + + "UTC/local boundary, which changes the instant the stored value denotes and cannot be repaired by " + + "rescaling. '%s' authorizes precision changes only, never a zone change. Keep writing the column " + + "with its existing zone, or add a new column and backfill it with an explicit conversion.", + col, from, to, HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key())); + } + + private static SchemaCompatibilityException timestampPrecisionChangeError(String col, Type from, Type to) { + return new SchemaCompatibilityException(String.format( + "Refusing to change the timestamp logical type of column '%s' from '%s' to '%s' without an explicit " + + "verdict. This precision change is not applied automatically because the correct target depends " + + "on the stored values, not the incoming schema. Inspect the raw long values of '%s' in the existing " + + "base files: for instants after 1990 epoch-millis is around 1e12 and epoch-micros is around 1e15, " + + "so the ranges do not overlap (TimestampLogicalTypeClassifier implements this verdict). Then set " + + "'%s' to the precision the values actually are, for example '%s:timestamp-micros' to keep the " + + "current precision and coerce the incoming values, or '%s:timestamp-millis' to evolve the column. " + + "Existing base files are not rewritten by this change; rewrite them via clustering or compaction " + + "so that non-Hudi readers also see the corrected type.", + col, from, to, col, HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key(), col, col)); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java index f02407b986ed7..e63165a1a295a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java @@ -28,7 +28,11 @@ import lombok.NoArgsConstructor; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; /** * Helper methods for schema Change. @@ -36,6 +40,96 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class SchemaChangeUtils { + /** + * Parses the {@code hoodie.write.timestamp.logical.type.overrides} value into a per-field map of + * the target timestamp {@link Type}. The value is a comma-separated list of {@code field:type} + * pairs, where type is one of timestamp-micros, timestamp-millis, local-timestamp-micros, + * local-timestamp-millis (case-insensitive). The tokens are a Hudi-owned vocabulary decoupled + * from any serialization format. + * + *

    Splits on the last {@code ':'} so dotted nested field names ({@code parent.child}) work + * unchanged. Field names containing a literal {@code ':'} are not supported. + * + * @param value the raw config value (may be null or empty) + * @return an unmodifiable map from field name to the pinned timestamp type; empty if unset + */ + public static Map parseTimestampLogicalTypeOverrides(String value) { + if (value == null || value.trim().isEmpty()) { + return Collections.emptyMap(); + } + Map result = new LinkedHashMap<>(); + for (String pair : value.split(",")) { + String trimmed = pair.trim(); + if (trimmed.isEmpty()) { + continue; + } + int sep = trimmed.lastIndexOf(':'); + if (sep <= 0 || sep == trimmed.length() - 1) { + throw new IllegalArgumentException("Invalid timestamp logical type override entry '" + trimmed + + "'. Expected 'field:type' where type is one of timestamp-micros, timestamp-millis, " + + "local-timestamp-micros, local-timestamp-millis."); + } + String field = trimmed.substring(0, sep).trim(); + Type type = timestampTypeFromToken(trimmed.substring(sep + 1).trim()); + result.put(field, type); + } + return Collections.unmodifiableMap(result); + } + + private static Type timestampTypeFromToken(String token) { + switch (token.toLowerCase(Locale.ROOT)) { + case "timestamp-micros": + return Types.TimestampType.get(); + case "timestamp-millis": + return Types.TimestampMillisType.get(); + case "local-timestamp-micros": + return Types.LocalTimestampMicrosType.get(); + case "local-timestamp-millis": + return Types.LocalTimestampMillisType.get(); + default: + throw new IllegalArgumentException("Unknown timestamp logical type token '" + token + + "'. Expected one of timestamp-micros, timestamp-millis, local-timestamp-micros, " + + "local-timestamp-millis."); + } + } + + /** + * Whether a column type change is a timestamp precision change that must be authorized by an + * explicit per-field override (see {@code hoodie.write.timestamp.logical.type.overrides}). This + * covers timestamp-micros/millis flips, local-timestamp-micros/millis flips, and the forward-fix + * from a bare {@code long} to a local-timestamp logical type that 0.x dropped. + */ + public static boolean isGatedTimestampChange(Type src, Type dst) { + if (src.equals(dst)) { + return false; + } + if (isUtcTimestamp(src) && isUtcTimestamp(dst)) { + return true; + } + if (isLocalTimestamp(src) && isLocalTimestamp(dst)) { + return true; + } + return src.typeId() == Type.TypeID.LONG && isLocalTimestamp(dst); + } + + /** + * Whether a column type change crosses the UTC/local timestamp boundary. Unlike a precision + * change this has no value-level repair: the same long denotes a different instant under each + * interpretation, so rescaling cannot express the conversion. A zone change is therefore always + * rejected and no per-field override authorizes it. + */ + public static boolean isCrossZoneTimestampChange(Type src, Type dst) { + return (isUtcTimestamp(src) && isLocalTimestamp(dst)) || (isLocalTimestamp(src) && isUtcTimestamp(dst)); + } + + private static boolean isUtcTimestamp(Type type) { + return type.typeId() == Type.TypeID.TIMESTAMP || type.typeId() == Type.TypeID.TIMESTAMP_MILLIS; + } + + private static boolean isLocalTimestamp(Type type) { + return type.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MILLIS || type.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MICROS; + } + /** * Whether to allow the column type to be updated. * now only support: @@ -52,29 +146,35 @@ public class SchemaChangeUtils { * @param dst new column type. * @return whether to allow the column type to be updated. */ - public static boolean isTypeUpdateAllow(Type src, Type dst) { + public static boolean isTypeUpdateAllow(Type src, Type dst, boolean allowTimestampPrecisionEvolution) { if (src.isNestedType() || dst.isNestedType()) { throw new IllegalArgumentException("only support update primitive type"); } if (src.equals(dst)) { return true; } - return isTypeUpdateAllowInternal(src, dst); + return isTypeUpdateAllowInternal(src, dst, allowTimestampPrecisionEvolution); } public static boolean shouldPromoteType(Type src, Type dst) { if (src.equals(dst) || src.isNestedType() || dst.isNestedType()) { return false; } - return isTypeUpdateAllowInternal(src, dst); + return isTypeUpdateAllowInternal(src, dst, false); } - private static boolean isTypeUpdateAllowInternal(Type src, Type dst) { + private static boolean isTypeUpdateAllowInternal(Type src, Type dst, boolean allowTimestampPrecisionEvolution) { switch (src.typeId()) { case INT: return dst == Types.LongType.get() || dst == Types.FloatType.get() || dst == Types.DoubleType.get() || dst == Types.StringType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED; case LONG: + if (allowTimestampPrecisionEvolution + && (dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MILLIS || dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MICROS)) { + // Forward-fix path: 0.x stored local-timestamp columns as bare long because its converter + // did not recognize the logical type. Allow attaching the logical type when the gate is open. + return true; + } return dst == Types.FloatType.get() || dst == Types.DoubleType.get() || dst == Types.StringType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED; case FLOAT: return dst == Types.DoubleType.get() || dst == Types.StringType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED; @@ -90,6 +190,18 @@ private static boolean isTypeUpdateAllowInternal(Type src, Type dst) { return isDecimalFixedUpdateAllowInternal(src, dst); case STRING: return dst == Types.DateType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED || dst == Types.BinaryType.get(); + case TIMESTAMP: + case TIMESTAMP_MILLIS: + if (!allowTimestampPrecisionEvolution) { + return false; + } + return dst.typeId() == Type.TypeID.TIMESTAMP || dst.typeId() == Type.TypeID.TIMESTAMP_MILLIS; + case LOCAL_TIMESTAMP_MILLIS: + case LOCAL_TIMESTAMP_MICROS: + if (!allowTimestampPrecisionEvolution) { + return false; + } + return dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MILLIS || dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MICROS; default: return false; } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTimestampLogicalTypeClassifier.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTimestampLogicalTypeClassifier.java new file mode 100644 index 0000000000000..1a81fa2cb338d --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTimestampLogicalTypeClassifier.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util; + +import org.apache.hudi.common.util.TimestampLogicalTypeClassifier.Bucket; +import org.apache.hudi.common.util.TimestampLogicalTypeClassifier.DataShape; + +import org.junit.jupiter.api.Test; + +import static org.apache.hudi.common.util.TimestampLogicalTypeClassifier.LogicalTimestampType.LOCAL_TIMESTAMP_MICROS; +import static org.apache.hudi.common.util.TimestampLogicalTypeClassifier.LogicalTimestampType.LOCAL_TIMESTAMP_MILLIS; +import static org.apache.hudi.common.util.TimestampLogicalTypeClassifier.LogicalTimestampType.NONE; +import static org.apache.hudi.common.util.TimestampLogicalTypeClassifier.LogicalTimestampType.TIMESTAMP_MICROS; +import static org.apache.hudi.common.util.TimestampLogicalTypeClassifier.LogicalTimestampType.TIMESTAMP_MILLIS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Tests {@link TimestampLogicalTypeClassifier}. + */ +public class TestTimestampLogicalTypeClassifier { + + // 2025-06-01 as millis (~13 digits) and micros (~16 digits). + private static final long MILLIS_2025 = 1748736000000L; + private static final long MICROS_2025 = 1748736000000000L; + // The year-9999 micros sentinel seen on real tables. + private static final long YEAR_9999_MICROS = 253402214400000000L; + + @Test + public void testValueShape() { + assertEquals(DataShape.MILLIS, TimestampLogicalTypeClassifier.classifyValueShape(MILLIS_2025)); + assertEquals(DataShape.MICROS, TimestampLogicalTypeClassifier.classifyValueShape(MICROS_2025)); + // Zero, negative, near-epoch, and sentinels are not judgeable. + assertEquals(DataShape.UNKNOWN, TimestampLogicalTypeClassifier.classifyValueShape(0L)); + assertEquals(DataShape.UNKNOWN, TimestampLogicalTypeClassifier.classifyValueShape(-1L)); + assertEquals(DataShape.UNKNOWN, TimestampLogicalTypeClassifier.classifyValueShape(1000L)); + assertEquals(DataShape.UNKNOWN, TimestampLogicalTypeClassifier.classifyValueShape(YEAR_9999_MICROS)); + } + + @Test + public void testReduceShape() { + assertEquals(DataShape.MICROS, TimestampLogicalTypeClassifier.reduceShape(null, DataShape.MICROS)); + assertEquals(DataShape.MICROS, TimestampLogicalTypeClassifier.reduceShape(DataShape.MICROS, DataShape.MICROS)); + // UNKNOWN samples do not pollute a settled shape. + assertEquals(DataShape.MICROS, TimestampLogicalTypeClassifier.reduceShape(DataShape.MICROS, DataShape.UNKNOWN)); + assertEquals(DataShape.MICROS, TimestampLogicalTypeClassifier.reduceShape(DataShape.UNKNOWN, DataShape.MICROS)); + // Genuinely mixed precision across files (for example a wrongly flipped table) surfaces as ambiguous. + assertEquals(DataShape.AMBIGUOUS, TimestampLogicalTypeClassifier.reduceShape(DataShape.MICROS, DataShape.MILLIS)); + } + + @Test + public void testBuckets() { + // Genuinely correct micros (the Apna case): all three signals agree. + assertEquals(Bucket.CORRECT, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MICROS, TIMESTAMP_MICROS, DataShape.MICROS)); + // Legit millis. + assertEquals(Bucket.CORRECT, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MILLIS, DataShape.MILLIS)); + // The 0.14.1 drift: label micros, values millis. + assertEquals(Bucket.LEGACY_0X_BUG, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MICROS, TIMESTAMP_MICROS, DataShape.MILLIS)); + // Symmetric inverse. + assertEquals(Bucket.LEGACY_0X_BUG, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MILLIS, DataShape.MICROS)); + // Dropped logical type: bare long, timestamp-shaped values. + assertEquals(Bucket.DROPPED_LOGICAL_TYPE, TimestampLogicalTypeClassifier.classifyBucket(NONE, NONE, DataShape.MICROS)); + // No logical type and no timestamp-shaped data: not a timestamp column at all. + assertEquals(Bucket.UNAFFECTED, TimestampLogicalTypeClassifier.classifyBucket(NONE, NONE, DataShape.UNKNOWN)); + // A labeled timestamp column with an unjudgeable value shape cannot be classified. + assertEquals(Bucket.AMBIGUOUS, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MICROS, TIMESTAMP_MICROS, DataShape.UNKNOWN)); + assertEquals(Bucket.AMBIGUOUS, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MICROS, TIMESTAMP_MICROS, DataShape.AMBIGUOUS)); + // Table and file disagree with no clean repair reading. + assertEquals(Bucket.DIVERGENT, TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MICROS, TIMESTAMP_MILLIS, DataShape.MICROS)); + } + + @Test + public void testSuggestedOverrideToken() { + assertEquals("timestamp-millis", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.LEGACY_0X_BUG, DataShape.MILLIS, false).get()); + assertEquals("timestamp-micros", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.CORRECT, DataShape.MICROS, false).get()); + assertEquals("local-timestamp-micros", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.DROPPED_LOGICAL_TYPE, DataShape.MICROS, true).get()); + assertEquals("local-timestamp-millis", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.DROPPED_LOGICAL_TYPE, DataShape.MILLIS, true).get()); + // Millis-side symmetry: DROPPED with millis data pins to timestamp-millis / local-timestamp-millis. + assertEquals("timestamp-millis", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.DROPPED_LOGICAL_TYPE, DataShape.MILLIS, false).get()); + assertEquals("timestamp-millis", + TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.CORRECT, DataShape.MILLIS, false).get()); + // Ambiguous / divergent / unaffected: no automatic suggestion. + assertFalse(TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.AMBIGUOUS, DataShape.UNKNOWN, false).isPresent()); + assertFalse(TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.UNAFFECTED, DataShape.UNKNOWN, false).isPresent()); + assertFalse(TimestampLogicalTypeClassifier.suggestedOverrideToken(Bucket.DIVERGENT, DataShape.MICROS, false).isPresent()); + } + + @Test + public void testBucketsMillisSideSymmetry() { + // Mirror the micros cases in testBuckets() with the millis side. The symmetric coverage guards + // against a future edit accidentally handling only one direction. + assertEquals(Bucket.CORRECT, + TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MILLIS, DataShape.MILLIS)); + // 0.14.1 drift on the millis side: label millis, values micros. + assertEquals(Bucket.LEGACY_0X_BUG, + TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MILLIS, DataShape.MICROS)); + // A millis-labeled column with unjudgeable value shape. + assertEquals(Bucket.AMBIGUOUS, + TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MILLIS, DataShape.UNKNOWN)); + // Table + file disagree in the reverse direction — falls through to DIVERGENT. + assertEquals(Bucket.DIVERGENT, + TimestampLogicalTypeClassifier.classifyBucket(TIMESTAMP_MILLIS, TIMESTAMP_MICROS, DataShape.MILLIS)); + // Dropped local-timestamp-millis: bare long everywhere, values millis. Covers the 0.x drop of + // local-timestamp logical types, millis side. + assertEquals(Bucket.DROPPED_LOGICAL_TYPE, + TimestampLogicalTypeClassifier.classifyBucket(NONE, NONE, DataShape.MILLIS)); + } + + @Test + public void testBucketsLocalTimestampVariants() { + // Local-timestamp variants must classify identically to their non-local counterparts — + // the bug happens against the same three signals, only the resulting override token differs. + assertEquals(Bucket.CORRECT, + TimestampLogicalTypeClassifier.classifyBucket(LOCAL_TIMESTAMP_MICROS, LOCAL_TIMESTAMP_MICROS, DataShape.MICROS)); + assertEquals(Bucket.CORRECT, + TimestampLogicalTypeClassifier.classifyBucket(LOCAL_TIMESTAMP_MILLIS, LOCAL_TIMESTAMP_MILLIS, DataShape.MILLIS)); + assertEquals(Bucket.LEGACY_0X_BUG, + TimestampLogicalTypeClassifier.classifyBucket(LOCAL_TIMESTAMP_MICROS, LOCAL_TIMESTAMP_MICROS, DataShape.MILLIS)); + assertEquals(Bucket.LEGACY_0X_BUG, + TimestampLogicalTypeClassifier.classifyBucket(LOCAL_TIMESTAMP_MILLIS, LOCAL_TIMESTAMP_MILLIS, DataShape.MICROS)); + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java index 08e425093974f..b4e476ce1e07e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java @@ -19,12 +19,14 @@ package org.apache.hudi.internal.schema.utils; import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.schema.HoodieJsonProperties; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.exception.HoodieNullSchemaTypeException; +import org.apache.hudi.exception.SchemaCompatibilityException; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.InternalSchemaBuilder; import org.apache.hudi.internal.schema.Type; @@ -486,7 +488,8 @@ public void testEvolutionSchemaFromNewAvroSchema() { ); evolvedRecord = (Types.RecordType)InternalSchemaBuilder.getBuilder().refreshNewId(evolvedRecord, new AtomicInteger(0)); HoodieSchema evolvedSchema = InternalSchemaConverter.convert(evolvedRecord, "test1"); - InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(evolvedSchema, oldSchema, false); + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(evolvedSchema, oldSchema, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")); Types.RecordType checkedRecord = Types.RecordType.get( Types.Field.get(0, false, "id", Types.IntType.get()), Types.Field.get(1, true, "data", Types.StringType.get()), @@ -541,7 +544,8 @@ public void testReconcileSchema() { + "{\"name\":\"d2\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null}]}"); HoodieSchema simpleReconcileSchema = InternalSchemaConverter.convert(AvroSchemaEvolutionUtils - .reconcileSchema(incomingSchema, InternalSchemaConverter.convert(schema), false), "schemaNameFallback"); + .reconcileSchema(incomingSchema, InternalSchemaConverter.convert(schema), false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")), "schemaNameFallback"); Assertions.assertEquals(simpleCheckSchema, simpleReconcileSchema); } @@ -563,7 +567,8 @@ public void testNotEvolveSchemaIfReconciledSchemaUnchanged() { InternalSchema oldInternalSchema = InternalSchemaConverter.convert(oldSchema); // set a non-default schema id for old table schema, e.g., 2. oldInternalSchema.setSchemaId(2); - InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldInternalSchema, false); + InternalSchema evolvedSchema = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldInternalSchema, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")); // the evolved schema should be the old table schema, since there is no type change at all. Assertions.assertEquals(oldInternalSchema, evolvedSchema); } @@ -590,7 +595,8 @@ public void testReconcileSchemaRelaxesExistingColumnToNullable() { incomingRecord = (Types.RecordType) InternalSchemaBuilder.getBuilder().refreshNewId(incomingRecord, new AtomicInteger(0)); HoodieSchema incomingSchema = InternalSchemaConverter.convert(incomingRecord, "test1"); - InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true); + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")); Types.RecordType checkedRecord = Types.RecordType.get( Types.Field.get(0, false, "id", Types.IntType.get()), @@ -619,7 +625,8 @@ public void testReconcileSchemaDoesNotTightenNullableToRequired() { incomingRecord = (Types.RecordType) InternalSchemaBuilder.getBuilder().refreshNewId(incomingRecord, new AtomicInteger(0)); HoodieSchema incomingSchema = InternalSchemaConverter.convert(incomingRecord, "test1"); - InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true); + InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchema, oldSchema, true, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")); Types.RecordType checkedRecord = Types.RecordType.get( Types.Field.get(0, false, "id", Types.IntType.get()), @@ -627,4 +634,210 @@ public void testReconcileSchemaDoesNotTightenNullableToRequired() { ); Assertions.assertEquals(checkedRecord, result.getRecord()); } + + private static Schema tripAvro(Schema tsType) { + return Schema.createRecord("trip", null, null, false, Arrays.asList( + new Schema.Field("id", Schema.create(Schema.Type.STRING), null, null), + new Schema.Field("ts", tsType, null, null))); + } + + @Test + public void testReconcileSchemaTimestampPrecisionEvolution() { + // A timestamp precision change is rejected unless the field has an explicit override in + // hoodie.write.timestamp.logical.type.overrides. The override pins the field: an entry equal to + // the table type coerces the incoming values and keeps the table precision, while a different + // entry evolves the column. No entry throws, so an unverified micros/millis flip cannot happen. + HoodieSchema tableSchemaMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema incomingSchemaMillis = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + + // Guard: with no override, the precision change is rejected in either direction with an + // actionable error that names the column and the config to set. + Throwable rejectedMicrosToMillis = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incomingSchemaMillis, tableSchemaMicros, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + assertTrue(rejectedMicrosToMillis.getMessage().contains("without an explicit")); + assertTrue(rejectedMicrosToMillis.getMessage().contains(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key())); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(tableSchemaMicros, incomingSchemaMillis, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + + // Override to millis: the micros table evolves to millis (the genuine-repair case). + Schema evolvedToMillis = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchemaMillis, tableSchemaMicros, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("timestamp-millis", evolvedToMillis.getField("ts").schema().getLogicalType().getName()); + + // Override to micros with a millis source (the Apna case): the table stays micros, no flip; the + // incoming millis values are coerced to micros on write. + Schema pinnedToMicros = AvroSchemaEvolutionUtils.reconcileSchema(incomingSchemaMillis, tableSchemaMicros, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + Assertions.assertEquals("timestamp-micros", pinnedToMicros.getField("ts").schema().getLogicalType().getName()); + + // Override to micros against a millis table: the reverse evolution is permitted. + Schema evolvedToMicros = AvroSchemaEvolutionUtils.reconcileSchema(tableSchemaMicros, incomingSchemaMillis, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + Assertions.assertEquals("timestamp-micros", evolvedToMicros.getField("ts").schema().getLogicalType().getName()); + + // The same override applies to the local-timestamp variants. + HoodieSchema tableLocalMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema incomingLocalMillis = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMillis, tableLocalMicros, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + Schema reconciledLocal = AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMillis, tableLocalMicros, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("local-timestamp-millis", reconciledLocal.getField("ts").schema().getLogicalType().getName()); + + // 0.x did not recognize the local-timestamp logical types, so affected tables persisted those + // columns as bare long. The override must also allow attaching the logical type on forward-fix. + HoodieSchema tableBareLong = HoodieSchema.fromAvroSchema(tripAvro(Schema.create(Schema.Type.LONG))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMillis, tableBareLong, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + Schema repairedToLocalMillis = AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMillis, tableBareLong, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("local-timestamp-millis", repairedToLocalMillis.getField("ts").schema().getLogicalType().getName()); + + HoodieSchema incomingLocalMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + Schema repairedToLocalMicros = AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMicros, tableBareLong, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-micros")).toAvroSchema(); + Assertions.assertEquals("local-timestamp-micros", repairedToLocalMicros.getField("ts").schema().getLogicalType().getName()); + } + + @Test + public void testReconcileTimestampLogicalTypeGuardsNonReconcilePath() { + // reconcileTimestampLogicalType is the guard applied to the deduced writer schema on every path, + // including the default set.null=false path whose Avro compatibility check is logical-type-blind. + HoodieSchema tableMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema writerMillis = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + + // Guard: no override and the precision differs, so the flip is rejected instead of silently applied. + Throwable rejected = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(writerMillis, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + assertTrue(rejected.getMessage().contains("without an explicit")); + assertTrue(rejected.getMessage().contains("'ts'")); + + // Override to micros coerces the millis writer back to micros (no flip, the Apna case). + Schema coerced = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(writerMillis, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + Assertions.assertEquals("timestamp-micros", coerced.getField("ts").schema().getLogicalType().getName()); + + // Override to millis keeps the writer at millis (authorized evolution). + Schema evolved = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(writerMillis, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("timestamp-millis", evolved.getField("ts").schema().getLogicalType().getName()); + + // No precision difference: returned unchanged, no override required and no throw. + Schema unchanged = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(tableMicros, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("")).toAvroSchema(); + Assertions.assertEquals("timestamp-micros", unchanged.getField("ts").schema().getLogicalType().getName()); + } + + /** + * End-to-end value assertion on the coerce/pin path — the Apna case. Source declares + * timestamp-millis, table is timestamp-micros, override pins the field to the table's micros + * type. The reconcile flips the writer schema back to micros. When a record whose source Avro + * schema declared millis is rewritten to the (now-micros) writer schema, the long must still be + * rescaled by 1000 — not left as-is because writer == table. + * + *

    The prior boolean flag would have flipped the table to millis without touching values, + * causing the "reads as year 58466" failure. This test guards that value-level behavior directly. + */ + @Test + public void testReconcileTimestampLogicalTypeCoercesValuesOnPin() { + HoodieSchema tableMicrosSchema = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + Schema sourceMillisSchema = tripAvro(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))); + + // Driver-plan step: apply the guard with the coerce override. The writer schema for `ts` + // should be pinned back to timestamp-micros (matching the table), not left as millis. + Schema writerSchema = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType( + HoodieSchema.fromAvroSchema(sourceMillisSchema), tableMicrosSchema, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + Assertions.assertEquals("timestamp-micros", writerSchema.getField("ts").schema().getLogicalType().getName()); + + // Executor step: an incoming record carrying the SOURCE schema (millis) is rewritten to the + // deduced WRITER schema (micros). rewriteRecordWithNewSchema must invoke the x1000 rescale so + // 2024-01-01T00:00:00Z millis (1704067200000L) becomes the equivalent micros + // (1704067200000000L) — not the same long reinterpreted, which would read as year 55965. + long millisValue = 1704067200000L; // 2024-01-01T00:00:00Z as epoch millis + long expectedMicros = 1704067200000000L; // same instant as epoch micros + GenericRecord sourceRecord = new GenericData.Record(sourceMillisSchema); + sourceRecord.put("id", "row-1"); + sourceRecord.put("ts", millisValue); + GenericRecord rewritten = HoodieAvroUtils.rewriteRecordWithNewSchema(sourceRecord, writerSchema); + Assertions.assertEquals(expectedMicros, rewritten.get("ts"), + "millis source value must be rescaled to micros when the writer schema is pinned to micros"); + + // Symmetric coverage: source declares micros, table is millis, override pins to millis. + // Rewrite must divide by 1000 (integer division). Pick a value that is exact. + HoodieSchema tableMillisSchema = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + Schema sourceMicrosSchema = tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG))); + Schema writerSchemaMillis = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType( + HoodieSchema.fromAvroSchema(sourceMicrosSchema), tableMillisSchema, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("timestamp-millis", writerSchemaMillis.getField("ts").schema().getLogicalType().getName()); + GenericRecord sourceMicros = new GenericData.Record(sourceMicrosSchema); + sourceMicros.put("id", "row-2"); + sourceMicros.put("ts", expectedMicros); + GenericRecord rewrittenMillis = HoodieAvroUtils.rewriteRecordWithNewSchema(sourceMicros, writerSchemaMillis); + Assertions.assertEquals(millisValue, rewrittenMillis.get("ts"), + "micros source value must be rescaled to millis when the writer schema is pinned to millis"); + } + + /** + * A UTC/local zone change is not a precision repair. The stored long means a different instant + * under each interpretation and no rescale can fix that, so a zone change must be rejected on + * every path and no per-field override may authorize it. + * + *

    Both entry points have to enforce it. reconcileSchema rejects via isTypeUpdateAllow, but + * reconcileTimestampLogicalType is the only guard on the default non-reconcile path, and the + * Avro reader/writer compatibility check that runs after it is logical-type-blind for two + * long-backed fields -- so if the guard skips a zone change, nothing else catches it. + */ + @Test + public void testCrossZoneTimestampChangeIsRejected() { + HoodieSchema tableMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema localMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema tableMillis = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + HoodieSchema localMillis = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))); + + // No override: rejected by both entry points, in both zone directions. + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(localMicros, InternalSchemaConverter.convert(tableMicros), false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(tableMicros, InternalSchemaConverter.convert(localMicros), false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + Throwable guarded = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMicros, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + assertTrue(guarded.getMessage().contains("'ts'"), "Unexpected message: " + guarded.getMessage()); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(tableMicros, localMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + + // An override must NOT unlock a zone change, whichever zone it names. + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMicros, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-micros"))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMicros, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros"))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(localMicros, InternalSchemaConverter.convert(tableMicros), false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-micros"))); + + // A zone change that also crosses precision is still a zone change. + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMillis, tableMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-millis"))); + assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMicros, tableMillis, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""))); + + // Same-zone precision changes are unaffected: still gated by the override, not by the zone check. + Schema stillWorks = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(localMillis, localMicros, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-millis")).toAvroSchema(); + Assertions.assertEquals("local-timestamp-millis", stillWorks.getField("ts").schema().getLogicalType().getName()); + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java new file mode 100644 index 0000000000000..f192e3fe32f67 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.internal.schema.utils; + +import org.apache.hudi.internal.schema.Type; +import org.apache.hudi.internal.schema.Types; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link SchemaChangeUtils#parseTimestampLogicalTypeOverrides(String)}. Validation must + * happen at parse time — the config is threaded through the writer schema deduction path and a + * malformed value would otherwise surface deep inside deduceWriterSchema on the first commit. + */ +public class TestSchemaChangeUtils { + + @Test + public void parseEmptyValueYieldsEmptyMap() { + assertTrue(SchemaChangeUtils.parseTimestampLogicalTypeOverrides(null).isEmpty()); + assertTrue(SchemaChangeUtils.parseTimestampLogicalTypeOverrides("").isEmpty()); + assertTrue(SchemaChangeUtils.parseTimestampLogicalTypeOverrides(" ").isEmpty()); + } + + @Test + public void parseValidSingleEntry() { + Map overrides = SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-millis"); + assertEquals(1, overrides.size()); + assertEquals(Types.TimestampMillisType.get(), overrides.get("ts")); + } + + @Test + public void parseAllFourTokens() { + Map overrides = SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + "a:timestamp-micros,b:timestamp-millis,c:local-timestamp-micros,d:local-timestamp-millis"); + assertEquals(4, overrides.size()); + assertEquals(Types.TimestampType.get(), overrides.get("a")); + assertEquals(Types.TimestampMillisType.get(), overrides.get("b")); + assertEquals(Types.LocalTimestampMicrosType.get(), overrides.get("c")); + assertEquals(Types.LocalTimestampMillisType.get(), overrides.get("d")); + } + + @Test + public void parseIsCaseInsensitive() { + // Tokens are case-insensitive so an operator pasting "Timestamp-Millis" is not surprised. + assertEquals(Types.TimestampMillisType.get(), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:TIMESTAMP-MILLIS").get("ts")); + assertEquals(Types.LocalTimestampMicrosType.get(), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:Local-Timestamp-Micros").get("ts")); + } + + @Test + public void parseSupportsDottedNestedFieldNames() { + // Nested field names use '.'; the parser splits on the LAST ':' so this works unchanged. + Map overrides = + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("payload.event_time:timestamp-millis"); + assertEquals(1, overrides.size()); + assertEquals(Types.TimestampMillisType.get(), overrides.get("payload.event_time")); + } + + @Test + public void parseTrimmedWhitespaceAndSkipEmptySegments() { + Map overrides = SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + " a:timestamp-micros ,, b : timestamp-millis ,"); + assertEquals(2, overrides.size()); + assertEquals(Types.TimestampType.get(), overrides.get("a")); + assertEquals(Types.TimestampMillisType.get(), overrides.get("b")); + } + + @Test + public void parseRejectsMissingColon() { + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> SchemaChangeUtils.parseTimestampLogicalTypeOverrides("field_only")); + assertTrue(ex.getMessage().contains("field_only"), "message should include the offending entry"); + } + + @Test + public void parseRejectsMissingType() { + assertThrows(IllegalArgumentException.class, + () -> SchemaChangeUtils.parseTimestampLogicalTypeOverrides("field:")); + } + + @Test + public void parseRejectsMissingField() { + assertThrows(IllegalArgumentException.class, + () -> SchemaChangeUtils.parseTimestampLogicalTypeOverrides(":timestamp-micros")); + } + + @Test + public void parseRejectsUnknownToken() { + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> SchemaChangeUtils.parseTimestampLogicalTypeOverrides("field:not-a-real-type")); + assertTrue(ex.getMessage().contains("not-a-real-type"), "message should include the bad token"); + } + + @Test + public void parseResultIsUnmodifiable() { + Map overrides = SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros"); + assertThrows(UnsupportedOperationException.class, + () -> overrides.put("other", Types.TimestampMillisType.get())); + } +} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala index 6c353481595ea..bf0187c69ba5b 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala @@ -28,9 +28,11 @@ import org.apache.hudi.common.util.ConfigUtils import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.exception.{HoodieException, SchemaCompatibilityException} import org.apache.hudi.internal.schema.InternalSchema +import org.apache.hudi.internal.schema.Type import org.apache.hudi.internal.schema.convert.InternalSchemaConverter import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils.reconcileSchemaRequirements +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils import org.apache.spark.sql.types.StructField import org.apache.spark.sql.types.StructType @@ -141,10 +143,26 @@ object HoodieSchemaUtils { InternalSchemaConverter.fixNullOrdering(sourceSchema) } + // Parse the per-field timestamp overrides once and thread the parsed map through the + // upfront guard and every downstream branch — reduces cognitive load and matches the + // "single source of truth" theme of the config. + val timestampLogicalTypeOverrides = SchemaChangeUtils.parseTimestampLogicalTypeOverrides( + opts.getOrElse(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key, + HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.defaultValue)) + + // Reconcile timestamp precision up front so every downstream branch (including the + // non-reconcile default path, whose Avro compatibility check is logical-type-blind) is + // guarded: an unverified micros/millis change throws here rather than silently flipping the + // table on the next commit. + val precisionReconciledSourceSchema = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType( + canonicalizedSourceSchema, latestTableSchema, timestampLogicalTypeOverrides) + if (shouldReconcileSchema) { - deduceWriterSchemaWithReconcile(sourceSchema, canonicalizedSourceSchema, latestTableSchema, internalSchemaOpt, opts) + deduceWriterSchemaWithReconcile(sourceSchema, precisionReconciledSourceSchema, latestTableSchema, + internalSchemaOpt, opts, timestampLogicalTypeOverrides) } else { - deduceWriterSchemaWithoutReconcile(sourceSchema, canonicalizedSourceSchema, latestTableSchema, opts) + deduceWriterSchemaWithoutReconcile(sourceSchema, precisionReconciledSourceSchema, latestTableSchema, + opts, timestampLogicalTypeOverrides) } } } @@ -157,7 +175,8 @@ object HoodieSchemaUtils { private def deduceWriterSchemaWithoutReconcile(sourceSchema: HoodieSchema, canonicalizedSourceSchema: HoodieSchema, latestTableSchema: HoodieSchema, - opts: Map[String, String]): HoodieSchema = { + opts: Map[String, String], + timestampLogicalTypeOverrides: java.util.Map[String, Type]): HoodieSchema = { // NOTE: In some cases we need to relax constraint of incoming dataset's schema to be compatible // w/ the table's one and allow schemas to diverge. This is required in cases where // partial updates will be performed (for ex, `MERGE INTO` Spark SQL statement) and as such @@ -173,7 +192,8 @@ object HoodieSchemaUtils { if (!mergeIntoWrites && !shouldValidateSchemasCompatibility && !allowAutoEvolutionColumnDrop) { // Default behaviour val reconciledSchema = if (setNullForMissingColumns) { - AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, latestTableSchema, setNullForMissingColumns) + AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, latestTableSchema, + setNullForMissingColumns, timestampLogicalTypeOverrides) } else { canonicalizedSourceSchema } @@ -199,13 +219,15 @@ object HoodieSchemaUtils { canonicalizedSourceSchema: HoodieSchema, latestTableSchema: HoodieSchema, internalSchemaOpt: Option[InternalSchema], - opts: Map[String, String]): HoodieSchema = { + opts: Map[String, String], + timestampLogicalTypeOverrides: java.util.Map[String, Type]): HoodieSchema = { internalSchemaOpt match { case Some(internalSchema) => // Apply schema evolution, by auto-merging write schema and read schema val setNullForMissingColumns = opts.getOrElse(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key(), HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.defaultValue()).toBoolean - val mergedInternalSchema = AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, internalSchema, setNullForMissingColumns) + val mergedInternalSchema = AvroSchemaEvolutionUtils.reconcileSchema(canonicalizedSourceSchema, internalSchema, + setNullForMissingColumns, timestampLogicalTypeOverrides) val evolvedSchema = InternalSchemaConverter.convert(mergedInternalSchema, latestTableSchema.getFullName) val shouldRemoveMetaDataFromInternalSchema = sourceSchema.getFields.asScala.filter(f => f.name().equalsIgnoreCase(HoodieRecord.RECORD_KEY_METADATA_FIELD)).isEmpty if (shouldRemoveMetaDataFromInternalSchema) HoodieCommonSchemaUtils.removeMetadataFields(evolvedSchema) else evolvedSchema diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 684352d033096..1bd53c605ae49 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -27,6 +27,7 @@ import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient; import org.apache.hudi.client.transaction.lock.InProcessLockProvider; import org.apache.hudi.common.config.DFSPropertiesConfiguration; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.LockConfiguration; @@ -79,6 +80,7 @@ import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.SchemaCompatibilityException; import org.apache.hudi.exception.TableNotFoundException; import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode; import org.apache.hudi.hadoop.fs.HadoopFSUtils; @@ -210,6 +212,11 @@ @Slf4j public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { + // Per-field verdict for the corrupt logical-repair fixtures: relabel ts_millis to millis and + // attach the local-timestamp logical types that 0.x dropped. ts_micros is already micros. + private static final String LOGICAL_REPAIR_TS_OVERRIDES = + "ts_millis:timestamp-millis,local_ts_millis:local-timestamp-millis,local_ts_micros:local-timestamp-micros"; + private void addRecordMerger(HoodieRecordType type, List hoodieConfig) { if (type == HoodieRecordType.SPARK) { Map opts = new HashMap<>(); @@ -967,6 +974,13 @@ public void testBackwardsCompatibility(HoodieTableVersion version) throws Except String schemaPath = zipOutput + "/schema.avsc"; cfg.configs.add(String.format(("%s=%s"), "hoodie.streamer.schemaprovider.source.schema.file", schemaPath)); cfg.configs.add(String.format(("%s=%s"), "hoodie.streamer.schemaprovider.target.schema.file", schemaPath)); + // The v6/v8 col-stats fixture reuses the same trips_logical_types_json corrupt schema as + // the logical-repair tests — 0.x collapsed ts_millis to timestamp-micros and dropped the + // local-timestamp logical types entirely. Provide the same explicit verdict so the guard + // in HoodieSchemaUtils.deduceWriterSchema authorizes the repair rather than rejecting the + // unverified precision change. + cfg.configs.add(String.format(("%s=%s"), + HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key(), LOGICAL_REPAIR_TS_OVERRIDES)); cfg.forceDisableCompaction = true; cfg.sourceLimit = 100_000; cfg.ignoreCheckpoint = "12345"; @@ -1109,8 +1123,19 @@ private void assertBoundaryCounts(Dataset df, String exprZero, String exprT } @ParameterizedTest - @CsvSource(value = {"SIX,AVRO,CLUSTER", "EIGHT,AVRO,CLUSTER", "CURRENT,AVRO,NONE", "CURRENT,AVRO,CLUSTER", "CURRENT,SPARK,NONE", "CURRENT,SPARK,CLUSTER"}) - void testCOWLogicalRepair(String tableVersion, String recordType, String operation) throws Exception { + @CsvSource(value = { + // Repair succeeds when a per-field verdict is set, on the default (non-reconcile) write path... + "SIX,AVRO,CLUSTER,false,true", "EIGHT,AVRO,CLUSTER,false,true", + "CURRENT,AVRO,NONE,false,true", "CURRENT,AVRO,CLUSTER,false,true", + "CURRENT,SPARK,NONE,false,true", "CURRENT,SPARK,CLUSTER,false,true", + // ...and on the reconcile path (setNullForMissingColumns=true). + "SIX,AVRO,CLUSTER,true,true", "EIGHT,AVRO,CLUSTER,true,true", "CURRENT,AVRO,CLUSTER,true,true", + // Guard: with no verdict, the mislabeled timestamp/local-timestamp columns must be rejected on + // the first sync, on both the reconcile path and the default path. + "SIX,AVRO,CLUSTER,true,false", "SIX,AVRO,CLUSTER,false,false"}) + void testCOWLogicalRepair(String tableVersion, String recordType, String operation, + boolean setNullForMissingColumns, + boolean setTimestampOverride) throws Exception { TestMercifulJsonToRowConverterBase.timestampNTZCompatibility(() -> { String dirName = "trips_logical_types_json_cow_write"; String dataPath = basePath + "/" + dirName; @@ -1139,9 +1164,36 @@ void testCOWLogicalRepair(String tableVersion, String recordType, String operati properties.setProperty("hoodie.parquet.small.file.limit", "-1"); properties.setProperty("hoodie.cleaner.commits.retained", "10"); properties.setProperty(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), tableVersionString); + properties.setProperty(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key(), + Boolean.toString(setNullForMissingColumns)); + if (setTimestampOverride) { + // Per-field verdict authorizing the repair: relabel ts_millis to millis and attach the + // local-timestamp logical types 0.x dropped. ts_micros stays micros (no entry needed). + properties.setProperty(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key(), + LOGICAL_REPAIR_TS_OVERRIDES); + } Option propt = Option.of(properties); + if (!setTimestampOverride) { + // No per-field verdict. The mislabeled timestamp/local-timestamp columns must be rejected on + // the first sync rather than silently flipped, on both the reconcile and default write paths. + // syncOnce wraps the guard's SchemaCompatibilityException in a HoodieIngestionException, so + // walk the cause chain to assert on the underlying exception. + Throwable thrown = assertThrows(Exception.class, + () -> syncOnce(prepCfgForCowLogicalRepair(tableBasePath, "456"), propt)); + Throwable cause = thrown; + while (cause != null && !(cause instanceof SchemaCompatibilityException)) { + cause = cause.getCause(); + } + assertTrue(cause instanceof SchemaCompatibilityException, + "Expected a SchemaCompatibilityException in the cause chain, got: " + thrown); + assertTrue(cause.getMessage().contains("column 'ts_millis'") + && cause.getMessage().contains("without an explicit"), + "Unexpected message: " + cause.getMessage()); + return; + } + syncOnce(prepCfgForCowLogicalRepair(tableBasePath, "456"), propt); inputDataPath = getClass().getClassLoader().getResource("logical-repair/cow_write_updates/3").toURI().toString(); @@ -1189,11 +1241,17 @@ void testCOWLogicalRepair(String tableVersion, String recordType, String operati } @ParameterizedTest - @CsvSource(value = {"SIX,AVRO,CLUSTER,AVRO", "EIGHT,AVRO,CLUSTER,AVRO", - "CURRENT,AVRO,NONE,AVRO", "CURRENT,AVRO,CLUSTER,AVRO", "CURRENT,AVRO,COMPACT,AVRO", - "CURRENT,AVRO,NONE,PARQUET", "CURRENT,AVRO,CLUSTER,PARQUET", "CURRENT,AVRO,COMPACT,PARQUET", - "CURRENT,SPARK,NONE,PARQUET", "CURRENT,SPARK,CLUSTER,PARQUET", "CURRENT,SPARK,COMPACT,PARQUET"}) - void testMORLogicalRepair(String tableVersion, String recordType, String operation, String logBlockType) throws Exception { + @CsvSource(value = {"SIX,AVRO,CLUSTER,AVRO,false,true", "EIGHT,AVRO,CLUSTER,AVRO,false,true", + "CURRENT,AVRO,NONE,AVRO,false,true", "CURRENT,AVRO,CLUSTER,AVRO,false,true", "CURRENT,AVRO,COMPACT,AVRO,false,true", + "CURRENT,AVRO,NONE,PARQUET,false,true", "CURRENT,AVRO,CLUSTER,PARQUET,false,true", "CURRENT,AVRO,COMPACT,PARQUET,false,true", + "CURRENT,SPARK,NONE,PARQUET,false,true", "CURRENT,SPARK,CLUSTER,PARQUET,false,true", "CURRENT,SPARK,COMPACT,PARQUET,false,true", + // Variants that exercise the schema-reconcile path (setNullForMissingColumns=true) with a verdict. + "SIX,AVRO,CLUSTER,AVRO,true,true", "EIGHT,AVRO,CLUSTER,AVRO,true,true", "CURRENT,AVRO,CLUSTER,AVRO,true,true", + // Guard: with no verdict, the first sync must throw, on both the reconcile and default paths. + "SIX,AVRO,CLUSTER,AVRO,true,false", "SIX,AVRO,CLUSTER,AVRO,false,false"}) + void testMORLogicalRepair(String tableVersion, String recordType, String operation, String logBlockType, + boolean setNullForMissingColumns, + boolean setTimestampOverride) throws Exception { TestMercifulJsonToRowConverterBase.timestampNTZCompatibility(() -> { String tableSuffix; String logFormatValue; @@ -1241,6 +1299,12 @@ void testMORLogicalRepair(String tableVersion, String recordType, String operati properties.setProperty("hoodie.cleaner.commits.retained", "10"); properties.setProperty(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), tableVersionString); properties.setProperty(HoodieStorageConfig.LOGFILE_DATA_BLOCK_FORMAT.key(), logFormatValue); + properties.setProperty(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key(), + Boolean.toString(setNullForMissingColumns)); + if (setTimestampOverride) { + properties.setProperty(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key(), + LOGICAL_REPAIR_TS_OVERRIDES); + } boolean disableCompaction; if ("COMPACT".equals(operation)) { @@ -1262,6 +1326,25 @@ void testMORLogicalRepair(String tableVersion, String recordType, String operati Option propt = Option.of(properties); + if (!setTimestampOverride) { + // No per-field verdict. The mislabeled timestamp/local-timestamp columns must be rejected on + // the first sync rather than silently flipped, on both the reconcile and default write paths. + // syncOnce wraps the guard's SchemaCompatibilityException in a HoodieIngestionException, so + // walk the cause chain to assert on the underlying exception. + Throwable thrown = assertThrows(Exception.class, + () -> syncOnce(prepCfgForMorLogicalRepair(tableBasePath, dirName, "123", disableCompaction), propt)); + Throwable cause = thrown; + while (cause != null && !(cause instanceof SchemaCompatibilityException)) { + cause = cause.getCause(); + } + assertTrue(cause instanceof SchemaCompatibilityException, + "Expected a SchemaCompatibilityException in the cause chain, got: " + thrown); + assertTrue(cause.getMessage().contains("column 'ts_millis'") + && cause.getMessage().contains("without an explicit"), + "Unexpected message: " + cause.getMessage()); + return; + } + syncOnce(prepCfgForMorLogicalRepair(tableBasePath, dirName, "123", disableCompaction), propt); String prevTimezone = sparkSession.conf().get("spark.sql.session.timeZone"); From b3c7115524b654f982af4cdc4d8341080e33f01e Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 16:36:39 +0800 Subject: [PATCH 213/255] fix(build): drop duplicate MetricRegistry import in TestHoodieIncrSource 920d63ccb55c (#17877, "Add Lombok annotations to hudi-utilities (Part 3)") left a second `import com.codahale.metrics.MetricRegistry;` alongside the existing one, so hudi-utilities failed checkstyle with "RedundantImport: Duplicate import to line 21". Remove the duplicate. Unrelated to any cherry-pick; it has been failing since that commit landed on this branch. --- .../org/apache/hudi/utilities/sources/TestHoodieIncrSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java index 8e6bbd4f024a3..d94953f75da41 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java @@ -67,7 +67,6 @@ import org.apache.hudi.utilities.streamer.SourceProfile; import org.apache.hudi.utilities.streamer.SourceProfileSupplier; -import com.codahale.metrics.MetricRegistry; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; From 07da1c5ae465027b445f952e69be35f2030da458 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Tue, 28 Jul 2026 20:08:37 +0700 Subject: [PATCH 214/255] fix(reader): give the file group reader schema handler the merged record-merge properties (#19389) Adapted for release-1.2.1: dropped the identical two-line change to HoodieLsmFileGroupReader. That class lives in org.apache.hudi.common.table.read.lsm, which arrives with 6170ac905273 (#18987, "feat: Add a lsm-tree based FG reader") and is not backported. The fix to HoodieFileGroupReader, the new TestFileGroupReaderDeleteMarkerProps and the TestHoodieFileGroupReaderOnSpark changes all apply unchanged. (cherry picked from commit 34e15860082b118c098dccdb84034b8fb36bf14b) --- .../table/read/HoodieFileGroupReader.java | 3 +- .../TestFileGroupReaderDeleteMarkerProps.java | 219 ++++++++++++++++++ .../TestHoodieFileGroupReaderOnSpark.scala | 37 ++- 3 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupReaderDeleteMarkerProps.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java index 0b536b511db10..e4aae8f629401 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java @@ -198,7 +198,8 @@ private HoodieFileGroupReader( throw new IllegalArgumentException("Filegroup reader is doing log file merge but not reading from the start of the base file"); } HoodieTableConfig tableConfig = hoodieTableMetaClient.getTableConfig(); - this.props = ConfigUtils.getMergeProps(props, tableConfig); + props = ConfigUtils.getMergeProps(props, tableConfig); + this.props = props; this.partitionPathFields = tableConfig.getPartitionFields(); readerContext.initRecordMerger(props); readerContext.setTablePath(tablePath); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupReaderDeleteMarkerProps.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupReaderDeleteMarkerProps.java new file mode 100644 index 0000000000000..5646e34aaa205 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupReaderDeleteMarkerProps.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.table.read; + +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieReaderContext; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.storage.StoragePath; + +import org.apache.avro.SchemaBuilder; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER; +import static org.apache.hudi.common.table.HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A table at version 9 or later persists a custom delete marker only under the + * {@code hoodie.record.merge.property.} prefix, and {@link ConfigUtils#getMergeProps} is what strips that + * prefix back to the plain {@code hoodie.payload.delete.field} / {@code .marker} keys that + * {@link DeleteContext} reads. Read-path callers hand {@link HoodieFileGroupReader} the properties as they + * come, so the reader is the only thing that can perform that merge before the schema handler is built. + */ +public class TestFileGroupReaderDeleteMarkerProps { + + private static final String DELETE_FIELD = "op"; + private static final String DELETE_VALUE = "D"; + + private static final HoodieSchema TABLE_SCHEMA = HoodieSchema.fromAvroSchema( + SchemaBuilder.record("rec").fields() + .requiredString(HoodieRecord.RECORD_KEY_METADATA_FIELD) + .requiredString("key") + .requiredLong("ts") + .requiredString(DELETE_FIELD) + .endRecord()); + + private static final HoodieSchema REQUESTED_SCHEMA = HoodieSchema.fromAvroSchema( + SchemaBuilder.record("rec").fields() + .requiredString("key") + .endRecord()); + + private static final HoodieSchema REQUESTED_SCHEMA_WITH_DELETE_FIELD = HoodieSchema.fromAvroSchema( + SchemaBuilder.record("rec").fields() + .requiredString("key") + .requiredString(DELETE_FIELD) + .endRecord()); + + private static HoodieTableConfig versionNineTableConfigWithCustomDeleteMarker() { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, String.valueOf(HoodieTableVersion.NINE.versionCode())); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE, RecordMergeMode.COMMIT_TIME_ORDERING.name()); + tableConfig.setValue(RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY, DELETE_FIELD); + tableConfig.setValue(RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER, DELETE_VALUE); + return tableConfig; + } + + /** + * The properties a query engine hands the reader: the table's own properties, in which the custom delete + * marker only exists in its prefixed form. Nothing un-prefixes them before the reader is built. + */ + private static TypedProperties readerProps(HoodieTableConfig tableConfig) { + return TypedProperties.copy(tableConfig.getProps()); + } + + /** + * Builds a file group reader over a log-file-bearing split and returns the schema handler it installed on + * the reader context. + */ + private static FileGroupReaderSchemaHandler schemaHandlerOfReader(TypedProperties properties, + HoodieTableConfig tableConfig) { + return schemaHandlerOfReader(properties, tableConfig, REQUESTED_SCHEMA); + } + + private static FileGroupReaderSchemaHandler schemaHandlerOfReader(TypedProperties properties, + HoodieTableConfig tableConfig, + HoodieSchema requestedSchema) { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(metaClient.getBasePath()).thenReturn(new StoragePath("file:///tmp/hoodie_test_table")); + + AtomicReference> installedHandler = new AtomicReference<>(); + HoodieReaderContext readerContext = mock(HoodieReaderContext.class, RETURNS_DEEP_STUBS); + // The reader context reports log files so the schema handler takes the merge path; the split itself stays + // empty because the reader is only built, never iterated - no file is ever opened. + when(readerContext.getHasLogFiles()).thenReturn(true); + when(readerContext.getHasBootstrapBaseFile()).thenReturn(false); + when(readerContext.getInstantRange()).thenReturn(Option.empty()); + when(readerContext.getMergeMode()).thenReturn(RecordMergeMode.COMMIT_TIME_ORDERING); + when(readerContext.getRecordContext().supportsParquetRowIndex()).thenReturn(false); + doAnswer(invocation -> { + installedHandler.set(invocation.getArgument(0)); + return null; + }).when(readerContext).setSchemaHandler(any()); + when(readerContext.getSchemaHandler()).thenAnswer(invocation -> installedHandler.get()); + + HoodieFileGroupReader.builder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(metaClient) + .withLatestCommitTime("001") + .withDataSchema(TABLE_SCHEMA) + .withRequestedSchema(requestedSchema) + .withProps(properties) + .withLogFiles(Stream.empty()) + .withPartitionPath("") + .withStart(0L) + .withLength(Long.MAX_VALUE) + .build(); + + return installedHandler.get(); + } + + private static List requiredFieldNames(FileGroupReaderSchemaHandler handler) { + return handler.getRequiredSchema().getFields().stream().map(field -> field.name()).collect(Collectors.toList()); + } + + /** + * Sanity check on the premise: the un-prefixing lives in getMergeProps, so the properties the reader is + * handed do not carry the plain delete keys at all. + */ + @Test + public void mergePropsIsWhatUnprefixesTheDeleteMarker() { + HoodieTableConfig tableConfig = versionNineTableConfigWithCustomDeleteMarker(); + TypedProperties props = readerProps(tableConfig); + + assertNull(props.getProperty(DELETE_KEY), "reader props must not carry the unprefixed delete key"); + assertEquals(DELETE_FIELD, ConfigUtils.getMergeProps(props, tableConfig).getProperty(DELETE_KEY)); + + assertTrue(new DeleteContext(props, TABLE_SCHEMA).getCustomDeleteMarkerKeyValue().isEmpty(), + "DeleteContext built from the reader props sees no custom delete marker"); + assertTrue(new DeleteContext(ConfigUtils.getMergeProps(props, tableConfig), TABLE_SCHEMA) + .getCustomDeleteMarkerKeyValue().isPresent(), + "DeleteContext built from merged props sees the custom delete marker"); + } + + /** + * The reader must merge the table's record-merge properties in before building the schema handler. + * Otherwise the handler's DeleteContext carries no marker, and since FileGroupRecordBuffer takes its + * DeleteContext from that very handler, custom deletes stop being recognised on the whole read path. + */ + @Test + public void readerResolvesTheCustomDeleteMarkerFromPrefixedTableProps() { + HoodieTableConfig tableConfig = versionNineTableConfigWithCustomDeleteMarker(); + + FileGroupReaderSchemaHandler handler = schemaHandlerOfReader(readerProps(tableConfig), tableConfig); + + assertTrue(handler.getDeleteContext().getCustomDeleteMarkerKeyValue().isPresent(), + "the schema handler the reader installs must resolve the custom delete marker"); + assertTrue(requiredFieldNames(handler).contains(DELETE_FIELD), + "required schema must contain the custom delete column " + DELETE_FIELD); + } + + /** + * The delete column becomes a mandatory field, so it must not be appended twice when the query already + * asked for it. + */ + @Test + public void deleteColumnIsNotDuplicatedWhenAlreadyRequested() { + HoodieTableConfig tableConfig = versionNineTableConfigWithCustomDeleteMarker(); + + FileGroupReaderSchemaHandler handler = + schemaHandlerOfReader(readerProps(tableConfig), tableConfig, REQUESTED_SCHEMA_WITH_DELETE_FIELD); + + assertEquals(1, requiredFieldNames(handler).stream().filter(DELETE_FIELD::equals).count(), + "the custom delete column must appear exactly once in the required schema"); + } + + /** + * Control: a reader handed properties that already carry the plain delete keys - which is how the existing + * engine tests are configured - resolves the marker either way, which is why this defect stayed hidden. + */ + @Test + public void readerAlsoResolvesAMarkerSuppliedInPlainForm() { + HoodieTableConfig tableConfig = versionNineTableConfigWithCustomDeleteMarker(); + TypedProperties props = readerProps(tableConfig); + props.setProperty(DELETE_KEY, DELETE_FIELD); + props.setProperty(DELETE_MARKER, DELETE_VALUE); + + FileGroupReaderSchemaHandler handler = schemaHandlerOfReader(props, tableConfig); + + assertTrue(handler.getDeleteContext().getCustomDeleteMarkerKeyValue().isPresent()); + assertTrue(requiredFieldNames(handler).contains(DELETE_FIELD)); + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala index d4eae9e79d74d..6098f4719598e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala @@ -231,7 +231,8 @@ class TestHoodieFileGroupReaderOnSpark extends TestHoodieFileGroupReaderBase[Int def testCustomDelete(useFgReader: String, tableType: String, positionUsed: String, - mergeMode: String): Unit = { + mergeMode: String, + markerFromTableConfigOnly: String): Unit = { val payloadClass = "org.apache.hudi.common.table.read.CustomPayloadForTesting" val fgReaderOpts: Map[String, String] = Map( HoodieWriteConfig.MERGE_SMALL_FILE_GROUP_CANDIDATES_LIMIT.key -> "0", @@ -241,13 +242,17 @@ class TestHoodieFileGroupReaderOnSpark extends TestHoodieFileGroupReaderBase[Int ) val deleteOpts: Map[String, String] = Map( DELETE_KEY -> "op", DELETE_MARKER -> "d") - val readOpts = if (mergeMode.equals("CUSTOM")) { - fgReaderOpts ++ deleteOpts ++ Map( - HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key -> payloadClass) + val payloadOpts = if (mergeMode.equals("CUSTOM")) { + Map(HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key -> payloadClass) } else { - fgReaderOpts ++ deleteOpts + Map.empty[String, String] } - val opts = readOpts + val opts = fgReaderOpts ++ deleteOpts ++ payloadOpts + // The write persists the marker on the table under the record-merge property prefix. When the query does + // not restate the delete options, the table config is the only place the reader can learn about them - + // which is what a query that just loads the path looks like. + val tableConfigOnly = markerFromTableConfigOnly.equals("true") + val readOpts = if (tableConfigOnly) fgReaderOpts ++ payloadOpts else opts val columns = Seq("ts", "key", "rider", "driver", "fare", "op") val data = Seq( @@ -269,6 +274,11 @@ class TestHoodieFileGroupReaderOnSpark extends TestHoodieFileGroupReaderBase[Int val metaClient = HoodieTableMetaClient .builder().setConf(getStorageConf).setBasePath(getBasePath).build assertEquals((1, 0), getFileCount(metaClient, getBasePath)) + if (tableConfigOnly) { + assertTrue(metaClient.getTableConfig.getProps.containsKey( + HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY), + "the delete marker must be persisted on the table, otherwise the read side has nothing to pick up") + } // Delete using delete markers. val updateData = Seq( @@ -456,12 +466,15 @@ class TestHoodieFileGroupReaderOnSpark extends TestHoodieFileGroupReaderBase[Int object TestHoodieFileGroupReaderOnSpark { def customDeleteTestParams(): java.util.List[Arguments] = { java.util.Arrays.asList( - Arguments.of("true", "MERGE_ON_READ", "false", "EVENT_TIME_ORDERING"), - Arguments.of("true", "MERGE_ON_READ", "true", "EVENT_TIME_ORDERING"), - Arguments.of("true", "MERGE_ON_READ", "false", "COMMIT_TIME_ORDERING"), - Arguments.of("true", "MERGE_ON_READ", "true", "COMMIT_TIME_ORDERING"), - Arguments.of("true", "MERGE_ON_READ", "false", "CUSTOM"), - Arguments.of("true", "MERGE_ON_READ", "true", "CUSTOM")) + Arguments.of("true", "MERGE_ON_READ", "false", "EVENT_TIME_ORDERING", "false"), + Arguments.of("true", "MERGE_ON_READ", "true", "EVENT_TIME_ORDERING", "false"), + Arguments.of("true", "MERGE_ON_READ", "false", "COMMIT_TIME_ORDERING", "false"), + Arguments.of("true", "MERGE_ON_READ", "true", "COMMIT_TIME_ORDERING", "false"), + Arguments.of("true", "MERGE_ON_READ", "false", "CUSTOM", "false"), + Arguments.of("true", "MERGE_ON_READ", "true", "CUSTOM", "false"), + // The query does not restate the delete options, so the table config is the reader's only source. + Arguments.of("true", "MERGE_ON_READ", "false", "EVENT_TIME_ORDERING", "true"), + Arguments.of("true", "MERGE_ON_READ", "false", "COMMIT_TIME_ORDERING", "true")) } def getFileCount(metaClient: HoodieTableMetaClient, basePath: String): (Long, Long) = { From 63f32d8120c4fb682d9eb467acf383425914830d Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 29 Jul 2026 09:08:56 +0700 Subject: [PATCH 215/255] fix(hive-sync): stop HiveDriverPool from swallowing a failed SQL batch (#19391) * fix(hive-sync): stop HiveDriverPool from swallowing a failed SQL batch * fix(hive-sync): address review nits on HiveDriverPool --------- Co-authored-by: Vova Kolmakov (cherry picked from commit db8d02cbfac1eb45ae2cfecd9bad049083ac6d48) --- .../apache/hudi/hive/util/HiveDriverPool.java | 34 ++++++++++-- .../hudi/hive/util/TestHiveDriverPool.java | 54 +++++++++++++++++-- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java index 5950bf442313d..d9ebea7e10efc 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveDriverPool.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; @@ -135,7 +136,7 @@ public void runOnEachWorker(List setupSqls) { worker.driver.run(sql); } } catch (Throwable t) { - dispatch.abort(); + dispatch.recordFailure(t); throw t; } finally { dispatch.taskSettled(); @@ -179,7 +180,7 @@ public Dispatch dispatchAll(List sqls) { try { worker.driver.run(sql); } catch (Throwable t) { - dispatch.abort(); + dispatch.recordFailure(t); throw t; } finally { dispatch.taskSettled(); @@ -208,7 +209,11 @@ public void awaitAll(Dispatch dispatch) { dispatch.awaitSettledOrAborted(); int cancelled = dispatch.cancelPending(); - Exception firstError = null; + // Seeded from the batch's own record rather than discovered by walking the futures: + // cancelPending() above may have erased the failing task's exception. See + // Dispatch#recordFailure. The walk below still runs, to count outcomes and to catch + // a failure that somehow never made it into the record. + Throwable firstError = dispatch.firstFailure(); int completed = 0; for (Future f : dispatch.futures()) { try { @@ -229,7 +234,11 @@ public void awaitAll(Dispatch dispatch) { cancelled++; } else if (firstError == null) { firstError = cause; - } else { + } else if (ee.getCause() != firstError) { + // Identity check against the raw cause, not the unwrapped one: when the failing + // task wins the race against cancelPending(), its future reports the very + // Throwable already held in firstError, and re-logging it here would duplicate + // the exception this method is about to throw. LOG.warn("Additional SQL batch failed (suppressed in favor of first error)", cause); } } @@ -256,6 +265,7 @@ public static final class Dispatch { private final int total; private final AtomicInteger settled = new AtomicInteger(0); private final AtomicBoolean aborted = new AtomicBoolean(false); + private final AtomicReference firstFailureRef = new AtomicReference<>(); private final CountDownLatch done = new CountDownLatch(1); private volatile boolean sealed; @@ -279,11 +289,25 @@ private boolean aborted() { return aborted.get(); } - private void abort() { + /** + * Records a task's failure and aborts the batch. The Throwable is kept here rather + * than being left for {@link Future#get()} to report, because the failing task is + * racing the awaiting thread: this call releases {@link #awaitSettledOrAborted()}, + * but the task's exception only reaches its {@code FutureTask} after {@code call()} + * returns. {@link #cancelPending()} in between wins the {@code FutureTask} state CAS + * (cancel succeeds on any task still NEW, which includes one mid-unwind), turning the + * later {@code setException} into a no-op and the error into a CancellationException. + */ + private void recordFailure(Throwable t) { + firstFailureRef.compareAndSet(null, t); aborted.set(true); done.countDown(); } + private Throwable firstFailure() { + return firstFailureRef.get(); + } + private void taskSettled() { settled.incrementAndGet(); signalIfComplete(); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java index fd58b7023aa19..5b946447ba4dd 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveDriverPool.java @@ -40,6 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -138,7 +139,8 @@ void awaitAllThrowsFirstError() throws Exception { HiveDriverPool.Dispatch futures = pool.dispatchAll(Arrays.asList("OK", "FAIL", "OK")); HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, () -> pool.awaitAll(futures)); - assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + assertNotNull(ex.getCause()); + assertTrue(ex.getCause().getMessage().contains("boom")); } } @@ -254,7 +256,8 @@ void awaitAllCancelsPendingFuturesOnFirstError() throws Exception { HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, () -> pool.awaitAll(dispatch)); - assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + assertNotNull(ex.getCause()); + assertTrue(ex.getCause().getMessage().contains("boom")); assertEquals(Collections.singletonList("FAIL"), executed, "Statements queued behind the failure must never reach the Driver"); } @@ -307,10 +310,55 @@ void awaitAllStopsLaterWorkerWhenEarlierFutureIsSlow() throws Exception { HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, () -> pool.awaitAll(dispatch)); - assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("boom")); + assertNotNull(ex.getCause()); + assertTrue(ex.getCause().getMessage().contains("boom")); assertFalse(executed.contains("AFTER_FAIL"), "Statement queued behind a failure on the same worker must not be applied, " + "even while an earlier future on another worker is still running"); } } + + /** + * Regression for the swallowed-failure race: awaitAll must still report the error when + * its own cancel() sweep has already marked the failing task's future CANCELLED. + * + *

    The failing worker aborts the batch from inside its catch block, which releases + * awaitAll, but its exception only reaches the FutureTask after {@code call()} returns. + * {@code FutureTask.cancel(false)} succeeds on any task still in state NEW - a task + * mid-unwind included - so the cancel wins the state CAS, the later {@code setException} + * becomes a no-op, and {@code get()} reports CancellationException instead of the error. + * awaitAll then counted it as merely cancelled and returned normally, reporting a failed + * partition-DDL batch as a successful sync. + * + *

    Deterministic where the three tests above are not: instead of hoping the awaiting + * thread wins, this cancels the future explicitly - exactly what cancelPending() does - + * while the Driver is parked, and only then lets it throw. + */ + @Test + void awaitAllReportsFailureWhenFailingFutureIsCancelledMidFlight() throws Exception { + HiveSyncConfig config = configWithEmptyHiveConf(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + HiveDriverPool.DriverFactory factory = (db) -> { + Driver d = mock(Driver.class); + doAnswer(inv -> { + entered.countDown(); + release.await(10, TimeUnit.SECONDS); + throw new RuntimeException("boom"); + }).when(d).run(anyString()); + return d; + }; + try (HiveDriverPool pool = new HiveDriverPool(config, 1, factory)) { + HiveDriverPool.Dispatch dispatch = pool.dispatchAll(Collections.singletonList("FAIL")); + assertTrue(entered.await(10, TimeUnit.SECONDS), "Driver must have started the statement"); + assertTrue(dispatch.futureAt(0).cancel(false), + "Sanity: a running FutureTask is still NEW, so cancel(false) must succeed"); + release.countDown(); + + HoodieHiveSyncException ex = assertThrows(HoodieHiveSyncException.class, + () -> pool.awaitAll(dispatch)); + assertNotNull(ex.getCause()); + assertTrue(ex.getCause().getMessage().contains("boom")); + } + } } From dd454cd2a92b3d7aca3e8ac8808e89bfe519fc51 Mon Sep 17 00:00:00 2001 From: voonhous Date: Wed, 29 Jul 2026 15:19:18 +0800 Subject: [PATCH 216/255] feat(trino): resolve merge-required columns from the table schema (#19288) * feat(trino): resolve merge-required columns from the full table schema Fixes apache/hudi#19249, including the scope update in issue comment 4937143911; relates to the base read path of apache/hudi#18837. The file-group reader can require merge columns the query projection does not carry: Hudi meta fields like _hoodie_commit_time, delete markers, record-key data columns, a custom merger's mandatory fields, or the whole table schema when a CUSTOM merger is not projection compatible. buildRequiredColumnHandles previously hand-synthesized a HIVE_STRING/VARCHAR handle behind a HOODIE_META_COLUMNS guard and threw NOT_SUPPORTED for anything else, and the base-file read silently returned projection-only records that made such mergers see nulls. Log side: - Every required-schema field carries its real Avro type from the table schema, so unresolved handles are typed directly from the field (NativeLogicalTypesAvroTypeBlockHandler + HiveTypeTranslator) via the new HudiUtil.toColumnHandle; the synthesis, guard and exception are gone. trino-hive-formats moves from runtime to compile scope. Base side: - HudiPageSourceProvider mirrors the file-group reader's decision: HudiUtil.resolveMergeModeAndStrategyId applies the same version-gated inference as hudi-common (merge mode below table version 9, strategy id below version 8), and when the resolved CUSTOM merger is not projection compatible (interface default; also every payload-based table, since the payload strategy resolves HoodieAvroRecordMerger), the read projection expands to every table-schema column via HudiUtil.appendMissingSchemaColumns. Matches Spark; the extra I/O is inherent to such mergers. - getMergeRequiredColumnHandles predicts every column FileGroupReaderSchemaHandler.getMandatoryFieldsForMerging can demand: ordering columns under the inferred (not raw) merge mode, _hoodie_is_deleted and _hoodie_operation unconditionally (dropped when not table columns), the custom delete key when both hoodie.payload.delete.field and .marker are set, and the record-key data columns when hoodie.populate.meta.fields is false. - The base read reuses the pre-built page source, so its projection MUST cover requiredSchema: HudiTrinoReaderContext now guards that invariant and fails with HUDI_SCHEMA_ERROR naming any missing columns instead of merging with nulls. MOR splits with log files but no base file fail with a clear NOT_SUPPORTED instead of a bare NoSuchElementException. Cleanups in the same area: - Replace SynthesizedColumnHandler with PrefilledColumnValues, a thin per-split adapter over trino-hive's HiveUtil.getPrefilledColumnValue. Fixes $partition rendering multi-key partitions in hash order, handles the hive null marker (backslash-N) in partition values, adds previously unsupported partition-key types, and deletes SynthesizedColumnStrategy plus the hand-rolled appendPartitionKey dispatch. - Remove dead code (HudiTrinoReaderContext.colToPosMap and dataHandles, HudiPageSource.readerContext); make fields private final. - remapColumnIndicesToPhysical throws a diagnosable TrinoException instead of an autoboxing NPE for columns absent from the parquet file schema, and lowercases with Locale.ROOT. Tests: - TestHudiNonProjectionCompatibleMerger + NonProjectionCompatibleRankMerger (no isProjectionCompatible/getMandatoryFieldsForMerging overrides) + initializer: narrow-projection MOR queries that never project merge_rank, laid out so one key's winning rank is on the log record and the other's on the base record; sum(value)=199 discriminates the correct merge from base-only (110) and newest-wins (103). - Unit coverage: toColumnHandle, appendMissingSchemaColumns, usesNonProjectionCompatibleMerger, resolveMergeModeAndStrategyId (table versions 6/8/9), merge-required column prediction, PrefilledColumnValues (incl. the $partition ordering fix), and the typed missing-column remap error (both remap tests). - Existing projection-compatible merger tests stay as fast-path regression coverage; stale MaxRankRecordMerger javadoc refreshed. * test(trino): flip non-projection-compatible merger rejection test TestHudiCustomMerger's negative test was added on master while this change was in flight, pinning the old NOT_SUPPORTED rejection per the plan in issue comment 4937143911. Flip it to the positive assertions that plan called for: narrow-projection sum(value)=199 plus a full-row check under NonProjectionCompatibleTestRecordMerger, whose javadoc is refreshed to describe the full-schema path. * test(trino): drop issue reference and history note from flipped test comments * review(trino): address first review round on 19288 - soften requiresFullSchemaRead javadoc to the actual guarantee - remap missing columns one past the last field so the parquet reader null-fills them like the name-based path (was a typed hard failure) - reject CUSTOM merge mode without a persisted strategy id with an actionable HUDI_BAD_DATA error instead of an NPE in hudi-common - recover merge-required columns the metastore lacks from the resolved table schema on the merge path (appendMissingMergeRequiredColumns); no extra I/O, pinned by the file-operation-count tests - document the $file_modified_time zone change in PrefilledColumnValues - extract AbstractMergerHudiTablesInitializer; the three MoR fixture initializers keep only their deltas - add payload-only table version 6 fixture (RankBasedTestPayload) and E2E coverage in TestHudiNonProjectionCompatibleMerger * test(trino): pin metastore-omitted merge-column recovery end to end New EVENT_TIME_ORDERING MOR fixture (omitted_ordering_field_mor) whose metastore column list omits the ts its Avro schema carries, the shape hive sync leaves when a column is not synced. Event-time merging needs ts on both sides, so the row-level and narrow-projection sum(value)=199 cases only pass when the merge path recovers the column from the table schema; reverting the recovery line fails both with the missing-column guard error. * review(trino): address second review round on 19288 - recover a CUSTOM merger's own mandatory fields on the merge path too: appendMissingMergeRequiredColumns asks the same resolved merger the file-group reader will use (classloading only, no I/O), unit-tested with MaxRankRecordMerger and pinned end to end by a fixture whose metastore omits the merger-declared merge_rank column - widen the no-strategy-id guard wording to version 8 or later - scope the schema-gate javadoc claims to _hoodie_is_deleted and _hoodie_operation, the two fields the reader actually schema-gates, and split the metastore/schema halves across the functions that implement them - move the withPreCombineField constraint comment back into the write config builder chain - document that the composite initializer cannot host fixtures that keep their write client open - strip only the merge strategy id in the payload-only v6 fixture: the merge mode never reaches disk (its config is since-version 1.0.0, dropped by table creation itself) * test(trino): pin the CUSTOM-block guards in appendMissingMergeRequiredColumns Mirror the two usesNonProjectionCompatibleMerger cases onto the append: CUSTOM without a strategy id and CUSTOM with an id no configured merger declares both fall back to the table-config-derived names, so removing either guard fails a unit test instead of resurfacing the NPE. The unresolvable case must not use the all-zeros uuid -- that is PAYLOAD_BASED_MERGE_STRATEGY_UUID, which always resolves; the first version of the test passed vacuously for exactly that reason until a mutation check caught it. Shared fixture helpers dedup the CUSTOM tests. (cherry picked from commit 3b146cebc27e33868e18c375d9a5087e4e47e273) --- hudi-trino/pom.xml | 11 +- .../hudi/HudiBaseFileOnlyPageSource.java | 22 +- .../io/trino/plugin/hudi/HudiPageSource.java | 24 +- .../plugin/hudi/HudiPageSourceProvider.java | 135 ++++++-- .../java/io/trino/plugin/hudi/HudiUtil.java | 267 ++++++++++++++- .../hudi/reader/HudiTrinoReaderContext.java | 108 +++--- .../plugin/hudi/util/HudiAvroSerializer.java | 10 +- .../hudi/util/PrefilledColumnValues.java | 128 +++++++ .../hudi/util/SynthesizedColumnHandler.java | 320 ------------------ .../hudi/util/SynthesizedColumnStrategy.java | 25 -- .../plugin/hudi/TestHudiCustomMerger.java | 23 +- .../hudi/TestHudiMergeRequiredColumns.java | 227 +++++++++++++ ...TestHudiNonProjectionCompatibleMerger.java | 192 +++++++++++ .../hudi/TestHudiPageSourceProviderTest.java | 23 +- .../hudi/TestHudiUtilColumnHandles.java | 303 +++++++++++++++++ .../hudi/TestPrefilledColumnValues.java | 182 ++++++++++ .../AbstractMergerHudiTablesInitializer.java | 291 ++++++++++++++++ .../CompositeHudiTablesInitializer.java | 47 +++ .../CustomMergerHudiTablesInitializer.java | 216 +++--------- ...ntalCustomMergerHudiTablesInitializer.java | 231 ++++--------- .../hudi/testing/MaxRankRecordMerger.java | 6 +- ...CompatibleMergerHudiTablesInitializer.java | 133 ++++++++ .../NonProjectionCompatibleRankMerger.java | 85 +++++ ...nProjectionCompatibleTestRecordMerger.java | 7 +- ...tedOrderingFieldHudiTablesInitializer.java | 126 +++++++ ...OmittedRankFieldHudiTablesInitializer.java | 135 ++++++++ ...ayloadOnlyMergerHudiTablesInitializer.java | 209 ++++++++++++ .../hudi/testing/RankBasedTestPayload.java | 79 +++++ 28 files changed, 2710 insertions(+), 855 deletions(-) create mode 100644 hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java delete mode 100644 hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java delete mode 100644 hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNonProjectionCompatibleMerger.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUtilColumnHandles.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CompositeHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleMergerHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleRankMerger.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedOrderingFieldHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedRankFieldHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/PayloadOnlyMergerHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/RankBasedTestPayload.java diff --git a/hudi-trino/pom.xml b/hudi-trino/pom.xml index 58f33855bbc03..a272cd1865a39 100644 --- a/hudi-trino/pom.xml +++ b/hudi-trino/pom.xml @@ -212,6 +212,11 @@ trino-hive + + io.trino + trino-hive-formats + + io.trino trino-memory-context @@ -416,12 +421,6 @@ runtime - - io.trino - trino-hive-formats - runtime - - org.jetbrains annotations diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java index 5564e2cdffc0e..b809f005a2dda 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java @@ -15,7 +15,7 @@ import com.google.common.collect.ImmutableList; import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hudi.util.SynthesizedColumnHandler; +import io.trino.plugin.hudi.util.PrefilledColumnValues; import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.connector.ConnectorPageSource; @@ -38,8 +38,8 @@ public class HudiBaseFileOnlyPageSource { private final ConnectorPageSource dataPageSource; private final List allOutputColumns; - private final SynthesizedColumnHandler synthesizedColumnHandler; - // Maps output channel to physical source channel, or -1 if synthesized + private final PrefilledColumnValues prefilledColumnValues; + // Maps output channel to physical source channel, or -1 if prefilled private final int[] physicalSourceChannelMap; public HudiBaseFileOnlyPageSource( @@ -47,12 +47,13 @@ public HudiBaseFileOnlyPageSource( List allOutputColumns, // Columns provided by dataPageSource List dataColumns, - // Handler to manage synthesized/virtual in Hudi tables such as partition columns and metadata, i.e. file size (not hudi metadata) - SynthesizedColumnHandler synthesizedColumnHandler) + // Per-split constant values for columns not present in the data file, such as partition + // columns and Trino's hidden metadata columns, e.g. file size (not hudi metadata) + PrefilledColumnValues prefilledColumnValues) { this.dataPageSource = requireNonNull(dataPageSource, "dataPageSource is null"); this.allOutputColumns = ImmutableList.copyOf(requireNonNull(allOutputColumns, "allOutputColumns is null")); - this.synthesizedColumnHandler = requireNonNull(synthesizedColumnHandler, "synthesizedColumnHandler is null"); + this.prefilledColumnValues = requireNonNull(prefilledColumnValues, "prefilledColumnValues is null"); // Create a mapping from the channel index in the output page to the channel index in the physicalDataPageSource's page this.physicalSourceChannelMap = new int[allOutputColumns.size()]; @@ -93,11 +94,6 @@ public SourcePage getNextSourcePage() } int positionCount = physicalSourcePage.getPositionCount(); - if (positionCount == 0 && synthesizedColumnHandler.getSynthesizedColumnCount() == 0) { - // If only physical columns and page is empty - return physicalSourcePage; - } - if (allOutputColumns.isEmpty()) { // Forward the zero-block page so positionCount survives -- new Page(new Block[0]) would infer positionCount=0. return physicalSourcePage; @@ -110,8 +106,8 @@ public SourcePage getNextSourcePage() outputBlocks[i] = physicalSourcePage.getBlock(physicalSourceChannelMap[i]); } else { - // Column is synthesized - outputBlocks[i] = synthesizedColumnHandler.createRleSynthesizedBlock(outputColumn, positionCount); + // Column is not in the data file; fill with the split's constant value + outputBlocks[i] = prefilledColumnValues.toRleBlock(outputColumn, positionCount); } } return SourcePage.create(new Page(outputBlocks)); diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java index 1d5eb13be1c87..ee7753edf34e1 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSource.java @@ -14,9 +14,8 @@ package io.trino.plugin.hudi; import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hudi.reader.HudiTrinoReaderContext; import io.trino.plugin.hudi.util.HudiAvroSerializer; -import io.trino.plugin.hudi.util.SynthesizedColumnHandler; +import io.trino.plugin.hudi.util.PrefilledColumnValues; import io.trino.spi.Page; import io.trino.spi.PageBuilder; import io.trino.spi.TrinoException; @@ -39,28 +38,23 @@ public class HudiPageSource implements ConnectorPageSource { - HoodieFileGroupReader fileGroupReader; - // TODO: Remove pageSource here, Hudi doesn't use this page source to read - ConnectorPageSource pageSource; - HudiTrinoReaderContext readerContext; - PageBuilder pageBuilder; - HudiAvroSerializer avroSerializer; - List columnHandles; - ClosableIterator recordIterator; + private final HoodieFileGroupReader fileGroupReader; + // Reads flow through fileGroupReader; pageSource is kept for stats/isBlocked delegation + private final ConnectorPageSource pageSource; + private final PageBuilder pageBuilder; + private final HudiAvroSerializer avroSerializer; + private final ClosableIterator recordIterator; public HudiPageSource( ConnectorPageSource pageSource, HoodieFileGroupReader fileGroupReader, - HudiTrinoReaderContext readerContext, List columnHandles, - SynthesizedColumnHandler synthesizedColumnHandler) + PrefilledColumnValues prefilledColumnValues) { this.pageSource = pageSource; this.fileGroupReader = fileGroupReader; - this.readerContext = readerContext; - this.columnHandles = columnHandles; this.pageBuilder = new PageBuilder(columnHandles.stream().map(HiveColumnHandle::getType).toList()); - this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler); + this.avroSerializer = new HudiAvroSerializer(columnHandles, prefilledColumnValues); try { this.recordIterator = fileGroupReader.getClosableIterator(); } diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index b8d0cec0de8ff..7ffca56717eff 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -37,7 +37,7 @@ import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hudi.file.HudiBaseFile; import io.trino.plugin.hudi.reader.HudiTrinoReaderContext; -import io.trino.plugin.hudi.util.SynthesizedColumnHandler; +import io.trino.plugin.hudi.util.PrefilledColumnValues; import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorPageSource; @@ -51,12 +51,15 @@ import io.trino.spi.predicate.TupleDomain; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.storage.StoragePath; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.io.MessageColumnIO; @@ -96,13 +99,19 @@ import static io.trino.plugin.hudi.HudiSessionProperties.isParquetVectorizedDecodingEnabled; import static io.trino.plugin.hudi.HudiSessionProperties.shouldUseParquetColumnNames; import static io.trino.plugin.hudi.HudiSessionProperties.useParquetBloomFilter; +import static io.trino.plugin.hudi.HudiUtil.appendMissingMergeRequiredColumns; +import static io.trino.plugin.hudi.HudiUtil.appendMissingSchemaColumns; import static io.trino.plugin.hudi.HudiUtil.buildTableMetaClient; import static io.trino.plugin.hudi.HudiUtil.constructSchema; import static io.trino.plugin.hudi.HudiUtil.convertToFileSlice; import static io.trino.plugin.hudi.HudiUtil.getLatestTableSchema; import static io.trino.plugin.hudi.HudiUtil.prependHudiMetaAndMergeRequiredColumns; +import static io.trino.plugin.hudi.HudiUtil.resolveMergeModeAndStrategyId; +import static io.trino.plugin.hudi.HudiUtil.usesNonProjectionCompatibleMerger; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY; import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; public class HudiPageSourceProvider @@ -192,33 +201,52 @@ public ConnectorPageSource createPageSource( .withBloomFilter(useParquetBloomFilter(session)) .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)) .build(); - ConnectorPageSource dataPageSource = createPageSource( - session, - isBaseFileOnly ? dataColumnHandles : hudiMetaAndDataColumnHandles, - hudiSplit, - fileSystem.newInputFile(Location.of(hudiBaseFileOpt.get().getPath()), hudiBaseFileOpt.get().getFileSize()), - hudiBaseFileOpt.get().getPath(), - start, - length, - OptionalLong.of(hudiBaseFileOpt.get().getFileSize()), - dataSourceStats, - sessionOptions, - timeZone, dynamicFilter, isBaseFileOnly); - - SynthesizedColumnHandler synthesizedColumnHandler = SynthesizedColumnHandler.create(hudiSplit); + PrefilledColumnValues prefilledColumnValues = PrefilledColumnValues.create(hudiSplit); // Avoid avro serialization if split/filegroup only contains base files if (isBaseFileOnly) { return new HudiBaseFileOnlyPageSource( - dataPageSource, + createBaseFilePageSource(session, dataColumnHandles, hudiSplit, fileSystem, sessionOptions, start, length, dynamicFilter, true), hiveColumnHandles, dataColumnHandles, - synthesizedColumnHandler); + prefilledColumnValues); + } + + // The merge path below is built around a base-file page source; fail log-only file slices with a + // clear error instead of an opaque NoSuchElementException from getBaseFile().orElseThrow(). + // TODO: support log-only file slices by feeding the file-group reader an empty base page source. + if (hudiBaseFileOpt.isEmpty()) { + throw new TrinoException(NOT_SUPPORTED, "Hudi splits with log files but no base file are not supported: " + + hudiSplit.getLogFiles().getFirst().getPath()); } // TODO: Move this into HudiTableHandle HoodieTableMetaClient metaClient = buildTableMetaClient( fileSystemFactory.create(session), hudiTableHandle.getSchemaTableName().toString(), hudiTableHandle.getBasePath()); + HoodieSchema dataSchema = + Optional.ofNullable(hudiTableHandle.getTableSchema()) + .orElseGet(() -> getLatestTableSchema(metaClient, hudiTableHandle.getTableName())); + TypedProperties readerProps = buildReaderProperties(session, metaClient); + + // A non-projection-compatible CUSTOM merger makes the file-group reader demand the FULL table + // schema as requiredSchema for this split (it has log files), for the base and log reads alike. + // Expand the read projection to the full schema up front so the base page source carries every + // merge column; the log page sources resolve their columns from the same expanded handles. + List readColumnHandles; + if (requiresFullSchemaRead(metaClient.getTableConfig(), readerProps)) { + log.debug("Expanding the read projection of %s to the full table schema: the resolved record merger is not projection compatible", + hudiTableHandle.getSchemaTableName()); + readColumnHandles = appendMissingSchemaColumns(dataSchema, hudiMetaAndDataColumnHandles); + } + else { + // The metastore may lack merge-required columns the table schema carries (e.g. hive sync with + // omit_metadata_fields=true drops _hoodie_operation); recover them from the already-resolved + // schema so the base read is not starved of them. + readColumnHandles = appendMissingMergeRequiredColumns(dataSchema, hudiMetaAndDataColumnHandles, metaClient.getTableConfig(), readerProps); + } + + ConnectorPageSource dataPageSource = + createBaseFilePageSource(session, readColumnHandles, hudiSplit, fileSystem, sessionOptions, start, length, dynamicFilter, false); // Build native (RFC-103) delta-log parquet page sources on demand, projected on the file-group // reader's requiredSchema with predicate pushdown OFF so every log record is read and merged. HudiTrinoReaderContext.LogFileParquetPageSourceFactory logPageSourceFactory = @@ -240,15 +268,11 @@ public ConnectorPageSource createPageSource( metaClient.getStorageConf(), metaClient.getTableConfig(), dataPageSource, - dataColumnHandles, - hudiMetaAndDataColumnHandles, - synthesizedColumnHandler, + readColumnHandles, + prefilledColumnValues, logPageSourceFactory); - HoodieSchema dataSchema = - Optional.ofNullable(hudiTableHandle.getTableSchema()) - .orElseGet(() -> getLatestTableSchema(metaClient, hudiTableHandle.getTableName())); - Schema requestedSchema = constructSchema(dataSchema.toAvroSchema(), hudiMetaAndDataColumnHandles.stream().map(HiveColumnHandle::getName).toList()); + Schema requestedSchema = constructSchema(dataSchema.toAvroSchema(), readColumnHandles.stream().map(HiveColumnHandle::getName).toList()); FileSlice fileSlice = convertToFileSlice(hudiSplit, hudiTableHandle.getBasePath()); HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.builder() @@ -260,7 +284,7 @@ public ConnectorPageSource createPageSource( .withDataSchema(dataSchema) .withRequestedSchema(HoodieSchema.fromAvroSchema(requestedSchema)) .withLatestCommitTime(hudiTableHandle.getLatestCommitTime()) - .withProps(buildReaderProperties(session, metaClient)) + .withProps(readerProps) .withShouldUseRecordPosition(false) .withStart(start) .withLength(length) @@ -268,9 +292,52 @@ public ConnectorPageSource createPageSource( return new HudiPageSource( dataPageSource, fileGroupReader, - readerContext, hiveColumnHandles, - synthesizedColumnHandler); + prefilledColumnValues); + } + + private ConnectorPageSource createBaseFilePageSource( + ConnectorSession session, + List columns, + HudiSplit hudiSplit, + TrinoFileSystem fileSystem, + ParquetReaderOptions sessionOptions, + long start, + long length, + DynamicFilter dynamicFilter, + boolean enablePredicatePushDown) + { + HudiBaseFile baseFile = hudiSplit.getBaseFile().orElseThrow(); + return createPageSource( + session, + columns, + hudiSplit, + fileSystem.newInputFile(Location.of(baseFile.getPath()), baseFile.getFileSize()), + baseFile.getPath(), + start, + length, + OptionalLong.of(baseFile.getFileSize()), + dataSourceStats, + sessionOptions, + timeZone, + dynamicFilter, + enablePredicatePushDown); + } + + /** + * Mirrors {@code FileGroupReaderSchemaHandler.generateRequiredSchema}'s full-schema decision for + * CUSTOM merge mode: resolves the merge mode and strategy id with the version-gated inference the + * file-group reader applies ({@link HudiUtil#resolveMergeModeAndStrategyId}) and asks the same + * resolved merger whether it is projection compatible. Only this decision is mirrored exactly; the + * mandatory-fields side ({@link HudiUtil#getMergeRequiredColumnHandles}) is a superset prediction, + * and the {@code HudiTrinoReaderContext.getFileRecordIterator} guard catches any residual drift. + */ + private static boolean requiresFullSchemaRead(HoodieTableConfig tableConfig, TypedProperties readerProps) + { + Pair mergeModeAndStrategyId = resolveMergeModeAndStrategyId(tableConfig); + String mergeImplClasses = readerProps.getString(RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY, + readerProps.getString(RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY, "")); + return usesNonProjectionCompatibleMerger(mergeModeAndStrategyId.getLeft(), mergeModeAndStrategyId.getRight(), mergeImplClasses); } /** @@ -397,6 +464,11 @@ private static TrinoException handleException(ParquetDataSourceId dataSourceId, * Creates a new list of ColumnHandles where the index associated with each handle corresponds to its physical position within the provided fileSchema (MessageType). * This is necessary when a downstream component relies on the handle's index for physical data access, and the logical schema order (potentially reflected in the * original handles) differs from the physical file layout. + *

    + * A requested column the file schema does not carry is mapped one past the last physical field instead of failing: base files written before the column was added + * legitimately lack it (schema evolution), and so do old records for a merge-required column. {@code ParquetPageSourceFactory} reads an index-based column through + * {@code getBaseColumnParquetType}, which reports any index at or beyond the file's field count as absent, so the parquet reader skips it and the page source emits + * a null block -- the same result the name-based path ({@code hudi.parquet.use-column-names=true}) produces. * * @param fileSchema The MessageType representing the physical schema of the Parquet file. * @param requestedColumns The original list of Trino ColumnHandles as received from the engine. @@ -415,7 +487,7 @@ public static List remapColumnIndicesToPhysical( for (int i = 0; i < fileFields.size(); i++) { Type field = fileFields.get(i); String fieldName = field.getName(); - String mapKey = caseSensitive ? fieldName : fieldName.toLowerCase(Locale.getDefault()); + String mapKey = caseSensitive ? fieldName : fieldName.toLowerCase(Locale.ROOT); physicalIndexMap.put(mapKey, i); } @@ -425,14 +497,15 @@ public static List remapColumnIndicesToPhysical( String requestedName = originalHandle.getBaseColumnName(); // Determine the key to use for looking up the physical index - String lookupKey = caseSensitive ? requestedName : requestedName.toLowerCase(Locale.getDefault()); + String lookupKey = caseSensitive ? requestedName : requestedName.toLowerCase(Locale.ROOT); - // Find the physical index from the file schema map constructed from fielSchema + // Find the physical index from the file schema map constructed from fileSchema. A column the file + // does not carry keeps an index one past the last field, which the parquet reader null-fills. Integer physicalIndex = physicalIndexMap.get(lookupKey); HiveColumnHandle remappedHandle = new HiveColumnHandle( requestedName, - physicalIndex, + physicalIndex == null ? fileFields.size() : physicalIndex, originalHandle.getBaseHiveType(), originalHandle.getType(), originalHandle.getHiveColumnProjectionInfo(), diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java index 27484ecc2d1c0..4d339e7d56c42 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.hudi; +import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.Weigher; import com.google.common.collect.ImmutableList; @@ -22,6 +23,8 @@ import io.trino.filesystem.FileIterator; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; +import io.trino.hive.formats.avro.AvroTypeException; +import io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeBlockHandler; import io.trino.metastore.Column; import io.trino.metastore.HivePartition; import io.trino.metastore.HiveType; @@ -30,6 +33,7 @@ import io.trino.plugin.hive.HivePartitionKey; import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.avro.AvroHiveFileUtils; +import io.trino.plugin.hive.util.HiveTypeTranslator; import io.trino.plugin.hudi.storage.HudiTrinoStorage; import io.trino.plugin.hudi.storage.TrinoStorageConfiguration; import io.trino.spi.TrinoException; @@ -38,6 +42,7 @@ import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.NullableValue; import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; import io.trino.spi.type.VarcharType; import org.apache.avro.Schema; @@ -54,14 +59,19 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.common.util.collection.Triple; import org.apache.hudi.exception.TableNotFoundException; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePath; @@ -70,6 +80,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -99,7 +110,12 @@ import static io.trino.plugin.hudi.HudiErrorCode.HUDI_SCHEMA_ERROR; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_UNSUPPORTED_FILE_FORMAT; import static java.lang.Math.toIntExact; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY; import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER; +import static org.apache.hudi.common.model.HoodieRecord.HOODIE_IS_DELETED_FIELD; +import static org.apache.hudi.common.model.HoodieRecord.OPERATION_METADATA_FIELD; import static org.apache.hudi.common.model.HoodieRecord.PARTITION_PATH_METADATA_FIELD; import static org.apache.hudi.common.model.HoodieRecord.RECORD_KEY_METADATA_FIELD; @@ -111,6 +127,8 @@ public final class HudiUtil CollectionUtils.createImmutableList(RECORD_KEY_METADATA_FIELD, PARTITION_PATH_METADATA_FIELD); private static final Logger log = Logger.get(HudiUtil.class); + // Maps Avro schemas (incl. logical types) to Trino types for toColumnHandle. Stateless, so shared. + private static final NativeLogicalTypesAvroTypeBlockHandler AVRO_TYPE_HANDLER = new NativeLogicalTypesAvroTypeBlockHandler(); private static final Cache> SCHEMA_FIELD_CACHE = EvictableCacheBuilder.newBuilder() .maximumWeight(10L * 1000L * 1024L) // 10MB @@ -348,6 +366,136 @@ public static Schema.Field getFieldFromSchema(String columnName, Schema schema) "Failed to get column " + columnName + " from table schema"); } + /** + * Builds a {@link HiveColumnHandle} for a table-schema field, typing it from the field's Avro schema. + * Used to resolve columns the file-group reader requires for merging but the connector projection does + * not carry (e.g. {@code _hoodie_commit_time} or a delete-marker column on a narrow query). + *

    + * The handle's {@code hiveColumnIndex} is a placeholder: page sources built from these handles resolve + * parquet columns by NAME (directly when {@code hudi.parquet.use-column-names=true}, otherwise + * {@link HudiPageSourceProvider#remapColumnIndicesToPhysical} rebuilds every index from the file + * schema by name), so the ordinal is never used to locate data. + *

    + * Avro types whose Trino mapping has no Hive counterpart (uuid, time-millis/micros) fail with + * NOT_SUPPORTED from {@link HiveTypeTranslator#toHiveType}; such columns are equally unreadable + * through the metastore schema, so this surfaces the same limitation with a clear error. + */ + public static HiveColumnHandle toColumnHandle(HoodieSchemaField field) + { + Type trinoType; + try { + trinoType = AVRO_TYPE_HANDLER.typeFor(field.schema().toAvroSchema()); + } + catch (AvroTypeException e) { + throw new TrinoException(HUDI_SCHEMA_ERROR, + "Failed to map Avro type of column " + field.name() + " to a Trino type", e); + } + return new HiveColumnHandle( + field.name(), + 0, + HiveTypeTranslator.toHiveType(trinoType), + trinoType, + Optional.empty(), + HiveColumnHandle.ColumnType.REGULAR, + Optional.empty()); + } + + /** + * Resolves the merge mode and merge strategy id the file-group reader will use for this table, + * applying hudi-common's version-gated inference: the merge MODE is inferred for any table below + * version 9 ({@code FileGroupReaderSchemaHandler.generateRequiredSchema}), while the STRATEGY ID + * the merger is resolved with is inferred only below version 8 + * ({@code HoodieReaderContext.initRecordMerger}). The asymmetry is deliberate and must mirror + * hudi-common exactly: pre-v9 tables may persist neither config (a 0.x table with a custom payload + * class infers CUSTOM mode with the payload-based strategy id), so a connector-side decision built + * on the raw configs would disagree with the file-group reader's. + */ + public static Pair resolveMergeModeAndStrategyId(HoodieTableConfig tableConfig) + { + RecordMergeMode mergeMode = tableConfig.getRecordMergeMode(); + String mergeStrategyId = tableConfig.getRecordMergeStrategyId(); + if (tableConfig.getTableVersion().lesserThan(HoodieTableVersion.NINE)) { + Triple inferred = HoodieTableConfig.inferMergingConfigsForPreV9Table( + tableConfig.getRecordMergeMode(), + tableConfig.getPayloadClass(), + tableConfig.getRecordMergeStrategyId(), + tableConfig.getOrderingFieldsStr().orElse(null), + tableConfig.getTableVersion()); + mergeMode = inferred.getLeft(); + if (tableConfig.getTableVersion().lesserThan(HoodieTableVersion.EIGHT)) { + mergeStrategyId = inferred.getRight(); + } + } + return Pair.of(mergeMode, mergeStrategyId); + } + + /** + * Returns whether reads of this table go through a CUSTOM record merger that is NOT projection + * compatible. For such mergers the file-group reader demands the FULL table schema as its required + * schema on any split with log files ({@code FileGroupReaderSchemaHandler.generateRequiredSchema}), + * so the connector must read every table column, not just the query projection. + *

    + * Callers must pass the merge mode and strategy id resolved by {@link #resolveMergeModeAndStrategyId} + * so the {@link HoodieRecordUtils#createValidRecordMerger} call here sees the same inputs as the + * file-group reader's own resolution. A CUSTOM mode without a strategy id returns false: no projection + * expansion is possible without a merger to ask, and {@code HudiTrinoReaderContext.getRecordMerger} + * rejects the read with an actionable error when the merger is actually needed -- hudi-common's + * {@link HoodieRecordUtils#createValidRecordMerger} would otherwise NPE on the null id. + *

    + * Note the payload-based strategy id resolves {@code HoodieAvroRecordMerger}, which keeps the + * interface default {@code isProjectionCompatible() == false} -- so EVERY custom-payload table takes + * the full-schema read path, not just tables with exotic custom mergers. That matches the file-group + * reader (and Spark); the extra I/O is inherent to payload-based merging. + */ + public static boolean usesNonProjectionCompatibleMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) + { + if (mergeMode != RecordMergeMode.CUSTOM || StringUtils.isNullOrEmpty(mergeStrategyId)) { + return false; + } + Option merger = HoodieRecordUtils.createValidRecordMerger(EngineType.JAVA, mergeImplClasses, mergeStrategyId); + return merger.isPresent() && !merger.get().isProjectionCompatible(); + } + + /** + * Rejects a CUSTOM-mode read that has no merge strategy id to resolve a record merger with. The + * strategy id is only inferred below table version 8 ({@link #resolveMergeModeAndStrategyId}) and + * passed through unchanged from there up, so any table at version 8 or later without a persisted id + * reaches merger resolution with a null id, which + * {@link HoodieRecordUtils#createValidRecordMerger} dereferences straight away. + */ + public static void validateCustomMergeStrategyId(String mergeStrategyId) + { + if (StringUtils.isNullOrEmpty(mergeStrategyId)) { + String strategyIdKey = HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key(); + throw new TrinoException(HUDI_BAD_DATA, + ("Table resolved to %s merge mode but persists no `%s`, so no record merger can be resolved for merging log files. " + + "Tables at version 8 or later written without a persisted strategy id resolve this way; set `%s` on the table to the " + + "strategy id declared by one of the configured record merger implementations.") + .formatted(RecordMergeMode.CUSTOM, strategyIdKey, strategyIdKey)); + } + } + + /** + * Returns {@code projection} extended with a handle for every {@code dataSchema} field it does not + * already carry (matched case-insensitively), typed from the field's Avro schema via + * {@link #toColumnHandle}. Projection handles keep their order and instances; missing fields are + * appended in schema order. + */ + public static List appendMissingSchemaColumns(HoodieSchema dataSchema, List projection) + { + Set existingColumns = projection.stream() + .map(handle -> handle.getName().toLowerCase(Locale.ROOT)) + .collect(Collectors.toCollection(HashSet::new)); + + List columns = new ArrayList<>(projection); + for (HoodieSchemaField field : dataSchema.getFields()) { + if (existingColumns.add(field.name().toLowerCase(Locale.ROOT))) { + columns.add(toColumnHandle(field)); + } + } + return columns; + } + public static List prependHudiMetaAndMergeRequiredColumns(HudiTableHandle tableHandle, List dataColumns) { Set existingColumns = dataColumns.stream() @@ -423,13 +571,15 @@ public static HoodieSchema getLatestTableSchema(HoodieTableMetaClient metaClient /** * Returns the column handles that must be present in the read schema for the file group reader to merge - * correctly: the ordering columns, plus the mandatory merge columns declared by a configured custom record - * merger (via {@link HoodieRecordMerger#getMandatoryFieldsForMerging}). + * correctly, mirroring {@code FileGroupReaderSchemaHandler.getMandatoryFieldsForMerging}: the ordering + * columns, the delete-marker and operation columns when the table carries them, the record-key data + * columns when meta fields are not populated, plus the mandatory merge columns declared by a configured + * custom record merger (via {@link HoodieRecordMerger#getMandatoryFieldsForMerging}). *

    - * For COMMIT_TIME / EVENT_TIME tables this is exactly the ordering columns (so behavior is unchanged). For a - * CUSTOM merge mode with a registered merger, it additionally includes any data columns the merger reads at - * merge time (e.g. an arbitrary decision column) so that those columns are read from the base file even when - * the query does not project them -- without this the merger would see null for an un-projected column. + * For a CUSTOM merge mode with a registered merger, it includes any data columns the merger reads at + * merge time (e.g. an arbitrary decision column) so that those columns are read from the base file even + * when the query does not project them -- without this the merger would see null for an un-projected + * column. */ public static List getMergeRequiredColumnHandles( Table table, @@ -440,19 +590,22 @@ public static List getMergeRequiredColumnHandles( { HoodieTableMetaClient metaClient = lazyMetaClient.get(); HoodieTableConfig tableConfig = metaClient.getTableConfig(); - RecordMergeMode recordMergeMode = tableConfig.getRecordMergeMode(); + // Resolve the merge mode/strategy exactly as the file-group reader does. Pre-v9 tables may persist + // neither config, so deciding on the raw values would e.g. miss the ordering columns of a 0.x + // event-time table entirely. + Pair mergeModeAndStrategyId = resolveMergeModeAndStrategyId(tableConfig); + RecordMergeMode recordMergeMode = mergeModeAndStrategyId.getLeft(); + String mergeStrategyId = mergeModeAndStrategyId.getRight(); - LinkedHashSet requiredColumnNames = new LinkedHashSet<>(); - if (recordMergeMode != null && recordMergeMode != RecordMergeMode.COMMIT_TIME_ORDERING) { - requiredColumnNames.addAll(tableConfig.getOrderingFields()); - } + LinkedHashSet requiredColumnNames = mergeRequiredColumnNames(tableConfig, recordMergeMode); // For a CUSTOM merge mode, ask the configured merger which fields it needs at merge time and include them // so they are read even when not projected. Only the merger's declared columns are added (not all columns), // so non-custom tables and mergers that only use the key/ordering fields incur no extra reads. - if (recordMergeMode == RecordMergeMode.CUSTOM && recordMergerImpls != null && !recordMergerImpls.isEmpty()) { + if (recordMergeMode == RecordMergeMode.CUSTOM && recordMergerImpls != null && !recordMergerImpls.isEmpty() + && !StringUtils.isNullOrEmpty(mergeStrategyId)) { Option merger = HoodieRecordUtils.createValidRecordMerger( - EngineType.JAVA, String.join(",", recordMergerImpls), tableConfig.getRecordMergeStrategyId()); + EngineType.JAVA, String.join(",", recordMergerImpls), mergeStrategyId); if (merger.isPresent()) { TypedProperties props = new TypedProperties(); props.putAll(tableConfig.getProps()); @@ -471,12 +624,94 @@ public static List getMergeRequiredColumnHandles( } } - if (requiredColumnNames.isEmpty()) { - return Collections.emptyList(); - } return buildColumnHandles(table, typeManager, requiredColumnNames, timestampPrecision); } + /** + * The merge-mandatory column names that do not depend on a custom merger, mirroring the non-CUSTOM + * branch of {@code FileGroupReaderSchemaHandler.getMandatoryFieldsForMerging}. The delete-marker and + * operation fields are added unconditionally: {@code buildColumnHandles} keeps only metastore data + * columns, and the merge read path recovers any name the metastore does not carry from the resolved + * table schema ({@link #appendMissingMergeRequiredColumns}). For those two fields that mirrors the + * file-group reader's own schema gate, so tables whose schema has neither read nothing extra; the + * ordering, record-key and delete-key names the reader adds without a schema check, but they are + * table columns whenever the table config is coherent. + */ + @VisibleForTesting + static LinkedHashSet mergeRequiredColumnNames(HoodieTableConfig tableConfig, RecordMergeMode recordMergeMode) + { + LinkedHashSet requiredColumnNames = new LinkedHashSet<>(); + if (recordMergeMode != null && recordMergeMode != RecordMergeMode.COMMIT_TIME_ORDERING) { + requiredColumnNames.addAll(tableConfig.getOrderingFields()); + } + // Without populated meta fields the file-group reader keys records on the record-key data columns + if (!tableConfig.populateMetaFields()) { + tableConfig.getRecordKeyFields().ifPresent(fields -> requiredColumnNames.addAll(Arrays.asList(fields))); + } + // Delete markers and the operation field decide record deletion at merge time + requiredColumnNames.add(HOODIE_IS_DELETED_FIELD); + requiredColumnNames.add(OPERATION_METADATA_FIELD); + String deleteKey = tableConfig.getProps().getProperty(DELETE_KEY); + String deleteMarker = tableConfig.getProps().getProperty(DELETE_MARKER); + // DeleteContext only honors a custom delete key when the marker value is also set + if (!StringUtils.isNullOrEmpty(deleteKey) && !StringUtils.isNullOrEmpty(deleteMarker)) { + requiredColumnNames.add(deleteKey); + } + return requiredColumnNames; + } + + /** + * Returns {@code projection} extended with handles for the merge-required column names it does not + * already carry (matched case-insensitively) and the metastore could not resolve, typed from + * {@code dataSchema} via {@link #toColumnHandle}: the table-config-derived names + * ({@link #mergeRequiredColumnNames}) plus, under a CUSTOM merge mode, the mandatory fields declared + * by the same resolved merger the file-group reader will use. Names absent from the schema as well are + * dropped -- for {@code _hoodie_is_deleted} and {@code _hoodie_operation} that mirrors the file-group + * reader's own schema gate; the rest are table columns whenever the table config is coherent. A + * metastore that omits a field the table schema carries -- hive sync with + * {@code omit_metadata_fields=true} drops {@code _hoodie_operation}, hand-written external DDL can drop + * any data column -- must not starve the base-file read of it; the + * {@code HudiTrinoReaderContext.getFileRecordIterator} guard would otherwise fail the read. Called on + * the merge read path only, where the table schema is already resolved, so the recovery costs no I/O. + */ + public static List appendMissingMergeRequiredColumns( + HoodieSchema dataSchema, + List projection, + HoodieTableConfig tableConfig, + TypedProperties readerProps) + { + Pair mergeModeAndStrategyId = resolveMergeModeAndStrategyId(tableConfig); + LinkedHashSet requiredNames = mergeRequiredColumnNames(tableConfig, mergeModeAndStrategyId.getLeft()); + // A CUSTOM merger can declare mandatory fields of its own (getMandatoryFieldsForMerging); ask the + // same resolved merger the file-group reader will use, against the same resolved schema, so its + // declarations are recovered too. Resolution is classloading only -- no I/O. + if (mergeModeAndStrategyId.getLeft() == RecordMergeMode.CUSTOM && !StringUtils.isNullOrEmpty(mergeModeAndStrategyId.getRight())) { + String mergeImplClasses = readerProps.getString(RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY, + readerProps.getString(RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY, "")); + Option merger = HoodieRecordUtils.createValidRecordMerger(EngineType.JAVA, mergeImplClasses, mergeModeAndStrategyId.getRight()); + if (merger.isPresent()) { + String[] mandatoryFields = merger.get().getMandatoryFieldsForMerging(dataSchema, tableConfig, readerProps); + if (mandatoryFields != null) { + Collections.addAll(requiredNames, mandatoryFields); + } + } + } + Set existingColumns = projection.stream() + .map(handle -> handle.getName().toLowerCase(Locale.ROOT)) + .collect(Collectors.toCollection(HashSet::new)); + Map schemaFields = dataSchema.getFields().stream() + .collect(Collectors.toMap(field -> field.name().toLowerCase(Locale.ROOT), field -> field, (first, second) -> first)); + + List columns = new ArrayList<>(projection); + for (String name : requiredNames) { + HoodieSchemaField field = schemaFields.get(name.toLowerCase(Locale.ROOT)); + if (field != null && existingColumns.add(field.name().toLowerCase(Locale.ROOT))) { + columns.add(toColumnHandle(field)); + } + } + return columns; + } + /** * Builds {@link HiveColumnHandle}s, preserving physical (data-column) index, for the data columns whose names * appear in {@code columnNames}. Names that are not data columns (e.g. Hudi meta fields) or whose types are not diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java index 60bb33eb5405a..57e8ad5d288c2 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java @@ -13,15 +13,15 @@ */ package io.trino.plugin.hudi.reader; -import io.trino.metastore.HiveType; +import com.google.common.collect.ImmutableSet; import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hudi.HudiUtil; import io.trino.plugin.hudi.util.HudiAvroSerializer; -import io.trino.plugin.hudi.util.SynthesizedColumnHandler; +import io.trino.plugin.hudi.util.PrefilledColumnValues; import io.trino.spi.Page; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorPageSource; import io.trino.spi.connector.SourcePage; -import io.trino.spi.type.VarcharType; import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.avro.AvroRecordContext; import org.apache.hudi.common.config.RecordMergeMode; @@ -29,7 +29,6 @@ import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieAvroRecordMerger; -import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.OverwriteWithLatestMerger; import org.apache.hudi.common.schema.HoodieSchema; @@ -51,23 +50,22 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; +import java.util.NoSuchElementException; +import java.util.Set; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.plugin.hudi.HudiErrorCode.HUDI_SCHEMA_ERROR; import static java.lang.String.format; import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; public class HudiTrinoReaderContext extends HoodieReaderContext { - ConnectorPageSource pageSource; + private final ConnectorPageSource pageSource; private final HudiAvroSerializer avroSerializer; - private final SynthesizedColumnHandler synthesizedColumnHandler; + private final PrefilledColumnValues prefilledColumnValues; private final LogFileParquetPageSourceFactory logPageSourceFactory; - Map colToPosMap; - Map colNameToHandle; - List dataHandles; - List columnHandles; + private final Map colNameToHandle; + private final Set baseProjectionNames; /** * Factory for building a Trino parquet page source over a native (RFC-103) delta-log file on demand. @@ -83,25 +81,23 @@ public HudiTrinoReaderContext( StorageConfiguration storageConfiguration, HoodieTableConfig tableConfig, ConnectorPageSource pageSource, - List dataHandles, List columnHandles, - SynthesizedColumnHandler synthesizedColumnHandler, + PrefilledColumnValues prefilledColumnValues, LogFileParquetPageSourceFactory logPageSourceFactory) { super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), new AvroRecordContext(tableConfig, tableConfig.getPayloadClass())); this.pageSource = pageSource; - this.synthesizedColumnHandler = synthesizedColumnHandler; - this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler); - this.dataHandles = dataHandles; - this.columnHandles = columnHandles; + this.prefilledColumnValues = prefilledColumnValues; + this.avroSerializer = new HudiAvroSerializer(columnHandles, prefilledColumnValues); this.logPageSourceFactory = logPageSourceFactory; - this.colToPosMap = new HashMap<>(); this.colNameToHandle = new HashMap<>(); - for (int i = 0; i < columnHandles.size(); i++) { - HiveColumnHandle handle = columnHandles.get(i); - colToPosMap.put(handle.getBaseColumnName(), i); + for (HiveColumnHandle handle : columnHandles) { colNameToHandle.put(handle.getBaseColumnName().toLowerCase(Locale.ROOT), handle); } + // Immutable snapshot of the read projection for the base-read guard in getFileRecordIterator; + // colNameToHandle cannot serve that purpose because buildRequiredColumnHandles adds log-side + // handles to it on demand. + this.baseProjectionNames = ImmutableSet.copyOf(colNameToHandle.keySet()); } @Override @@ -147,54 +143,42 @@ private ClosableIterator getFileRecordIterator( } List logProjection = buildRequiredColumnHandles(requiredSchema); ConnectorPageSource logSource = logPageSourceFactory.create(path.toString(), start, length, logProjection); - HudiAvroSerializer logSerializer = new HudiAvroSerializer(logProjection, synthesizedColumnHandler); + HudiAvroSerializer logSerializer = new HudiAvroSerializer(logProjection, prefilledColumnValues); return createRecordIterator(logSource, logSerializer); } + // The base read reuses the pre-built page source, so it can only satisfy requiredSchema fields the + // read projection carries. The connector predicts the file-group reader's demands up front + // (HudiUtil.getMergeRequiredColumnHandles, HudiPageSourceProvider.requiresFullSchemaRead); if the + // two ever drift, fail loudly here instead of silently merging with null column values. + List missingColumns = requiredSchema.getFields().stream() + .map(HoodieSchemaField::name) + .filter(name -> !baseProjectionNames.contains(name.toLowerCase(Locale.ROOT))) + .toList(); + if (!missingColumns.isEmpty()) { + throw new TrinoException(HUDI_SCHEMA_ERROR, format( + "The file-group reader requires columns %s for merging, but the base-file read projection " + + "does not carry them. The connector's merge projection is out of sync with " + + "FileGroupReaderSchemaHandler.generateRequiredSchema.", + missingColumns)); + } return createRecordIterator(pageSource, avroSerializer); } /** - * Resolves the {@link HiveColumnHandle} for each field of {@code requiredSchema} against the reader's column - * handles (keyed by lowercased base column name) so the on-demand log page source reads exactly the columns - * the file-group reader needs to merge. The file-group reader can add meta fields to {@code requiredSchema} - * that the connector projection does not carry -- the projection holds only the query columns plus - * {@code HUDI_REQUIRED_META_COLUMNS} (record key + partition path), whereas the merge may also need e.g. - * {@code _hoodie_commit_time}; for such a Hudi meta column a handle is synthesized (see the inline note) - * rather than failing the read. + * Resolves the {@link HiveColumnHandle} for each field of {@code requiredSchema} so the on-demand log + * page source reads exactly the columns the file-group reader needs to merge. Fields the connector + * projection carries resolve to their projection handles (planner-authoritative types); the file-group + * reader can also require fields the projection does not carry (e.g. {@code _hoodie_commit_time} or a + * delete-marker column on a narrow query) -- every such field originates from the table schema and + * carries its real Avro type, so its handle is typed directly from the field and cached. */ private List buildRequiredColumnHandles(HoodieSchema requiredSchema) { List handles = new ArrayList<>(); for (HoodieSchemaField field : requiredSchema.getFields()) { - String name = field.name(); - HiveColumnHandle handle = colNameToHandle.get(name.toLowerCase(Locale.ROOT)); - if (handle == null) { - if (!HoodieRecord.HOODIE_META_COLUMNS.contains(name)) { - // A data column outside the projection cannot be typed at this layer. The file-group reader - // only asks for one when the table's custom merger is not projection compatible (it then - // reads the FULL table schema), which this connector does not support. - throw new TrinoException(NOT_SUPPORTED, format( - "Column '%s' is required for merging but is not in the connector's read projection. " - + "This usually means the table's custom record merger is not projection compatible; " - + "the Hudi Trino connector requires custom mergers to override isProjectionCompatible() " - + "to true and declare getMandatoryFieldsForMerging().", name)); - } - // Synthesize a handle for a Hudi meta column absent from the connector projection. This is safe: - // 1. Every Hudi meta column is a UTF8 string on disk, so HIVE_STRING/VARCHAR is the correct - // physical type -- the same handle HudiUtil.prependHudiMetaAndMergeRequiredColumns builds for the - // HUDI_REQUIRED_META_COLUMNS. - // 2. hiveColumnIndex (0) is a throwaway placeholder: HudiPageSourceProvider.createPageSource - // resolves parquet columns by NAME (directly when useColumnNames=true, otherwise - // remapColumnIndicesToPhysical rebuilds every index from the file schema by name), so this - // ordinal is never read. - // TODO(apache/hudi#19249): remove this synthesis. Build colNameToHandle from the full data schema - // (all meta + data columns, typed once from the table schema) so every requiredSchema field - // resolves by lookup, dropping both the hand-built handle and the HOODIE_META_COLUMNS guard - // above -- which would also lift the projection-compatible-merger restriction. - handle = new HiveColumnHandle(name, 0, HiveType.HIVE_STRING, VarcharType.VARCHAR, - Optional.empty(), HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); - } - handles.add(handle); + handles.add(colNameToHandle.computeIfAbsent( + field.name().toLowerCase(Locale.ROOT), + _ -> HudiUtil.toColumnHandle(field))); } return handles; } @@ -243,8 +227,7 @@ public boolean hasNext() public IndexedRecord next() { if (!hasNext()) { - // TODO: This can probably be removed or ignored, added this as a sanity check - throw new RuntimeException("No more records in the iterator"); + throw new NoSuchElementException("No more records in the iterator"); } IndexedRecord record = serializer.serialize(currentPage, currentPosition); @@ -270,6 +253,9 @@ protected Option getRecordMerger(RecordMergeMode mergeMode, return Option.of(new OverwriteWithLatestMerger()); case CUSTOM: default: + // createValidRecordMerger dereferences the strategy id on its first line, so a table that + // resolved to CUSTOM without persisting one must be rejected before it reaches hudi-common. + HudiUtil.validateCustomMergeStrategyId(mergeStrategyId); Option recordMerger = HoodieRecordUtils.createValidRecordMerger(EngineType.JAVA, mergeImplClasses, mergeStrategyId); if (recordMerger.isEmpty()) { throw new IllegalArgumentException("No valid merger implementation set for `" + RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY + "`"); diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index 24c653511fe0d..4791507489286 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -107,20 +107,20 @@ public class HudiAvroSerializer }; private static final AvroDecimalConverter DECIMAL_CONVERTER = new AvroDecimalConverter(); - private final SynthesizedColumnHandler synthesizedColumnHandler; + private final PrefilledColumnValues prefilledColumnValues; private final List columnHandles; private final List columnTypes; private final Schema schema; - public HudiAvroSerializer(List columnHandles, SynthesizedColumnHandler synthesizedColumnHandler) + public HudiAvroSerializer(List columnHandles, PrefilledColumnValues prefilledColumnValues) { this.columnHandles = columnHandles; this.columnTypes = columnHandles.stream().map(HiveColumnHandle::getType).toList(); // Fetches projected schema this.schema = constructSchema(columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getName).toList(), columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getHiveType).toList()); - this.synthesizedColumnHandler = synthesizedColumnHandler; + this.prefilledColumnValues = prefilledColumnValues; } public IndexedRecord serialize(Page sourcePage, int position) @@ -145,8 +145,8 @@ public void buildRecordInPage(PageBuilder pageBuilder, IndexedRecord record) for (int channel = 0; channel < columnTypes.size(); channel++, blockSeq++) { BlockBuilder output = pageBuilder.getBlockBuilder(blockSeq); HiveColumnHandle columnHandle = columnHandles.get(channel); - if (synthesizedColumnHandler.isSynthesizedColumn(columnHandle)) { - synthesizedColumnHandler.getColumnStrategy(columnHandle).appendToBlock(output, columnTypes.get(channel)); + if (prefilledColumnValues.isPrefilled(columnHandle)) { + prefilledColumnValues.appendTo(columnHandle, output); } else { // Record may not be projected, get index from it diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java new file mode 100644 index 0000000000000..19c6c9d01322a --- /dev/null +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.util; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hive.HivePartitionKey; +import io.trino.plugin.hudi.HudiSplit; +import io.trino.plugin.hudi.file.HudiFile; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.RunLengthEncodedBlock; + +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +import static io.trino.metastore.Partitions.makePartName; +import static io.trino.plugin.hive.HiveColumnHandle.isFileModifiedTimeColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.isFileSizeColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.isPartitionColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.isPathColumnHandle; +import static io.trino.plugin.hive.util.HiveUtil.getPrefilledColumnValue; +import static io.trino.spi.type.TypeUtils.writeNativeValue; + +/** + * Per-split constant values for the output columns that are not stored in the data file: Hive-style + * partition columns and Trino's hidden metadata columns ({@code $path}, {@code $file_size}, + * {@code $file_modified_time}, {@code $partition}). Value computation delegates to + * {@link io.trino.plugin.hive.util.HiveUtil#getPrefilledColumnValue}, the same implementation the Hive + * connector uses for these columns (including the {@code "\N"} hive-null partition-value convention). + * Note {@code $file_modified_time} is packed with the JVM default zone, as in the Hive connector; the + * replaced Hudi-specific code pinned UTC (same instant, different rendered zone). + */ +public class PrefilledColumnValues +{ + private final Map partitionKeysByName; + private final String partitionName; + private final String filePath; + private final long fileSize; + private final long fileModifiedTime; + + public static PrefilledColumnValues create(HudiSplit hudiSplit) + { + return new PrefilledColumnValues(hudiSplit); + } + + private PrefilledColumnValues(HudiSplit hudiSplit) + { + List partitionKeys = hudiSplit.getPartitionKeys(); + // ImmutableMap preserves insertion order, so $partition renders the keys in the split's + // partition-column order. + ImmutableMap.Builder byName = ImmutableMap.builder(); + partitionKeys.forEach(partitionKey -> byName.put(partitionKey.name(), partitionKey)); + this.partitionKeysByName = byName.buildOrThrow(); + this.partitionName = makePartName( + partitionKeys.stream().map(HivePartitionKey::name).toList(), + partitionKeys.stream().map(HivePartitionKey::value).toList()); + // Parquet files will be prioritised over log files + HudiFile hudiFile = hudiSplit.getBaseFile().isPresent() + ? hudiSplit.getBaseFile().get() + : hudiSplit.getLogFiles().getFirst(); + this.filePath = hudiFile.getPath(); + this.fileSize = hudiFile.getFileSize(); + this.fileModifiedTime = hudiFile.getModificationTime(); + } + + /** + * Returns whether this split can provide a value for the column, i.e. it is a partition column of + * the split or a hidden metadata column Hudi populates. + */ + public boolean isPrefilled(HiveColumnHandle columnHandle) + { + return partitionKeysByName.containsKey(columnHandle.getName()) + || isPathColumnHandle(columnHandle) + || isFileSizeColumnHandle(columnHandle) + || isFileModifiedTimeColumnHandle(columnHandle) + || isPartitionColumnHandle(columnHandle); + } + + /** + * Appends the column's value for this split to the builder; appends null for a column this split + * cannot provide. + */ + public void appendTo(HiveColumnHandle columnHandle, BlockBuilder blockBuilder) + { + writeNativeValue(columnHandle.getType(), blockBuilder, nativeValueOf(columnHandle)); + } + + /** + * Builds a run-length-encoded {@link Block} repeating the column's constant value for + * {@code positionCount} positions (null-filled for a column this split cannot provide). + */ + public Block toRleBlock(HiveColumnHandle columnHandle, int positionCount) + { + return RunLengthEncodedBlock.create(columnHandle.getType(), nativeValueOf(columnHandle), positionCount); + } + + private Object nativeValueOf(HiveColumnHandle columnHandle) + { + if (!isPrefilled(columnHandle)) { + // Lenient null fill, e.g. for a hidden column Trino defines but Hudi does not populate. + return null; + } + HivePartitionKey partitionKey = partitionKeysByName.get(columnHandle.getName()); + return getPrefilledColumnValue( + columnHandle, + partitionKey, + filePath, + // Hudi tables are never hive-bucketed, so no $bucket value can be requested here + OptionalInt.empty(), + fileSize, + fileModifiedTime, + partitionName) + .getValue(); + } +} diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java deleted file mode 100644 index 5d55adb5577c5..0000000000000 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hudi.util; - -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hive.HivePartitionKey; -import io.trino.plugin.hudi.HudiSplit; -import io.trino.plugin.hudi.file.HudiFile; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.block.RunLengthEncodedBlock; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.SqlDecimal; -import io.trino.spi.type.TimestampWithTimeZoneType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.metastore.Partitions.makePartName; -import static io.trino.plugin.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME; -import static io.trino.plugin.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME; -import static io.trino.plugin.hive.HiveColumnHandle.PARTITION_COLUMN_NAME; -import static io.trino.plugin.hive.HiveColumnHandle.PATH_COLUMN_NAME; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone; -import static io.trino.spi.type.Decimals.writeBigDecimal; -import static io.trino.spi.type.Decimals.writeShortDecimal; -import static io.trino.spi.type.TimeZoneKey.UTC_KEY; -import static java.lang.Math.toIntExact; -import static java.lang.String.format; - -/** - * Handles synthesized (virtual) columns in Hudi tables, such as partition columns and metadata (not hudi metadata) - * columns. - */ -public class SynthesizedColumnHandler -{ - private final Map strategies; - private final SplitMetadata splitMetadata; - - public static SynthesizedColumnHandler create(HudiSplit hudiSplit) - { - return new SynthesizedColumnHandler(hudiSplit); - } - - /** - * Constructs a SynthesizedColumnHandler with the given partition keys. - */ - public SynthesizedColumnHandler(HudiSplit hudiSplit) - { - this.splitMetadata = SplitMetadata.of(hudiSplit); - ImmutableMap.Builder builder = ImmutableMap.builder(); - initSynthesizedColStrategies(builder); - initPartitionKeyStrategies(builder, hudiSplit); - strategies = builder.buildOrThrow(); - } - - /** - * Initializes strategies for synthesized columns. - */ - private void initSynthesizedColStrategies(ImmutableMap.Builder builder) - { - builder.put(PARTITION_COLUMN_NAME, (blockBuilder, _) -> - VarcharType.VARCHAR.writeSlice(blockBuilder, - utf8Slice(toPartitionName(splitMetadata.getPartitionKeyVals())))); - - builder.put(PATH_COLUMN_NAME, (blockBuilder, _) -> - VarcharType.VARCHAR.writeSlice(blockBuilder, utf8Slice(splitMetadata.getFilePath()))); - - builder.put(FILE_SIZE_COLUMN_NAME, (blockBuilder, _) -> - BigintType.BIGINT.writeLong(blockBuilder, splitMetadata.getFileSize())); - - builder.put(FILE_MODIFIED_TIME_COLUMN_NAME, (blockBuilder, _) -> { - long packedTimestamp = packDateTimeWithZone( - splitMetadata.getFileModificationTime(), UTC_KEY); - TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS.writeLong(blockBuilder, packedTimestamp); - }); - } - - /** - * Initializes strategies for partition columns. - */ - private void initPartitionKeyStrategies(ImmutableMap.Builder builder, - HudiSplit hudiSplit) - { - // Type is ignored here as input partitionKey.value() is always passed as a String type - for (HivePartitionKey partitionKey : hudiSplit.getPartitionKeys()) { - builder.put(partitionKey.name(), (blockBuilder, targetType) -> - appendPartitionKey(targetType, partitionKey.value(), blockBuilder)); - } - } - - /** - * Checks if a column is a synthesized column. - * - * @param columnName The column name. - * @return True if the column is synthesized, false otherwise. - */ - public boolean isSynthesizedColumn(String columnName) - { - return strategies.containsKey(columnName); - } - - /** - * Checks if a Hive column handle represents a synthesized column. - * - * @param columnHandle The Hive column handle. - * @return True if the column is synthesized, false otherwise. - */ - public boolean isSynthesizedColumn(HiveColumnHandle columnHandle) - { - return isSynthesizedColumn(columnHandle.getName()); - } - - /** - * Retrieves the strategy for a given synthesized column. - * - * @param columnHandle The Hive column handle. - * @return The corresponding column strategy, or null if not found. - */ - public SynthesizedColumnStrategy getColumnStrategy(HiveColumnHandle columnHandle) - { - return strategies.get(columnHandle.getName()); - } - - /** - * Retrieves the count of synthesized column strategies currently present. - * - * @return The number of synthesized column strategies. - */ - public int getSynthesizedColumnCount() - { - return strategies.size(); - } - - /** - * Converts partition key-value pairs into a partition name string. - * - * @param partitionKeyVals Map of partition key-value pairs. - * @return Partition name string. - */ - private static String toPartitionName(Map partitionKeyVals) - { - return makePartName(List.copyOf(partitionKeyVals.keySet()), List.copyOf(partitionKeyVals.values())); - } - - /** - * Creates a {@link Block} for the given synthesized column, typically a {@link RunLengthEncodedBlock} as the synthesized value is constant for all positions within a split. - * - * @param columnHandle The handle of the synthesized column to create a block for. - * @param positionCount The number of positions (rows) the resulting block should represent. - * @return A {@link Block} containing the synthesized values. - */ - public Block createRleSynthesizedBlock(HiveColumnHandle columnHandle, int positionCount) - { - Type columnType = columnHandle.getType(); - - if (positionCount == 0) { - return columnType.createBlockBuilder(null, 0).build(); - } - - SynthesizedColumnStrategy strategy = getColumnStrategy(columnHandle); - - // Because this builder will only hold the single constant value - int expectedEntriesForValueBlock = 1; - BlockBuilder valueBuilder = columnType.createBlockBuilder(null, expectedEntriesForValueBlock); - - if (strategy == null) { - valueBuilder.appendNull(); - } - else { - // Apply the strategy to write the single value into the builder - strategy.appendToBlock(valueBuilder, columnType); - } - Block valueBlock = valueBuilder.build(); - - return RunLengthEncodedBlock.create(valueBlock, positionCount); - } - - /** - * Represents metadata about split being processed. - * Splits are assumed to be in the same partition. - */ - public static class SplitMetadata - { - private final Map partitionKeyVals; - private final String filePath; - private final long fileSize; - private final long modifiedTime; - - /** - * Creates SplitMetadata from a Hudi split and partition key list. - */ - public static SplitMetadata of(HudiSplit hudiSplit) - { - return new SplitMetadata(hudiSplit); - } - - public SplitMetadata(HudiSplit hudiSplit) - { - this.partitionKeyVals = hudiSplit.getPartitionKeys().stream() - .collect(Collectors.toMap(HivePartitionKey::name, HivePartitionKey::value)); - // Parquet files will be prioritised over log files - HudiFile hudiFile = hudiSplit.getBaseFile().isPresent() - ? hudiSplit.getBaseFile().get() - : hudiSplit.getLogFiles().getFirst(); - this.filePath = hudiFile.getPath(); - this.fileSize = hudiFile.getFileSize(); - this.modifiedTime = hudiFile.getModificationTime(); - } - - public Map getPartitionKeyVals() - { - return partitionKeyVals; - } - - public String getFilePath() - { - return filePath; - } - - public long getFileSize() - { - return fileSize; - } - - public long getFileModificationTime() - { - return modifiedTime; - } - } - - /** - * Helper function to prefill BlockBuilders with values from PartitionKeys which are in the String type. - * This function handles the casting of String type the actual column type. - */ - private static void appendPartitionKey(Type targetType, String value, BlockBuilder blockBuilder) - { - if (value == null) { - blockBuilder.appendNull(); - return; - } - - if (targetType instanceof VarcharType varcharType) { - varcharType.writeSlice(blockBuilder, utf8Slice(value)); - } - else if (targetType instanceof IntegerType integerType) { - integerType.writeInt(blockBuilder, Integer.parseInt(value)); - } - else if (targetType instanceof BigintType bigintType) { - bigintType.writeLong(blockBuilder, Long.parseLong(value)); - } - else if (targetType instanceof BooleanType booleanType) { - booleanType.writeBoolean(blockBuilder, Boolean.parseBoolean(value)); - } - else if (targetType instanceof DecimalType decimalType) { - SqlDecimal sqlDecimal = SqlDecimal.decimal(value, decimalType); - BigDecimal bigDecimal = sqlDecimal.toBigDecimal(); - - if (decimalType.isShort()) { - // For short decimals, get the unscaled long value - // SqlDecimal.toBigDecimal() is used for consistency with the original SqlDecimal path - // The unscaled value of a Trino short decimal (precision <= 18) fits in a long - writeShortDecimal(blockBuilder, bigDecimal.unscaledValue().longValue()); - } - else { - // For long decimals, use the BigDecimal representation obtained from SqlDecimal. - writeBigDecimal(decimalType, blockBuilder, bigDecimal); - } - } - else if (targetType instanceof DateType dateType) { - try { - // Parse the date string using "YYYY-MM-DD" format - LocalDate localDate = LocalDate.parse(value); - // Convert LocalDate to days since epoch where LocalDate#toEpochDay() returns a long - int daysSinceEpoch = toIntExact(localDate.toEpochDay()); - dateType.writeInt(blockBuilder, daysSinceEpoch); - } - catch (DateTimeParseException e) { - // Handle parsing error - throw new TrinoException(GENERIC_INTERNAL_ERROR, - format("Invalid date string format for DATE type: '%s'. Expected format like YYYY-MM-DD. Details: %s", - value, e.getMessage()), e); - } - catch (ArithmeticException e) { - // Handle potential overflow if toEpochDay() result is outside int range - throw new TrinoException(GENERIC_INTERNAL_ERROR, - format("Date string '%s' results in a day count out of integer range for DATE type. Details: %s", - value, e.getMessage()), e); - } - } - else { - throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Unknown target type '%s' for column '%s'", targetType, value)); - } - } -} diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java deleted file mode 100644 index 916e6cb83e4fd..0000000000000 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnStrategy.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hudi.util; - -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; - -/** - * Strategy interface for handling different types of synthesized columns - */ -public interface SynthesizedColumnStrategy -{ - void appendToBlock(BlockBuilder blockBuilder, Type type); -} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java index 285d61eec70a7..76b4db658941c 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiCustomMerger.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * Verifies that the Hudi Trino connector resolves and applies a user-supplied custom record merger @@ -86,19 +85,21 @@ public void testRealtimeTableSumIsDistinctFromBaseAndNewestWins() } @Test - public void testNonProjectionCompatibleMergerIsRejected() + public void testNonProjectionCompatibleMergerMergesWithFullTableSchema() { - // A merger that is not projection compatible makes the file-group reader ask for the FULL table schema. - // The connector can only resolve columns in the read projection (plus Hudi meta columns), so a data - // column outside the projection must fail loudly rather than silently merge against a null value. + // A merger that is not projection compatible makes the file-group reader ask for the FULL table + // schema, so base and log reads must resolve data columns outside the query projection. + // sum(value) projects neither key nor name, yet 199 proves the key-based merge still ran: + // base-only would be 110 and the built-in newest-wins 104. Session session = SessionBuilder.from(getSession()) .withRecordMergerImpls(NonProjectionCompatibleTestRecordMerger.class.getName()) .build(); - // The guard throws inside the file-group reader and Hudi wraps it in a generic HoodieException - // ("Exception when reading log file"), but HudiPageSource rethrows the TrinoException from the - // cause chain, so the actionable text must be the top-level query failure message. - assertThatThrownBy(() -> computeScalar(session, "SELECT sum(value) FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME)) - .hasMessageContaining("is required for merging but is not in the connector's read projection") - .hasMessageContaining("requires custom mergers to override isProjectionCompatible()"); + assertThat(computeScalar(session, "SELECT sum(value) FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME)) + .isEqualTo(199L); + // Row-level check: the same merge decisions hold for every column. + assertQuery( + session, + "SELECT key, name, value FROM " + CustomMergerHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(99 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java new file mode 100644 index 0000000000000..f6a4ffb874d06 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java @@ -0,0 +1,227 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.metastore.HiveType; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hudi.testing.MaxRankRecordMerger; +import org.apache.avro.SchemaBuilder; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; +import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; +import static io.trino.plugin.hudi.HudiUtil.appendMissingMergeRequiredColumns; +import static io.trino.plugin.hudi.HudiUtil.mergeRequiredColumnNames; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.apache.hudi.common.config.HoodieReaderConfig.RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY; +import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER; +import static org.apache.hudi.common.model.HoodieRecord.HOODIE_IS_DELETED_FIELD; +import static org.apache.hudi.common.model.HoodieRecord.OPERATION_METADATA_FIELD; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests {@link HudiUtil#mergeRequiredColumnNames}, which mirrors the non-CUSTOM branch of the file-group + * reader's {@code FileGroupReaderSchemaHandler.getMandatoryFieldsForMerging} so the base-file read + * projection carries every column the merge may consult (ordering fields, delete markers, the operation + * field, record-key data columns) even when the query does not project them, plus the merge-path recovery + * of names the metastore could not resolve ({@link HudiUtil#appendMissingMergeRequiredColumns}). + */ +class TestHudiMergeRequiredColumns +{ + @Test + public void testOrderingFieldsFollowMergeMode() + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.ORDERING_FIELDS, "ts,seq"); + + assertThat(mergeRequiredColumnNames(tableConfig, RecordMergeMode.EVENT_TIME_ORDERING)) + .contains("ts", "seq"); + // Commit-time merging ignores ordering fields + assertThat(mergeRequiredColumnNames(tableConfig, RecordMergeMode.COMMIT_TIME_ORDERING)) + .doesNotContain("ts", "seq"); + assertThat(mergeRequiredColumnNames(tableConfig, null)) + .doesNotContain("ts", "seq"); + } + + @Test + public void testDeleteAndOperationColumnsAlwaysRequested() + { + // Requested unconditionally: getMergeRequiredColumnHandles keeps only metastore data columns, and + // the merge path's appendMissingMergeRequiredColumns recovers schema-carried names, so a table + // with these fields in neither reads nothing extra + assertThat(mergeRequiredColumnNames(new HoodieTableConfig(), RecordMergeMode.COMMIT_TIME_ORDERING)) + .contains(HOODIE_IS_DELETED_FIELD, OPERATION_METADATA_FIELD); + } + + @Test + public void testCustomDeleteKeyRequiresMarkerToo() + { + HoodieTableConfig withKeyAndMarker = new HoodieTableConfig(); + withKeyAndMarker.setValue(DELETE_KEY, "op"); + withKeyAndMarker.setValue(DELETE_MARKER, "D"); + assertThat(mergeRequiredColumnNames(withKeyAndMarker, RecordMergeMode.EVENT_TIME_ORDERING)) + .contains("op"); + + // DeleteContext only honors the delete key when the marker value is also set + HoodieTableConfig keyOnly = new HoodieTableConfig(); + keyOnly.setValue(DELETE_KEY, "op"); + assertThat(mergeRequiredColumnNames(keyOnly, RecordMergeMode.EVENT_TIME_ORDERING)) + .doesNotContain("op"); + } + + @Test + public void testRecordKeyFieldsRequestedWithoutPopulatedMetaFields() + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, "id1,id2"); + + // With populated meta fields (the default) the merge keys on _hoodie_record_key, which the + // connector always prepends into the projection + assertThat(mergeRequiredColumnNames(tableConfig, RecordMergeMode.EVENT_TIME_ORDERING)) + .doesNotContain("id1", "id2"); + + tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, "false"); + assertThat(mergeRequiredColumnNames(tableConfig, RecordMergeMode.EVENT_TIME_ORDERING)) + .contains("id1", "id2"); + } + + @Test + public void testMergeRequiredColumnMissingFromProjectionResolvesFromTableSchema() + { + // The metastore never resolved _hoodie_operation (hive sync with omit_metadata_fields=true), so the + // projection arrives on the merge path without it; it must be recovered from the resolved table + // schema -- the gate the file-group reader itself applies -- along with the ordering field, while + // _hoodie_is_deleted, in neither the projection nor the schema, stays dropped + HoodieSchema dataSchema = HoodieSchema.fromAvroSchema(SchemaBuilder.record("rec").fields() + .requiredString("id") + .requiredLong("ts") + .requiredString(OPERATION_METADATA_FIELD) + .endRecord()); + List projection = List.of( + createBaseColumn("id", 0, HiveType.HIVE_STRING, VARCHAR, REGULAR, Optional.empty())); + + List extended = appendMissingMergeRequiredColumns(dataSchema, projection, eventTimeOrderedOn("ts"), new TypedProperties()); + + assertThat(extended).extracting(HiveColumnHandle::getName) + .containsExactly("id", "ts", OPERATION_METADATA_FIELD); + // The recovered handles are typed from their Avro fields + assertThat(extended.get(1).getType()).isEqualTo(BIGINT); + assertThat(extended.getLast().getType()).isEqualTo(VARCHAR); + } + + @Test + public void testCustomMergerMandatoryFieldMissingFromProjectionResolvesFromTableSchema() + { + // MaxRankRecordMerger declares merge_rank mandatory; a metastore that does not carry the column + // leaves it out of the projection, and the merge path must recover it by asking the same resolved + // merger the file-group reader will use + assertThat(appendMissingMergeRequiredColumns( + rankTableSchema(), idOnlyProjection(), customMergeOrderedOn("ts", MaxRankRecordMerger.MERGE_STRATEGY_ID), maxRankMergerProps())) + .extracting(HiveColumnHandle::getName) + .containsExactly("id", "ts", MaxRankRecordMerger.RANK_COLUMN); + } + + @Test + public void testCustomMergeModeWithoutStrategyIdSkipsMergerResolution() + { + // A CUSTOM table without a persisted strategy id cannot resolve a merger; the append must fall + // back to the table-config-derived names without dereferencing the null id -- the read then fails + // actionably in getRecordMerger, not with an NPE here + assertThat(appendMissingMergeRequiredColumns( + rankTableSchema(), idOnlyProjection(), customMergeOrderedOn("ts", null), maxRankMergerProps())) + .extracting(HiveColumnHandle::getName) + .containsExactly("id", "ts"); + } + + @Test + public void testCustomMergeModeWithUnresolvableMergerSkipsMandatoryFields() + { + // A strategy id no configured merger implementation declares resolves nothing; the append keeps + // the table-config-derived names and must not dereference the empty resolution. (The all-zeros + // uuid would NOT do here: it is PAYLOAD_BASED_MERGE_STRATEGY_UUID, which always resolves.) + assertThat(appendMissingMergeRequiredColumns( + rankTableSchema(), idOnlyProjection(), customMergeOrderedOn("ts", "e2a5b7c9-1d3f-4a68-9c0b-5e7d9f1a3b6c"), maxRankMergerProps())) + .extracting(HiveColumnHandle::getName) + .containsExactly("id", "ts"); + } + + @Test + public void testMergeRequiredColumnAlreadyProjectedIsNotDuplicated() + { + HoodieSchema dataSchema = HoodieSchema.fromAvroSchema(SchemaBuilder.record("rec").fields() + .requiredString("id") + .requiredLong("ts") + .endRecord()); + // The projection carries the ordering field under a different case; the append must match + // case-insensitively and leave the projection handle untouched + List projection = List.of( + createBaseColumn("TS", 1, HiveType.HIVE_LONG, BIGINT, REGULAR, Optional.empty())); + + assertThat(appendMissingMergeRequiredColumns(dataSchema, projection, eventTimeOrderedOn("ts"), new TypedProperties())) + .extracting(HiveColumnHandle::getName) + .containsExactly("TS"); + } + + private static HoodieSchema rankTableSchema() + { + return HoodieSchema.fromAvroSchema(SchemaBuilder.record("rec").fields() + .requiredString("id") + .requiredLong("ts") + .requiredLong(MaxRankRecordMerger.RANK_COLUMN) + .endRecord()); + } + + private static List idOnlyProjection() + { + return List.of(createBaseColumn("id", 0, HiveType.HIVE_STRING, VARCHAR, REGULAR, Optional.empty())); + } + + private static HoodieTableConfig customMergeOrderedOn(String orderingField, String mergeStrategyId) + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "9"); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE, RecordMergeMode.CUSTOM.name()); + if (mergeStrategyId != null) { + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID, mergeStrategyId); + } + tableConfig.setValue(HoodieTableConfig.ORDERING_FIELDS, orderingField); + return tableConfig; + } + + private static TypedProperties maxRankMergerProps() + { + TypedProperties readerProps = new TypedProperties(); + readerProps.setProperty(RECORD_MERGE_IMPL_CLASSES_WRITE_CONFIG_KEY, MaxRankRecordMerger.class.getName()); + return readerProps; + } + + private static HoodieTableConfig eventTimeOrderedOn(String orderingField) + { + // Pin the version so the names under test are not at the mercy of the pre-v9 merge-config inference + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "9"); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE, RecordMergeMode.EVENT_TIME_ORDERING.name()); + tableConfig.setValue(HoodieTableConfig.ORDERING_FIELDS, orderingField); + return tableConfig; + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNonProjectionCompatibleMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNonProjectionCompatibleMerger.java new file mode 100644 index 0000000000000..8c239105d721b --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiNonProjectionCompatibleMerger.java @@ -0,0 +1,192 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.plugin.hudi.testing.CompositeHudiTablesInitializer; +import io.trino.plugin.hudi.testing.MaxRankRecordMerger; +import io.trino.plugin.hudi.testing.NonProjectionCompatibleMergerHudiTablesInitializer; +import io.trino.plugin.hudi.testing.NonProjectionCompatibleRankMerger; +import io.trino.plugin.hudi.testing.OmittedOrderingFieldHudiTablesInitializer; +import io.trino.plugin.hudi.testing.OmittedRankFieldHudiTablesInitializer; +import io.trino.plugin.hudi.testing.PayloadOnlyMergerHudiTablesInitializer; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static io.trino.plugin.hudi.testing.NonProjectionCompatibleMergerHudiTablesInitializer.RT_TABLE_NAME; +import static io.trino.plugin.hudi.testing.NonProjectionCompatibleMergerHudiTablesInitializer.TABLE_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Acceptance test for full-table-schema merge reads (apache/hudi#19249, issue comment on scope): + * {@link NonProjectionCompatibleRankMerger} does NOT override {@code isProjectionCompatible()} (default + * {@code false}) and does NOT declare {@code merge_rank} mandatory, so the file-group reader demands the + * FULL table schema as its required schema for base and log reads alike, and nothing prepends + * {@code merge_rank} into the connector's read projection. + *

    + * The queries below never project {@code merge_rank}. The data is laid out so each merge direction is + * proven independently: {@code k1}'s winning rank is on the LOG record (update wins, value 99) and + * {@code k2}'s winning rank is on the BASE record (base wins, value 100). A correct result therefore + * requires the un-projected rank column to be read on BOTH sides of the merge. {@code sum(value)} is a + * three-way discriminator: merged = 199, base-only = 110, built-in newest-wins = 103. + *

    + * The same full-schema read path is reached without any configured merger at all by a pre-1.0 table that + * persists only a {@link org.apache.hudi.common.model.HoodieRecordPayload} class + * ({@link PayloadOnlyMergerHudiTablesInitializer}), covered by the {@code payloadOnly} tests below. + *

    + * A third fixture ({@link OmittedOrderingFieldHudiTablesInitializer}) pins the projection-compatible side + * end to end: its metastore omits the ordering field its Avro schema carries, so correct event-time results + * prove the merge path recovers metastore-unknown merge columns from the resolved table schema. A fourth + * ({@link OmittedRankFieldHudiTablesInitializer}) does the same for a column only the CUSTOM merger itself + * declares mandatory ({@code merge_rank}), proving the merge path asks the resolved merger too. + */ +public class TestHudiNonProjectionCompatibleMerger + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return HudiQueryRunner.builder() + .setDataLoader(new CompositeHudiTablesInitializer( + new NonProjectionCompatibleMergerHudiTablesInitializer(), + new PayloadOnlyMergerHudiTablesInitializer(), + new OmittedOrderingFieldHudiTablesInitializer(), + new OmittedRankFieldHudiTablesInitializer())) + // Both custom mergers; each table resolves its own by strategy id. + .addConnectorProperties(ImmutableMap.of( + "hudi.record-merger-impls", + NonProjectionCompatibleRankMerger.class.getName() + "," + MaxRankRecordMerger.class.getName())) + .build(); + } + + @Test + public void testReadOptimizedTableReturnsBaseFileValues() + { + // The read-optimized table reads base files only, so it reflects the initial insert. + assertQuery( + "SELECT key, name, value FROM " + TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + assertThat(computeScalar("SELECT sum(value) FROM " + TABLE_NAME)) + .isEqualTo(110L); + } + + @Test + public void testNarrowProjectionMergesViaFullSchemaRead() + { + // Neither query projects merge_rank; the merger can only see it through the full-schema read. + // - k1 keeps the update (99): the winning rank (7 > 5) is on the LOG record + // - k2 keeps the base (100): the winning rank (9 > 1) is on the BASE record, proving the base + // read honors the file-group reader's full-schema required schema + assertQuery( + "SELECT key, value FROM " + RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', CAST(99 AS BIGINT)), ('k2', CAST(100 AS BIGINT))"); + // 199 uniquely identifies the rank-based merge: base-only would be 110, newest-wins would be 103. + assertThat(computeScalar("SELECT sum(value) FROM " + RT_TABLE_NAME)) + .isEqualTo(199L); + } + + @Test + public void testSelectStarMergesAllColumns() + { + assertQuery( + "SELECT key, name, value, merge_rank, ts FROM " + RT_TABLE_NAME + " ORDER BY key", + "VALUES" + + " ('k1', 'k1_updated', CAST(99 AS BIGINT), CAST(7 AS BIGINT), CAST(2 AS BIGINT))," + + " ('k2', 'k2_base', CAST(100 AS BIGINT), CAST(9 AS BIGINT), CAST(1 AS BIGINT))"); + } + + @Test + public void testPayloadOnlyReadOptimizedTableReturnsBaseFileValues() + { + assertQuery( + noRecordMergerImplsSession(), + "SELECT key, name, value FROM " + PayloadOnlyMergerHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + } + + @Test + public void testPayloadOnlyRealtimeTableAppliesPayloadMerge() + { + // Table version 6 with nothing but a payload class in hoodie.properties: the reader infers CUSTOM + // merge mode with the payload-based strategy, which resolves HoodieAvroRecordMerger and runs + // RankBasedTestPayload. Both merge directions are exercised, as for the custom merger above. + assertQuery( + noRecordMergerImplsSession(), + "SELECT key, name, value FROM " + PayloadOnlyMergerHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(99 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + } + + @Test + public void testPayloadOnlyNarrowProjectionMergesViaFullSchemaRead() + { + // Projects neither key, name nor merge_rank, so 199 proves the full-schema read fed merge_rank to the + // payload on both sides: base-only would be 110, built-in newest-wins would be 103. + assertThat(computeScalar( + noRecordMergerImplsSession(), + "SELECT sum(value) FROM " + PayloadOnlyMergerHudiTablesInitializer.RT_TABLE_NAME)) + .isEqualTo(199L); + } + + @Test + public void testMetastoreOmittedOrderingFieldRealtimeTableMerges() + { + // This fixture's metastore does not carry the ordering field ts its Avro schema has (the hive-sync + // omission shape); event-time merging still needs ts on both sides, so these results only come out + // when the merge path recovers the column from the resolved table schema. + assertQuery( + "SELECT key, name, value FROM " + OmittedOrderingFieldHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(99 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + } + + @Test + public void testMetastoreOmittedOrderingFieldNarrowProjectionSum() + { + // 199 discriminates event-time merging from base-only (110) and commit-time newest-wins (103). + assertThat(computeScalar("SELECT sum(value) FROM " + OmittedOrderingFieldHudiTablesInitializer.RT_TABLE_NAME)) + .isEqualTo(199L); + } + + @Test + public void testMetastoreOmittedMergerMandatoryFieldRealtimeTableMerges() + { + // This fixture's metastore does not carry merge_rank, the column MaxRankRecordMerger declares + // mandatory; only asking the resolved merger on the merge path recovers it, so these results only + // come out when merger-declared columns are recovered from the table schema too. + assertQuery( + "SELECT key, name, value FROM " + OmittedRankFieldHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(99 AS BIGINT)), ('k2', 'k2_base', CAST(100 AS BIGINT))"); + } + + @Test + public void testMetastoreOmittedMergerMandatoryFieldNarrowProjectionSum() + { + // 199 discriminates the keep-max merge from base-only (110) and newest-wins (103). + assertThat(computeScalar("SELECT sum(value) FROM " + OmittedRankFieldHudiTablesInitializer.RT_TABLE_NAME)) + .isEqualTo(199L); + } + + /** + * The payload-only table must resolve its merger purely from hoodie.properties, so the merger impls the + * connector is configured with for {@link NonProjectionCompatibleRankMerger} are cleared for its queries. + */ + private Session noRecordMergerImplsSession() + { + return SessionBuilder.from(getSession()) + .withRecordMergerImpls() + .build(); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java index 8966553939742..4bce0ba0cfc55 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java @@ -32,7 +32,6 @@ import static io.trino.spi.type.VarcharType.VARCHAR; import static org.apache.parquet.schema.Type.Repetition.OPTIONAL; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; class TestHudiPageSourceProviderTest { @@ -93,9 +92,13 @@ public void testRemapCaseSensitiveMismatch() createDummyHandle("COL_A", 0, HiveType.HIVE_INT, INTEGER), // This will mismatch createDummyHandle("col_b", 1, HiveType.HIVE_STRING, VARCHAR)); - // Perform remapping (case-sensitive) - Expect NPE because "COL_A" won't be found - assertThatThrownBy(() -> remapColumnIndicesToPhysical(fileSchema, requestedColumns, true)) - .isInstanceOf(NullPointerException.class); // Check the exception type + // Perform remapping (case-sensitive) - "COL_A" won't be found + List remapped = remapColumnIndicesToPhysical(fileSchema, requestedColumns, true); + + assertThat(remapped).hasSize(2); + // An unmatched column maps one past the last physical field, so the parquet reader null-fills it + assertHandle(remapped.get(0), "COL_A", fileSchema.getFieldCount(), HiveType.HIVE_INT, INTEGER); + assertHandle(remapped.get(1), "col_b", 1, HiveType.HIVE_STRING, VARCHAR); } @Test @@ -178,12 +181,16 @@ public void testRemapColumnNotFound() // Requested Columns (includes a non-existent column) List requestedColumns = List.of( createDummyHandle("col_a", 0, HiveType.HIVE_INT, INTEGER), - // Not in schema + // Not in schema, e.g. a base file written before the column was added createDummyHandle("col_x", 1, HiveType.HIVE_STRING, VARCHAR)); - // Perform remapping (case-insensitive) - Expect NPE because "col_x" won't be found - assertThatThrownBy(() -> remapColumnIndicesToPhysical(fileSchema, requestedColumns, false)) - .isInstanceOf(NullPointerException.class); + // Perform remapping (case-insensitive) - "col_x" won't be found + List remapped = remapColumnIndicesToPhysical(fileSchema, requestedColumns, false); + + assertThat(remapped).hasSize(2); + assertHandle(remapped.get(0), "col_a", 0, HiveType.HIVE_INT, INTEGER); + // Out of range on purpose: ParquetPageSourceFactory reports such a column as absent and null-fills it + assertHandle(remapped.get(1), "col_x", fileSchema.getFieldCount(), HiveType.HIVE_STRING, VARCHAR); } /** diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUtilColumnHandles.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUtilColumnHandles.java new file mode 100644 index 0000000000000..ee1fb1473ec13 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUtilColumnHandles.java @@ -0,0 +1,303 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.metastore.HiveType; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hudi.testing.MaxRankRecordMerger; +import io.trino.plugin.hudi.testing.NonProjectionCompatibleRankMerger; +import io.trino.spi.TrinoException; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Type; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.collection.Pair; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static io.trino.plugin.hudi.HudiUtil.appendMissingSchemaColumns; +import static io.trino.plugin.hudi.HudiUtil.resolveMergeModeAndStrategyId; +import static io.trino.plugin.hudi.HudiUtil.usesNonProjectionCompatibleMerger; +import static io.trino.plugin.hudi.HudiUtil.validateCustomMergeStrategyId; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.apache.hudi.common.model.HoodieRecordMerger.PAYLOAD_BASED_MERGE_STRATEGY_UUID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Tests {@link HudiUtil#toColumnHandle}, which types a {@link HiveColumnHandle} from a table-schema + * field's Avro type, plus the merge-resolution helpers around it ({@link + * HudiUtil#appendMissingSchemaColumns}, {@link HudiUtil#usesNonProjectionCompatibleMerger} and {@link + * HudiUtil#validateCustomMergeStrategyId}). Used by the reader context and page-source provider to + * resolve merge-required columns that are absent from the query projection (see apache/hudi#19249). + */ +class TestHudiUtilColumnHandles +{ + @Test + public void testNullableStringMatchesHudiMetaColumnHandle() + { + // Hudi meta columns (e.g. _hoodie_commit_time) are ["null","string"] unions; the resolved handle + // must be identical to the HIVE_STRING/VARCHAR handle prependHudiMetaAndOrderingColumns builds + Schema nullableString = SchemaBuilder.unionOf().nullType().and().stringType().endUnion(); + HiveColumnHandle handle = toColumnHandle("_hoodie_commit_time", nullableString); + + assertThat(handle.getName()).isEqualTo("_hoodie_commit_time"); + assertThat(handle.getType()).isEqualTo(VARCHAR); + assertThat(handle.getHiveType()).isEqualTo(HiveType.HIVE_STRING); + assertThat(handle.getColumnType()).isEqualTo(HiveColumnHandle.ColumnType.REGULAR); + assertThat(handle.isHidden()).isFalse(); + } + + @Test + public void testPrimitiveTypes() + { + assertColumnHandle(Schema.create(Schema.Type.INT), INTEGER, HiveType.HIVE_INT); + assertColumnHandle(Schema.create(Schema.Type.LONG), BIGINT, HiveType.HIVE_LONG); + assertColumnHandle(Schema.create(Schema.Type.BOOLEAN), BOOLEAN, HiveType.HIVE_BOOLEAN); + assertColumnHandle(Schema.create(Schema.Type.STRING), VARCHAR, HiveType.HIVE_STRING); + } + + @Test + public void testLogicalTypes() + { + assertColumnHandle( + LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)), + DATE, HiveType.HIVE_DATE); + assertColumnHandle( + LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)), + TIMESTAMP_MICROS, HiveType.HIVE_TIMESTAMP); + assertColumnHandle( + LogicalTypes.decimal(10, 2).addToSchema(Schema.create(Schema.Type.BYTES)), + DecimalType.createDecimalType(10, 2), HiveType.valueOf("decimal(10,2)")); + assertColumnHandle( + LogicalTypes.decimal(10, 2).addToSchema(Schema.createFixed("fixed_dec", null, null, 5)), + DecimalType.createDecimalType(10, 2), HiveType.valueOf("decimal(10,2)")); + } + + @Test + public void testNestedTypes() + { + HiveColumnHandle arrayHandle = toColumnHandle("arr", SchemaBuilder.array().items().stringType()); + assertThat(arrayHandle.getHiveType()).isEqualTo(HiveType.valueOf("array")); + + HiveColumnHandle mapHandle = toColumnHandle("map", SchemaBuilder.map().values().intType()); + assertThat(mapHandle.getHiveType()).isEqualTo(HiveType.valueOf("map")); + + HiveColumnHandle rowHandle = toColumnHandle("rec", SchemaBuilder.record("rec").fields() + .requiredInt("a") + .requiredString("b") + .endRecord()); + assertThat(rowHandle.getHiveType()).isEqualTo(HiveType.valueOf("struct")); + } + + @Test + public void testNullableUnionOfLogicalType() + { + Schema nullableDate = SchemaBuilder.unionOf().nullType().and() + .type(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))).endUnion(); + assertColumnHandle(nullableDate, DATE, HiveType.HIVE_DATE); + } + + @Test + public void testTypeWithoutHiveCounterpartThrows() + { + // Avro uuid maps to Trino UUID, which has no Hive counterpart; must fail with a clear error + Schema uuid = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)); + assertThatThrownBy(() -> toColumnHandle("id", uuid)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Unsupported Hive type"); + } + + @Test + public void testAppendMissingSchemaColumns() + { + HoodieSchema dataSchema = HoodieSchema.fromAvroSchema(SchemaBuilder.record("rec").fields() + .requiredString("col_a") + .requiredLong("col_b") + .requiredInt("col_c") + .endRecord()); + HiveColumnHandle projected = HiveColumnHandle.createBaseColumn( + "col_b", 1, HiveType.HIVE_LONG, BIGINT, HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); + + List expanded = appendMissingSchemaColumns(dataSchema, List.of(projected)); + + // Projection handles keep their order and instances; missing fields append in schema order + assertThat(expanded).hasSize(3); + assertThat(expanded.get(0)).isSameAs(projected); + assertThat(expanded.get(1).getName()).isEqualTo("col_a"); + assertThat(expanded.get(1).getType()).isEqualTo(VARCHAR); + assertThat(expanded.get(2).getName()).isEqualTo("col_c"); + assertThat(expanded.get(2).getType()).isEqualTo(INTEGER); + } + + @Test + public void testAppendMissingSchemaColumnsMatchesCaseInsensitively() + { + HoodieSchema dataSchema = HoodieSchema.fromAvroSchema(SchemaBuilder.record("rec").fields() + .requiredString("COL_A") + .endRecord()); + HiveColumnHandle projected = HiveColumnHandle.createBaseColumn( + "col_a", 0, HiveType.HIVE_STRING, VARCHAR, HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); + + assertThat(appendMissingSchemaColumns(dataSchema, List.of(projected))) + .containsExactly(projected); + } + + @Test + public void testUsesNonProjectionCompatibleMerger() + { + // Merger without the isProjectionCompatible override (interface default: false) -> full-schema read + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.CUSTOM, + NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID, + NonProjectionCompatibleRankMerger.class.getName())) + .isTrue(); + + // Projection-compatible merger stays on the projected fast path + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.CUSTOM, + MaxRankRecordMerger.MERGE_STRATEGY_ID, + MaxRankRecordMerger.class.getName())) + .isFalse(); + + // Unresolvable merger: no expansion here; the file-group reader fails loudly on its own + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.CUSTOM, + NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID, + "")) + .isFalse(); + + // Non-CUSTOM merge modes never trigger the full-schema read + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.EVENT_TIME_ORDERING, + NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID, + NonProjectionCompatibleRankMerger.class.getName())) + .isFalse(); + } + + @Test + public void testUsesNonProjectionCompatibleMergerWithoutStrategyId() + { + // A CUSTOM mode without a strategy id cannot resolve a merger: no expansion here (and no NPE from + // createValidRecordMerger); the file-group reader then fails loudly on its own + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.CUSTOM, null, NonProjectionCompatibleRankMerger.class.getName())) + .isFalse(); + assertThat(usesNonProjectionCompatibleMerger( + RecordMergeMode.CUSTOM, "", NonProjectionCompatibleRankMerger.class.getName())) + .isFalse(); + } + + @Test + public void testValidateCustomMergeStrategyId() + { + // A version 8 table can resolve to CUSTOM merge mode with no persisted strategy id; the read must be + // rejected with an actionable error rather than NPE inside createValidRecordMerger + assertThatThrownBy(() -> validateCustomMergeStrategyId(null)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key()); + assertThatThrownBy(() -> validateCustomMergeStrategyId("")) + .isInstanceOf(TrinoException.class) + .hasMessageContaining(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key()); + + assertThatCode(() -> validateCustomMergeStrategyId(NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID)) + .doesNotThrowAnyException(); + } + + @Test + public void testResolveMergeModeAndStrategyIdPassesThroughV9Configs() + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "9"); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE, RecordMergeMode.CUSTOM.name()); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID, NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID); + + Pair resolved = resolveMergeModeAndStrategyId(tableConfig); + assertThat(resolved.getLeft()).isEqualTo(RecordMergeMode.CUSTOM); + assertThat(resolved.getRight()).isEqualTo(NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID); + } + + @Test + public void testResolveMergeModeAndStrategyIdInfersPreV8Configs() + { + // A 0.x table with a custom payload class persists neither merge mode nor strategy id; both must + // be inferred, mirroring FileGroupReaderSchemaHandler.generateRequiredSchema (mode, below v9) and + // HoodieReaderContext.initRecordMerger (strategy id, below v8). The inferred payload-based + // strategy resolves HoodieAvroRecordMerger, which is not projection compatible, so such tables + // take the full-schema read path even with no merger impls configured. + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "6"); + tableConfig.setValue(HoodieTableConfig.PAYLOAD_CLASS_NAME, "com.example.CustomPayload"); + + Pair resolved = resolveMergeModeAndStrategyId(tableConfig); + assertThat(resolved.getLeft()).isEqualTo(RecordMergeMode.CUSTOM); + assertThat(resolved.getRight()).isEqualTo(PAYLOAD_BASED_MERGE_STRATEGY_UUID); + assertThat(usesNonProjectionCompatibleMerger(resolved.getLeft(), resolved.getRight(), "")).isTrue(); + } + + @Test + public void testResolveMergeModeAndStrategyIdInfersCommitTimeForDefaultPayload() + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "6"); + tableConfig.setValue(HoodieTableConfig.PAYLOAD_CLASS_NAME, OverwriteWithLatestAvroPayload.class.getName()); + + Pair resolved = resolveMergeModeAndStrategyId(tableConfig); + assertThat(resolved.getLeft()).isEqualTo(RecordMergeMode.COMMIT_TIME_ORDERING); + assertThat(usesNonProjectionCompatibleMerger(resolved.getLeft(), resolved.getRight(), "")).isFalse(); + } + + @Test + public void testResolveMergeModeAndStrategyIdKeepsRawStrategyIdForV8() + { + // Version 8 tables infer the merge MODE (the schema handler gates on below-9) but NOT the + // strategy id (initRecordMerger gates on below-8); the raw null strategy id must flow through + // un-inferred and be rejected by usesNonProjectionCompatibleMerger instead of NPE-ing + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "8"); + tableConfig.setValue(HoodieTableConfig.PAYLOAD_CLASS_NAME, "com.example.CustomPayload"); + + Pair resolved = resolveMergeModeAndStrategyId(tableConfig); + assertThat(resolved.getLeft()).isEqualTo(RecordMergeMode.CUSTOM); + assertThat(resolved.getRight()).isNull(); + assertThat(usesNonProjectionCompatibleMerger(resolved.getLeft(), resolved.getRight(), "")).isFalse(); + } + + private static void assertColumnHandle(Schema avroSchema, Type expectedType, HiveType expectedHiveType) + { + HiveColumnHandle handle = toColumnHandle("col", avroSchema); + assertThat(handle.getName()).isEqualTo("col"); + assertThat(handle.getType()).isEqualTo(expectedType); + assertThat(handle.getHiveType()).isEqualTo(expectedHiveType); + } + + private static HiveColumnHandle toColumnHandle(String name, Schema avroSchema) + { + return HudiUtil.toColumnHandle(HoodieSchemaField.of(name, HoodieSchema.fromAvroSchema(avroSchema))); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java new file mode 100644 index 0000000000000..4ee508cada0b0 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java @@ -0,0 +1,182 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.metastore.HiveType; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hive.HivePartitionKey; +import io.trino.plugin.hudi.file.HudiBaseFile; +import io.trino.plugin.hudi.util.PrefilledColumnValues; +import io.trino.spi.SplitWeight; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.RunLengthEncodedBlock; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Type; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; + +import static io.trino.plugin.hive.HiveColumnHandle.fileModifiedTimeColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.fileSizeColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.partitionColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.pathColumnHandle; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateTimeEncoding.unpackMillisUtc; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests {@link PrefilledColumnValues}: per-split constant values for partition columns and Trino's + * hidden metadata columns, delegated to trino-hive's canonical prefilled-column implementation. + */ +class TestPrefilledColumnValues +{ + private static final String FILE_PATH = "s3://bucket/table/year=2020/month=5/file1.parquet"; + private static final long FILE_SIZE = 1234; + private static final long FILE_MODIFIED_TIME = 1700000000123L; + + @Test + public void testPartitionKeyValues() + { + PrefilledColumnValues values = prefilledValues( + new HivePartitionKey("pk_string", "abc"), + new HivePartitionKey("pk_int", "42"), + new HivePartitionKey("pk_bigint", "123456789012"), + new HivePartitionKey("pk_date", "2021-12-09"), + new HivePartitionKey("pk_decimal", "12.34")); + + assertThat(VARCHAR.getSlice(singleValueBlock(values, partitionKey("pk_string", VARCHAR, HiveType.HIVE_STRING)), 0).toStringUtf8()) + .isEqualTo("abc"); + assertThat(INTEGER.getInt(singleValueBlock(values, partitionKey("pk_int", INTEGER, HiveType.HIVE_INT)), 0)) + .isEqualTo(42); + assertThat(BIGINT.getLong(singleValueBlock(values, partitionKey("pk_bigint", BIGINT, HiveType.HIVE_LONG)), 0)) + .isEqualTo(123456789012L); + assertThat(DATE.getInt(singleValueBlock(values, partitionKey("pk_date", DATE, HiveType.HIVE_DATE)), 0)) + .isEqualTo((int) LocalDate.parse("2021-12-09").toEpochDay()); + DecimalType decimalType = DecimalType.createDecimalType(10, 2); + assertThat(decimalType.getLong(singleValueBlock(values, partitionKey("pk_decimal", decimalType, HiveType.valueOf("decimal(10,2)"))), 0)) + .isEqualTo(1234); + } + + @Test + public void testHiveNullPartitionValue() + { + // Trino's HivePartitionKey encodes a null partition value as the literal string "\N" + PrefilledColumnValues values = prefilledValues(new HivePartitionKey("pk_string", "\\N")); + + Block block = singleValueBlock(values, partitionKey("pk_string", VARCHAR, HiveType.HIVE_STRING)); + assertThat(block.isNull(0)).isTrue(); + } + + @Test + public void testHiddenColumns() + { + PrefilledColumnValues values = prefilledValues( + new HivePartitionKey("year", "2020"), + new HivePartitionKey("month", "5")); + + assertThat(VARCHAR.getSlice(singleValueBlock(values, pathColumnHandle()), 0).toStringUtf8()) + .isEqualTo(FILE_PATH); + assertThat(BIGINT.getLong(singleValueBlock(values, fileSizeColumnHandle()), 0)) + .isEqualTo(FILE_SIZE); + long packedTimestamp = TIMESTAMP_TZ_MILLIS.getLong(singleValueBlock(values, fileModifiedTimeColumnHandle()), 0); + assertThat(unpackMillisUtc(packedTimestamp)).isEqualTo(FILE_MODIFIED_TIME); + } + + @Test + public void testPartitionNamePreservesKeyOrder() + { + // Keys must render in the split's partition-column order, not e.g. hash order + PrefilledColumnValues values = prefilledValues( + new HivePartitionKey("year", "2020"), + new HivePartitionKey("month", "5"), + new HivePartitionKey("day", "17")); + + assertThat(VARCHAR.getSlice(singleValueBlock(values, partitionColumnHandle()), 0).toStringUtf8()) + .isEqualTo("year=2020/month=5/day=17"); + } + + @Test + public void testUnknownColumnIsNullFilled() + { + PrefilledColumnValues values = prefilledValues(); + HiveColumnHandle dataColumn = HiveColumnHandle.createBaseColumn( + "some_col", 0, HiveType.HIVE_STRING, VARCHAR, HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); + + assertThat(values.isPrefilled(dataColumn)).isFalse(); + Block block = values.toRleBlock(dataColumn, 3); + assertThat(block.getPositionCount()).isEqualTo(3); + assertThat(block.isNull(0)).isTrue(); + } + + @Test + public void testRleBlockShape() + { + PrefilledColumnValues values = prefilledValues(new HivePartitionKey("year", "2020")); + HiveColumnHandle handle = partitionKey("year", VARCHAR, HiveType.HIVE_STRING); + + assertThat(values.isPrefilled(handle)).isTrue(); + Block block = values.toRleBlock(handle, 5); + assertThat(block).isInstanceOf(RunLengthEncodedBlock.class); + assertThat(block.getPositionCount()).isEqualTo(5); + assertThat(VARCHAR.getSlice(block, 4).toStringUtf8()).isEqualTo("2020"); + + assertThat(values.toRleBlock(handle, 0).getPositionCount()).isEqualTo(0); + } + + @Test + public void testAppendTo() + { + PrefilledColumnValues values = prefilledValues(new HivePartitionKey("pk_int", "42")); + HiveColumnHandle handle = partitionKey("pk_int", INTEGER, HiveType.HIVE_INT); + + BlockBuilder blockBuilder = INTEGER.createBlockBuilder(null, 2); + values.appendTo(handle, blockBuilder); + values.appendTo(handle, blockBuilder); + Block block = blockBuilder.build(); + + assertThat(block.getPositionCount()).isEqualTo(2); + assertThat(INTEGER.getInt(block, 1)).isEqualTo(42); + } + + private static PrefilledColumnValues prefilledValues(HivePartitionKey... partitionKeys) + { + HudiBaseFile baseFile = new HudiBaseFile(FILE_PATH, "file1.parquet", FILE_SIZE, FILE_MODIFIED_TIME, 0, FILE_SIZE); + HudiSplit split = new HudiSplit( + baseFile, + List.of(), + "001", + TupleDomain.all(), + List.of(partitionKeys), + SplitWeight.standard()); + return PrefilledColumnValues.create(split); + } + + private static HiveColumnHandle partitionKey(String name, Type type, HiveType hiveType) + { + return HiveColumnHandle.createBaseColumn(name, -1, hiveType, type, HiveColumnHandle.ColumnType.PARTITION_KEY, Optional.empty()); + } + + private static Block singleValueBlock(PrefilledColumnValues values, HiveColumnHandle handle) + { + return values.toRleBlock(handle, 1); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java new file mode 100644 index 0000000000000..045e8a3386eaf --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java @@ -0,0 +1,291 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Column; +import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.StorageFormat; +import io.trino.metastore.Table; +import io.trino.plugin.hudi.HudiConnector; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.QueryRunner; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.common.HoodieJavaEngineContext; +import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; +import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static java.nio.file.Files.createTempDirectory; +import static java.util.Objects.requireNonNull; + +/** + * Shared machinery for the non-partitioned Merge-On-Read fixtures that exist to exercise record merging at + * read time. The table is written by the Hudi Java write client into a local staging directory and then + * mirrored into the Trino filesystem the connector reads from; two metastore tables are registered against + * that location, a read-optimized one (base files only) and a real-time one (suffix {@code _rt}) that merges + * base + log files through the file group reader. + *

    + * Inline compaction is off for every fixture so the log files written by the delta commits survive and must + * be merged at read time, and the metadata table is off because MDT writes need hbase dependencies that are + * not on the Trino test classpath. + *

    + * Subclasses supply the schema, the merge-related table and write configuration, and the commits; everything + * a fixture does not vary lives here. + */ +public abstract class AbstractMergerHudiTablesInitializer + implements HudiTablesInitializer +{ + /** Every fixture built on this base is keyed on {@code key} and ordered by {@code ts}, in a single unnamed partition. */ + protected static final String RECORD_KEY_FIELD = "key"; + protected static final String ORDERING_FIELD = "ts"; + + private static final String PARTITION_PATH = ""; + + /** Hudi metadata columns, prepended to every table's data columns in the metastore definition. */ + private static final List HUDI_META_COLUMNS = ImmutableList.of( + new Column("_hoodie_commit_time", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_commit_seqno", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_record_key", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_partition_path", HIVE_STRING, Optional.empty(), Map.of()), + new Column("_hoodie_file_name", HIVE_STRING, Optional.empty(), Map.of())); + + private final String tableName; + + private TrinoFileSystem fileSystem; + private Location tableLocation; + private java.nio.file.Path stagingDir; + private Path stagingTablePath; + private HoodieJavaWriteClient writeClient; + + protected AbstractMergerHudiTablesInitializer(String tableName) + { + this.tableName = requireNonNull(tableName, "tableName is null"); + } + + @Override + public final void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) + throws Exception + { + fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(TrinoFileSystemFactory.class) + .create(ConnectorIdentity.ofUser("test")); + HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(HiveMetastoreFactory.class) + .createMetastore(Optional.empty()); + + tableLocation = externalLocation.appendPath(tableName); + stagingDir = createTempDirectory(tableName.replace('_', '-')); + stagingTablePath = new Path(stagingDir.resolve(tableName).toUri()); + + boolean initialized = false; + try { + initTable(); + afterTableInit(); + writeClient = createWriteClient(); + writeInitialCommits(writeClient); + syncToTrino(); + + metastore.createTable(createTableDefinition(schemaName, tableName, false), PrincipalPrivileges.NO_PRIVILEGES); + metastore.createTable(createTableDefinition(schemaName, tableName + "_rt", true), PrincipalPrivileges.NO_PRIVILEGES); + initialized = true; + } + finally { + // Only fixtures that keep writing commits after initialization need the staging directory and the + // write client to outlive this call; they are responsible for calling close(). + if (!initialized || !keepsWriterOpen()) { + close(); + } + } + } + + /** The table's data columns, in schema order; the Hudi metadata columns are prepended by this class. */ + protected abstract List dataColumns(); + + /** The Avro schema the write client writes, matching {@link #dataColumns()}. */ + protected abstract Schema avroSchema(); + + /** + * Applies the fixture's merge-related table configuration (merge mode, merge strategy id, payload class). + * The table type, record key fields and ordering fields are set by this class. + */ + protected abstract void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder); + + /** Applies the fixture's merge-related write configuration (merge mode, merge strategy id, merger impl classes). */ + protected abstract void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder); + + /** Writes the commits that make up the fixture's initial state. */ + protected abstract void writeInitialCommits(HoodieJavaWriteClient client) + throws IOException; + + /** Runs on the freshly created table, before the write client exists and before any data is written. */ + protected void afterTableInit() + throws IOException {} + + /** Inline compaction is disabled, so this only has to stay above the number of delta commits a fixture writes. */ + protected int maxDeltaCommitsBeforeCompaction() + { + return 100; + } + + /** Whether the staging directory and write client survive {@link #initializeTables}; such fixtures must call {@link #close()}. */ + protected boolean keepsWriterOpen() + { + return false; + } + + public void close() + throws IOException + { + if (writeClient != null) { + writeClient.close(); + writeClient = null; + } + if (stagingDir != null) { + deleteRecursively(stagingDir, ALLOW_INSECURE); + stagingDir = null; + } + } + + /** Local directory the write client writes into, before {@link #syncToTrino()} mirrors it to the connector. */ + protected java.nio.file.Path stagingTableDirectory() + { + return stagingDir.resolve(tableName); + } + + protected HoodieJavaWriteClient writeClient() + { + return writeClient; + } + + protected static HoodieRecord avroRecord(GenericRecord record, String key) + { + return new HoodieAvroRecord<>(new HoodieKey(key, PARTITION_PATH), new HoodieAvroPayload(Option.of(record)), null); + } + + /** Mirrors the staged table into the Trino filesystem so the connector observes the commits written so far. */ + protected void syncToTrino() + { + try { + if (fileSystem.directoryExists(tableLocation).orElse(false)) { + fileSystem.deleteDirectory(tableLocation); + } + ResourceHudiTablesInitializer.copyDir(stagingTableDirectory(), fileSystem, tableLocation); + } + catch (IOException e) { + throw new RuntimeException("Failed to sync staged Hudi table to Trino filesystem", e); + } + } + + private void initTable() + { + HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.MERGE_ON_READ) + .setTableName(tableName) + .setTimelineLayoutVersion(1) + .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) + .setRecordKeyFields(RECORD_KEY_FIELD) + .setOrderingFields(ORDERING_FIELD); + configureTableConfig(tableBuilder); + try { + tableBuilder.initTable(new HadoopStorageConfiguration(new Configuration()), stagingTablePath.toString()); + } + catch (IOException e) { + throw new RuntimeException("Could not init table " + tableName, e); + } + } + + private HoodieJavaWriteClient createWriteClient() + { + Configuration conf = new Configuration(); + HoodieWriteConfig.Builder writeConfigBuilder = HoodieWriteConfig.newBuilder() + .withPath(stagingTablePath.toString()) + .withSchema(avroSchema().toString()) + .withParallelism(2, 2) + .withDeleteParallelism(2) + .forTable(tableName) + // No withPreCombineField here: the ordering field is carried by the table config + // (setOrderingFields), and the deprecated builder method fails the -Werror compile gate. + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + // Keep log files around so merging runs at read time. + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .withMaxNumDeltaCommitsBeforeCompaction(maxDeltaCommitsBeforeCompaction()) + .build()) + .withEmbeddedTimelineServerEnabled(false) + .withMarkersType(MarkerType.DIRECT.name()) + // MDT writes require hbase deps not present in the Trino runtime. + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()); + configureWriteConfig(writeConfigBuilder); + return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), writeConfigBuilder.build()); + } + + private Table createTableDefinition(String schemaName, String metastoreTableName, boolean isRtTable) + { + StorageFormat storageFormat = StorageFormat.create( + PARQUET_HIVE_SERDE_CLASS, + isRtTable ? HUDI_PARQUET_REALTIME_INPUT_FORMAT : HUDI_PARQUET_INPUT_FORMAT, + MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); + + return Table.builder() + .setDatabaseName(schemaName) + .setTableName(metastoreTableName) + .setTableType(EXTERNAL_TABLE.name()) + .setOwner(Optional.of("public")) + .setDataColumns(ImmutableList.builder() + .addAll(HUDI_META_COLUMNS) + .addAll(dataColumns()) + .build()) + .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) + .withStorage(storageBuilder -> storageBuilder + .setStorageFormat(storageFormat) + .setLocation(tableLocation.toString())) + .build(); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CompositeHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CompositeHudiTablesInitializer.java new file mode 100644 index 0000000000000..3fa4b7176f2d3 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CompositeHudiTablesInitializer.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.filesystem.Location; +import io.trino.testing.QueryRunner; + +import java.util.List; + +/** + * Runs several {@link HudiTablesInitializer}s against one query runner, in order, so a single test class can + * load more than one table fixture. Each delegate must create its own tables under a distinct name, and must + * finish its writes within {@code initializeTables}: a fixture that keeps its write client open (see + * {@code AbstractMergerHudiTablesInitializer.keepsWriterOpen}) cannot be composed here, because + * {@link HudiTablesInitializer} declares no close for this class to forward. + */ +public class CompositeHudiTablesInitializer + implements HudiTablesInitializer +{ + private final List delegates; + + public CompositeHudiTablesInitializer(HudiTablesInitializer... delegates) + { + this.delegates = ImmutableList.copyOf(delegates); + } + + @Override + public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) + throws Exception + { + for (HudiTablesInitializer delegate : delegates) { + delegate.initializeTables(queryRunner, externalLocation, schemaName); + } + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java index 1c5387419c5b3..b107cde6741a0 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CustomMergerHudiTablesInitializer.java @@ -14,60 +14,25 @@ package io.trino.plugin.hudi.testing; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.trino.filesystem.Location; -import io.trino.filesystem.TrinoFileSystem; -import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.Column; -import io.trino.metastore.HiveMetastore; -import io.trino.metastore.HiveMetastoreFactory; -import io.trino.metastore.PrincipalPrivileges; -import io.trino.metastore.StorageFormat; -import io.trino.metastore.Table; -import io.trino.plugin.hudi.HudiConnector; -import io.trino.spi.security.ConnectorIdentity; -import io.trino.testing.QueryRunner; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.client.HoodieJavaWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.client.common.HoodieJavaEngineContext; -import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; -import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.model.HoodieAvroPayload; -import org.apache.hudi.common.model.HoodieAvroRecord; -import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.marker.MarkerType; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.index.HoodieIndex; -import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; -import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; -import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; -import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; import static io.trino.metastore.HiveType.HIVE_LONG; import static io.trino.metastore.HiveType.HIVE_STRING; -import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; -import static java.nio.file.Files.createTempDirectory; /** * Creates a non-partitioned Merge-On-Read table at test runtime that is configured to use a custom record @@ -82,118 +47,74 @@ * built-in newest-wins behavior (see {@code TestHudiCustomMerger}). */ public class CustomMergerHudiTablesInitializer - implements HudiTablesInitializer + extends AbstractMergerHudiTablesInitializer { public static final String TABLE_NAME = "custom_merger_mor"; public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; - private static final String RECORD_KEY_FIELD = "key"; - private static final String ORDERING_FIELD = "ts"; - private static final String PARTITION_PATH = ""; - - private static final List DATA_COLUMNS = ImmutableList.of( - new Column("_hoodie_commit_time", HIVE_STRING, Optional.empty(), Map.of()), - new Column("_hoodie_commit_seqno", HIVE_STRING, Optional.empty(), Map.of()), - new Column("_hoodie_record_key", HIVE_STRING, Optional.empty(), Map.of()), - new Column("_hoodie_partition_path", HIVE_STRING, Optional.empty(), Map.of()), - new Column("_hoodie_file_name", HIVE_STRING, Optional.empty(), Map.of()), - new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), - new Column("name", HIVE_STRING, Optional.empty(), Map.of()), - new Column("value", HIVE_LONG, Optional.empty(), Map.of()), - new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + public CustomMergerHudiTablesInitializer() + { + super(TABLE_NAME); + } @Override - public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) - throws Exception + protected List dataColumns() { - TrinoFileSystem fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() - .getInstance(TrinoFileSystemFactory.class) - .create(ConnectorIdentity.ofUser("test")); - HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() - .getInstance(HiveMetastoreFactory.class) - .createMetastore(Optional.empty()); - - Location tableLocation = externalLocation.appendPath(TABLE_NAME); - - java.nio.file.Path tempDir = createTempDirectory("custom-merger-mor"); - try { - java.nio.file.Path tempTableDir = tempDir.resolve(TABLE_NAME); - writeTable(new Path(tempTableDir.toUri())); - ResourceHudiTablesInitializer.copyDir(tempTableDir, fileSystem, tableLocation); - - metastore.createTable(createTableDefinition(schemaName, TABLE_NAME, tableLocation, false), PrincipalPrivileges.NO_PRIVILEGES); - metastore.createTable(createTableDefinition(schemaName, RT_TABLE_NAME, tableLocation, true), PrincipalPrivileges.NO_PRIVILEGES); - } - finally { - deleteRecursively(tempDir, ALLOW_INSECURE); - } + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); } - private static void writeTable(Path tablePath) + @Override + protected Schema avroSchema() { - Schema schema = createAvroSchema(); - try (HoodieJavaWriteClient writeClient = createWriteClient(schema, tablePath)) { - // First commit: bulk insert base records (produces base parquet files). - String firstCommit = writeClient.startCommit(); - List firstStatuses = writeClient.bulkInsert(ImmutableList.of( - record(schema, "k1", "k1_base", 10L, 1L), - record(schema, "k2", "k2_base", 100L, 1L)), firstCommit); - writeClient.commit(firstCommit, firstStatuses); - - // Second commit: upserts the same keys (produces log files since inline compaction is disabled). - // k1 update has a larger value (99 > 10) -> keep-max keeps the update. - // k2 update has a smaller value (5 < 100) -> keep-max keeps the original base record. - String secondCommit = writeClient.startCommit(); - List secondStatuses = writeClient.upsert(ImmutableList.of( - record(schema, "k1", "k1_updated", 99L, 2L), - record(schema, "k2", "k2_updated", 5L, 2L)), secondCommit); - writeClient.commit(secondCommit, secondStatuses); - } + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); } - private static HoodieJavaWriteClient createWriteClient(Schema schema, Path tablePath) + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) { - Configuration conf = new Configuration(); - try { - HoodieTableMetaClient.newTableBuilder() - .setTableType(HoodieTableType.MERGE_ON_READ) - .setTableName(TABLE_NAME) - .setTimelineLayoutVersion(1) - .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) - .setPayloadClassName(HoodieAvroPayload.class.getName()) - .setRecordKeyFields(RECORD_KEY_FIELD) - .setOrderingFields(ORDERING_FIELD) - .setRecordMergeMode(RecordMergeMode.CUSTOM) - .setRecordMergeStrategyId(KeyBasedTestRecordMerger.MERGE_STRATEGY_ID) - .initTable(new HadoopStorageConfiguration(conf), tablePath.toString()); - } - catch (IOException e) { - throw new RuntimeException("Could not init table " + TABLE_NAME, e); - } + tableBuilder + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(KeyBasedTestRecordMerger.MERGE_STRATEGY_ID); + } - HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder() - .withPath(tablePath.toString()) - .withSchema(schema.toString()) - .withParallelism(2, 2) - .withDeleteParallelism(2) - .forTable(TABLE_NAME) - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) - // Ordering field is carried by the table config (setOrderingFields); avoid the deprecated - // withPreCombineField builder method which fails the -Werror compile gate. + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder .withRecordMergeMode(RecordMergeMode.CUSTOM) .withRecordMergeStrategyId(KeyBasedTestRecordMerger.MERGE_STRATEGY_ID) - .withRecordMergeImplClasses(KeyBasedTestRecordMerger.class.getName()) - // Keep log files around so the custom merger runs at read time. - .withCompactionConfig(HoodieCompactionConfig.newBuilder() - .withInlineCompaction(false) - .withMaxNumDeltaCommitsBeforeCompaction(100) - .build()) - .withEmbeddedTimelineServerEnabled(false) - .withMarkersType(MarkerType.DIRECT.name()) - // MDT writes require hbase deps not present in the Trino runtime; disable as other initializers do. - .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) - .build(); - return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), cfg); + .withRecordMergeImplClasses(KeyBasedTestRecordMerger.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 1L), + record(schema, "k2", "k2_base", 100L, 1L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1 update has a larger value (99 > 10) -> keep-max keeps the update. + // k2 update has a smaller value (5 < 100) -> keep-max keeps the original base record. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 2L), + record(schema, "k2", "k2_updated", 5L, 2L)), secondCommit); + client.commit(secondCommit, secondStatuses); } private static HoodieRecord record(Schema schema, String key, String name, long value, long ts) @@ -203,37 +124,6 @@ private static HoodieRecord record(Schema schema, String key, record.put("name", name); record.put("value", value); record.put(ORDERING_FIELD, ts); - HoodieKey hoodieKey = new HoodieKey(key, PARTITION_PATH); - return new HoodieAvroRecord<>(hoodieKey, new HoodieAvroPayload(Option.of(record)), null); - } - - private static Schema createAvroSchema() - { - List fields = ImmutableList.of( - new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), - new Schema.Field("name", Schema.create(Schema.Type.STRING)), - new Schema.Field("value", Schema.create(Schema.Type.LONG)), - new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); - return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); - } - - private static Table createTableDefinition(String schemaName, String tableName, Location location, boolean isRtTable) - { - StorageFormat storageFormat = StorageFormat.create( - PARQUET_HIVE_SERDE_CLASS, - isRtTable ? HUDI_PARQUET_REALTIME_INPUT_FORMAT : HUDI_PARQUET_INPUT_FORMAT, - MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); - - return Table.builder() - .setDatabaseName(schemaName) - .setTableName(tableName) - .setTableType(EXTERNAL_TABLE.name()) - .setOwner(Optional.of("public")) - .setDataColumns(DATA_COLUMNS) - .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) - .withStorage(storageBuilder -> storageBuilder - .setStorageFormat(storageFormat) - .setLocation(location.toString())) - .build(); + return avroRecord(record, key); } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java index e4f3f0e758a02..57d69ee3e584c 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/IncrementalCustomMergerHudiTablesInitializer.java @@ -14,67 +14,33 @@ package io.trino.plugin.hudi.testing; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.trino.filesystem.Location; -import io.trino.filesystem.TrinoFileSystem; -import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.Column; -import io.trino.metastore.HiveMetastore; -import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.HiveType; -import io.trino.metastore.PrincipalPrivileges; -import io.trino.metastore.StorageFormat; -import io.trino.metastore.Table; -import io.trino.plugin.hudi.HudiConnector; -import io.trino.spi.security.ConnectorIdentity; -import io.trino.testing.QueryRunner; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.client.HoodieJavaWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.client.common.HoodieJavaEngineContext; -import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; -import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.model.HoodieAvroPayload; -import org.apache.hudi.common.model.HoodieAvroRecord; -import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.marker.MarkerType; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.index.HoodieIndex; -import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; -import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; -import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; -import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; -import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; import static io.trino.metastore.HiveType.HIVE_BOOLEAN; import static io.trino.metastore.HiveType.HIVE_DOUBLE; import static io.trino.metastore.HiveType.HIVE_INT; import static io.trino.metastore.HiveType.HIVE_LONG; import static io.trino.metastore.HiveType.HIVE_STRING; -import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; /** * Creates a non-partitioned Merge-On-Read table configured with the {@link MaxRankRecordMerger} custom merger @@ -92,13 +58,11 @@ * commit), the winning commit for each key is known in closed form, so the full expected row for the real-time * table is reproduced exactly by {@link #expectedRows()} without reading anything back. *

    - * Two metastore tables are registered: a read-optimized table (base files only) and a real-time table (suffix - * {@code _rt}) that merges base + log files through the file group reader. The Hudi write client writes to a local - * staging directory; after each commit the staging directory is mirrored into the Trino filesystem location the - * connector reads from (see {@link #syncToTrino()}). + * Unlike the single-shot fixtures, the write client and its staging directory stay open after initialization so + * {@link #writeAndSyncNextCommit()} can add commits mid-test; the test must call {@link #close()} when done. */ public class IncrementalCustomMergerHudiTablesInitializer - implements HudiTablesInitializer + extends AbstractMergerHudiTablesInitializer { public static final String TABLE_NAME = "custom_merger_e2e"; public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; @@ -106,10 +70,7 @@ public class IncrementalCustomMergerHudiTablesInitializer public static final int TOTAL_COMMITS = 20; public static final int NUM_RECORDS = 10_000; - private static final String RECORD_KEY_FIELD = "key"; private static final String RANK_FIELD = MaxRankRecordMerger.RANK_COLUMN; - private static final String ORDERING_FIELD = "ts"; - private static final String PARTITION_PATH = ""; private enum Kind { @@ -124,50 +85,70 @@ private record ColumnSpec(String name, Kind kind) {} private static final List DATA_COLUMNS = buildDataColumns(); private static final Schema AVRO_SCHEMA = buildAvroSchema(); - // Mutable state driven across commits. - private TrinoFileSystem fileSystem; - private Location tableLocation; - private java.nio.file.Path stagingDir; - private Path stagingTablePath; - private HoodieJavaWriteClient writeClient; - /** Winning commit per record index under the max-rank merge policy (-1 = not yet inserted). */ private final int[] winningCommit = new int[NUM_RECORDS]; /** Most recent commit that touched each record index (used to measure divergence from newest-wins). */ private final int[] latestCommit = new int[NUM_RECORDS]; private int currentCommit; + public IncrementalCustomMergerHudiTablesInitializer() + { + super(TABLE_NAME); + Arrays.fill(winningCommit, -1); + Arrays.fill(latestCommit, -1); + } + @Override - public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) - throws Exception + protected List dataColumns() { - fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() - .getInstance(TrinoFileSystemFactory.class) - .create(ConnectorIdentity.ofUser("test")); - HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() - .getInstance(HiveMetastoreFactory.class) - .createMetastore(Optional.empty()); + return DATA_COLUMNS; + } - tableLocation = externalLocation.appendPath(TABLE_NAME); - stagingDir = createTempDirectory("custom-merger-e2e"); - stagingTablePath = new Path(stagingDir.resolve(TABLE_NAME).toUri()); + @Override + protected Schema avroSchema() + { + return AVRO_SCHEMA; + } - java.util.Arrays.fill(winningCommit, -1); - java.util.Arrays.fill(latestCommit, -1); + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID); + } - initTable(); - writeClient = createWriteClient(); + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder + .withRecordMergeMode(RecordMergeMode.CUSTOM) + .withRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) + .withRecordMergeImplClasses(MaxRankRecordMerger.class.getName()); + } + @Override + protected int maxDeltaCommitsBeforeCompaction() + { + return TOTAL_COMMITS + 100; + } + + @Override + protected boolean keepsWriterOpen() + { + return true; + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { // First commit: bulk insert all keys (produces the base parquet files the read-optimized table reads). currentCommit = 1; - String firstCommit = writeClient.startCommit(); - List statuses = writeClient.bulkInsert(buildRecords(currentCommit), firstCommit); - writeClient.commit(firstCommit, statuses); + String firstCommit = client.startCommit(); + List statuses = client.bulkInsert(buildRecords(currentCommit), firstCommit); + client.commit(firstCommit, statuses); recordCommit(currentCommit); - syncToTrino(); - - metastore.createTable(createTableDefinition(schemaName, TABLE_NAME, tableLocation, false), PrincipalPrivileges.NO_PRIVILEGES); - metastore.createTable(createTableDefinition(schemaName, RT_TABLE_NAME, tableLocation, true), PrincipalPrivileges.NO_PRIVILEGES); } /** @@ -177,9 +158,9 @@ public void initializeTables(QueryRunner queryRunner, Location externalLocation, public void writeAndSyncNextCommit() { currentCommit++; - String commitTime = writeClient.startCommit(); - List statuses = writeClient.upsert(buildRecords(currentCommit), commitTime); - writeClient.commit(commitTime, statuses); + String commitTime = writeClient().startCommit(); + List statuses = writeClient().upsert(buildRecords(currentCommit), commitTime); + writeClient().commit(commitTime, statuses); recordCommit(currentCommit); syncToTrino(); } @@ -227,19 +208,6 @@ public int divergentKeyCount() return count; } - public void close() - throws IOException - { - if (writeClient != null) { - writeClient.close(); - writeClient = null; - } - if (stagingDir != null) { - deleteRecursively(stagingDir, ALLOW_INSECURE); - stagingDir = null; - } - } - private void recordCommit(int commit) { for (int ki = 0; ki < NUM_RECORDS; ki++) { @@ -278,8 +246,7 @@ private static HoodieRecord record(int recordIndex, int commi for (int i = 0; i < COLUMN_SPECS.size(); i++) { record.put(COLUMN_SPECS.get(i).name(), valueFor(i, recordIndex, commit)); } - HoodieKey hoodieKey = new HoodieKey(key(recordIndex), PARTITION_PATH); - return new HoodieAvroRecord<>(hoodieKey, new HoodieAvroPayload(Option.of(record)), null); + return avroRecord(record, key(recordIndex)); } private static Object[] rowFor(int recordIndex, int commit) @@ -327,86 +294,6 @@ private static long mergeRank(int recordIndex, int commit) return Math.floorMod(recordIndex * 2_654_435_761L + commit * 40_503L, 100_000L); } - private void syncToTrino() - { - try { - if (fileSystem.directoryExists(tableLocation).orElse(false)) { - fileSystem.deleteDirectory(tableLocation); - } - ResourceHudiTablesInitializer.copyDir(stagingDir.resolve(TABLE_NAME), fileSystem, tableLocation); - } - catch (IOException e) { - throw new RuntimeException("Failed to sync staged Hudi table to Trino filesystem", e); - } - } - - private void initTable() - { - Configuration conf = new Configuration(); - try { - HoodieTableMetaClient.newTableBuilder() - .setTableType(HoodieTableType.MERGE_ON_READ) - .setTableName(TABLE_NAME) - .setTimelineLayoutVersion(1) - .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) - .setPayloadClassName(HoodieAvroPayload.class.getName()) - .setRecordKeyFields(RECORD_KEY_FIELD) - .setOrderingFields(ORDERING_FIELD) - .setRecordMergeMode(RecordMergeMode.CUSTOM) - .setRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) - .initTable(new HadoopStorageConfiguration(conf), stagingTablePath.toString()); - } - catch (IOException e) { - throw new RuntimeException("Could not init table " + TABLE_NAME, e); - } - } - - private HoodieJavaWriteClient createWriteClient() - { - Configuration conf = new Configuration(); - HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder() - .withPath(stagingTablePath.toString()) - .withSchema(AVRO_SCHEMA.toString()) - .withParallelism(2, 2) - .withDeleteParallelism(2) - .forTable(TABLE_NAME) - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) - .withRecordMergeMode(RecordMergeMode.CUSTOM) - .withRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) - .withRecordMergeImplClasses(MaxRankRecordMerger.class.getName()) - // Keep log files around so the custom merger runs at read time for every delta commit. - .withCompactionConfig(HoodieCompactionConfig.newBuilder() - .withInlineCompaction(false) - .withMaxNumDeltaCommitsBeforeCompaction(TOTAL_COMMITS + 100) - .build()) - .withEmbeddedTimelineServerEnabled(false) - .withMarkersType(MarkerType.DIRECT.name()) - // MDT writes require hbase deps not present in the Trino runtime; disable as other initializers do. - .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) - .build(); - return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), cfg); - } - - private static Table createTableDefinition(String schemaName, String tableName, Location location, boolean isRtTable) - { - StorageFormat storageFormat = StorageFormat.create( - PARQUET_HIVE_SERDE_CLASS, - isRtTable ? HUDI_PARQUET_REALTIME_INPUT_FORMAT : HUDI_PARQUET_INPUT_FORMAT, - MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); - - return Table.builder() - .setDatabaseName(schemaName) - .setTableName(tableName) - .setTableType(EXTERNAL_TABLE.name()) - .setOwner(Optional.of("public")) - .setDataColumns(DATA_COLUMNS) - .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) - .withStorage(storageBuilder -> storageBuilder - .setStorageFormat(storageFormat) - .setLocation(location.toString())) - .build(); - } - private static List buildColumnSpecs() { ImmutableList.Builder specs = ImmutableList.builder(); @@ -438,10 +325,6 @@ private static List buildColumnSpecs() private static List buildDataColumns() { ImmutableList.Builder columns = ImmutableList.builder(); - // Hudi metadata columns prepended to every table. - for (String meta : List.of("_hoodie_commit_time", "_hoodie_commit_seqno", "_hoodie_record_key", "_hoodie_partition_path", "_hoodie_file_name")) { - columns.add(new Column(meta, HIVE_STRING, Optional.empty(), Map.of())); - } for (ColumnSpec spec : COLUMN_SPECS) { columns.add(new Column(spec.name(), hiveType(spec.kind()), Optional.empty(), Map.of())); } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java index eff623d0a6e02..1a3d2a532a745 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MaxRankRecordMerger.java @@ -84,9 +84,9 @@ public String[] getMandatoryFieldsForMerging(HoodieSchema dataSchema, HoodieTabl /** * Merging depends only on {@code merge_rank}, which is declared mandatory above, so it works on projected - * records. Without this override the file-group reader reads ALL table columns for merging, which the Trino - * connector does not support -- it can only resolve columns in the projection plus the merger's declared - * mandatory fields. + * records. Without this override the file-group reader reads ALL table columns for merging (see + * {@link NonProjectionCompatibleRankMerger} for that path); declaring projection compatibility keeps reads + * on the cheaper projected fast path. */ @Override public boolean isProjectionCompatible() diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleMergerHudiTablesInitializer.java new file mode 100644 index 0000000000000..c007d60607ec7 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleMergerHudiTablesInitializer.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table at test runtime configured with the + * NON-projection-compatible {@link NonProjectionCompatibleRankMerger} via {@link RecordMergeMode#CUSTOM}. + * One {@code bulkInsert} (base files) is followed by one {@code upsert} of the same keys (log files), with + * inline compaction disabled so the merger runs at read time over a full-table-schema read. + *

    + * Data is laid out so each merge direction is distinguishable: one key's winning rank comes from the LOG + * record and the other's from the BASE record, so a correct result proves both sides of the merge supplied + * the (never projected) {@code merge_rank} column. See {@code TestHudiNonProjectionCompatibleMerger}. + */ +public class NonProjectionCompatibleMergerHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "non_projection_merger_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + private static final String RANK_FIELD = NonProjectionCompatibleRankMerger.RANK_COLUMN; + + public NonProjectionCompatibleMergerHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(RANK_FIELD, HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(RANK_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder + .withRecordMergeMode(RecordMergeMode.CUSTOM) + .withRecordMergeStrategyId(NonProjectionCompatibleRankMerger.MERGE_STRATEGY_ID) + .withRecordMergeImplClasses(NonProjectionCompatibleRankMerger.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 5L, 1L), + record(schema, "k2", "k2_base", 100L, 9L, 1L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1 update has a HIGHER rank (7 > 5) -> merge keeps the update (99): the LOG record's rank decides. + // k2 update has a LOWER rank (1 < 9) -> merge keeps the base record (100): the BASE record's rank + // decides, which only works when the base read carries merge_rank despite it never being projected. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 7L, 2L), + record(schema, "k2", "k2_updated", 4L, 1L, 2L)), secondCommit); + client.commit(secondCommit, secondStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long rank, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(RANK_FIELD, rank); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleRankMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleRankMerger.java new file mode 100644 index 0000000000000..4e332bc3be5a1 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleRankMerger.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.read.BufferedRecord; + +import java.io.IOException; + +/** + * Test-only custom {@link HoodieRecordMerger} with the same keep-max-{@code merge_rank} policy as + * {@link MaxRankRecordMerger}, but deliberately WITHOUT the {@code isProjectionCompatible()} and + * {@code getMandatoryFieldsForMerging()} overrides. It therefore reports the interface default + * {@code isProjectionCompatible() == false} and never declares {@code merge_rank} as mandatory, so + * nothing prepends the rank column into the connector's read projection. The file-group reader reacts + * by demanding the FULL table schema as its required schema for any split with log files -- for the + * base and log reads alike -- making this merger the acceptance case for full-schema reads + * (apache/hudi#19249): a query that does not project {@code merge_rank} merges correctly only if both + * sides of the merge supply it. + */ +public class NonProjectionCompatibleRankMerger + implements HoodieRecordMerger +{ + /** + * Unique strategy id identifying this custom merger. The test table's + * {@code hoodie.record.merge.strategy.id} is set to this value so the reader resolves this implementation. + */ + public static final String MERGE_STRATEGY_ID = "8e1f4a6b-2c9d-4d35-9a7e-5b0c8f3d1e42"; + + /** Name of the column whose value decides the merge. */ + public static final String RANK_COLUMN = "merge_rank"; + + @Override + public BufferedRecord merge(BufferedRecord older, BufferedRecord newer, RecordContext recordContext, TypedProperties props) + throws IOException + { + // Deletes are passed through so tombstones still win; this test does not exercise deletes. + if (older == null || older.isDelete() || newer.isDelete()) { + return newer; + } + long olderRank = rankOf(older, recordContext); + long newerRank = rankOf(newer, recordContext); + // Keep the newer record on ties so the policy stays deterministic. + return newerRank >= olderRank ? newer : older; + } + + private static long rankOf(BufferedRecord record, RecordContext recordContext) + { + HoodieSchema schema = recordContext.getSchemaFromBufferRecord(record); + Object value = recordContext.getValue(record.getRecord(), schema, RANK_COLUMN); + // A null here means the reader failed to supply merge_rank on this side of the merge (the very + // regression this merger exists to catch); fail loudly instead of merging arbitrarily. + if (value == null) { + throw new IllegalStateException("merge_rank is missing from a record of schema " + schema); + } + return ((Number) value).longValue(); + } + + @Override + public HoodieRecord.HoodieRecordType getRecordType() + { + return HoodieRecord.HoodieRecordType.AVRO; + } + + @Override + public String getMergingStrategy() + { + return MERGE_STRATEGY_ID; + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java index 89db0e7dffbeb..bf1d1c5672eb7 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/NonProjectionCompatibleTestRecordMerger.java @@ -18,10 +18,9 @@ * {@code isProjectionCompatible() == false}, which makes the file-group reader request the FULL table schema * instead of the connector's projection. *

    - * The Trino connector can only resolve columns that are in the read projection (plus Hudi meta columns), so a - * data column outside the projection is rejected with a {@code NOT_SUPPORTED} error. This merger exists purely - * to exercise that guard; it shares {@link KeyBasedTestRecordMerger#MERGE_STRATEGY_ID} so it resolves against - * the same test table. + * The connector expands the read projection to the full table schema for such mergers, so narrow queries + * still merge correctly. This merger exists purely to exercise that full-schema path; it shares + * {@link KeyBasedTestRecordMerger#MERGE_STRATEGY_ID} so it resolves against the same test table. */ public class NonProjectionCompatibleTestRecordMerger extends KeyBasedTestRecordMerger diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedOrderingFieldHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedOrderingFieldHudiTablesInitializer.java new file mode 100644 index 0000000000000..c9e31efef18d2 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedOrderingFieldHudiTablesInitializer.java @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.DefaultHoodieRecordPayload; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table in {@link RecordMergeMode#EVENT_TIME_ORDERING} whose + * METASTORE column list deliberately omits the ordering field {@code ts} that the Avro table schema (and the + * data files) carry -- the shape hive sync leaves behind when a column is not synced. Event-time merging + * still needs {@code ts} on both sides of every merge, so reads of the real-time table only produce correct + * results when the merge path recovers the column from the resolved table schema; without that recovery the + * base read cannot supply {@code ts} and the read fails. + *

    + * Data is laid out so each merge direction is distinguishable: {@code k1}'s winning event time is on the LOG + * record and {@code k2}'s on the BASE record. See {@code TestHudiNonProjectionCompatibleMerger}. + */ +public class OmittedOrderingFieldHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "omitted_ordering_field_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public OmittedOrderingFieldHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + // No ts column: the fixture's whole point is a metastore that does not know the ordering field. + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder + .setPayloadClassName(DefaultHoodieRecordPayload.class.getName()) + .setRecordMergeMode(RecordMergeMode.EVENT_TIME_ORDERING); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withRecordMergeMode(RecordMergeMode.EVENT_TIME_ORDERING); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 5L), + record(schema, "k2", "k2_base", 100L, 9L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1's update carries a NEWER event time (7 > 5) -> the update wins (99): the LOG record's ts decides. + // k2's update carries an OLDER event time (1 < 9) -> the base record wins (100): the BASE record's ts + // decides, which only works when the base read carries ts despite the metastore not knowing it. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 7L), + record(schema, "k2", "k2_updated", 4L, 1L)), secondCommit); + client.commit(secondCommit, secondStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedRankFieldHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedRankFieldHudiTablesInitializer.java new file mode 100644 index 0000000000000..94cf8deada0e9 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedRankFieldHudiTablesInitializer.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table using the projection-compatible {@link MaxRankRecordMerger} + * via {@link RecordMergeMode#CUSTOM}, whose METASTORE column list deliberately omits the merger's mandatory + * {@code merge_rank} column that the Avro table schema (and the data files) carry. The merger reads + * {@code merge_rank} on both sides of every merge, so reads of the real-time table only produce correct + * results when the merge path recovers merger-declared mandatory columns from the resolved table schema by + * asking the merger itself; without that the base read cannot supply {@code merge_rank} and the read fails. + *

    + * Data is laid out so each merge direction is distinguishable: {@code k1}'s winning rank is on the LOG + * record and {@code k2}'s on the BASE record. See {@code TestHudiNonProjectionCompatibleMerger}. + */ +public class OmittedRankFieldHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "omitted_rank_field_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + private static final String RANK_FIELD = MaxRankRecordMerger.RANK_COLUMN; + + public OmittedRankFieldHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + // No merge_rank column: the fixture's whole point is a metastore that does not know the column the + // merger declares mandatory. + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(RANK_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordMergeMode(RecordMergeMode.CUSTOM) + .setRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder + .withRecordMergeMode(RecordMergeMode.CUSTOM) + .withRecordMergeStrategyId(MaxRankRecordMerger.MERGE_STRATEGY_ID) + .withRecordMergeImplClasses(MaxRankRecordMerger.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 5L, 1L), + record(schema, "k2", "k2_base", 100L, 9L, 1L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1's update has a HIGHER rank (7 > 5) -> keep-max keeps the update (99): the LOG record's rank decides. + // k2's update has a LOWER rank (1 < 9) -> keep-max keeps the base record (100): the BASE record's rank + // decides, which only works when the base read carries merge_rank despite the metastore not knowing it. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 7L, 2L), + record(schema, "k2", "k2_updated", 4L, 1L, 2L)), secondCommit); + client.commit(secondCommit, secondStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long rank, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(RANK_FIELD, rank); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/PayloadOnlyMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/PayloadOnlyMergerHudiTablesInitializer.java new file mode 100644 index 0000000000000..b1b4456cbc3ae --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/PayloadOnlyMergerHudiTablesInitializer.java @@ -0,0 +1,209 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Creates a non-partitioned Merge-On-Read table at TABLE VERSION 6 whose only merge configuration is a + * {@link org.apache.hudi.common.model.HoodieRecordPayload} class ({@link RankBasedTestPayload}), the way a + * genuine pre-1.0 table looks on storage. + *

    + * Such a table resolves at read time to {@code CUSTOM} merge mode with the payload-based merge strategy, which + * in turn resolves {@code HoodieAvroRecordMerger}. That merger is not projection compatible, so the file group + * reader demands the FULL table schema for base and log reads alike and nothing prepends the payload's decision + * column into the connector's read projection. + *

    + * One {@code bulkInsert} (base files) is followed by one {@code upsert} of the same keys (log files), with data + * laid out so each merge direction is distinguishable: {@code k1}'s winning rank is on the LOG record and + * {@code k2}'s on the BASE record. See {@code TestHudiNonProjectionCompatibleMerger}. + */ +public class PayloadOnlyMergerHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "payload_only_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + private static final String RANK_FIELD = RankBasedTestPayload.RANK_COLUMN; + private static final HoodieTableVersion TABLE_VERSION = HoodieTableVersion.SIX; + + public PayloadOnlyMergerHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(RANK_FIELD, HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(RANK_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + // Only the payload class: the merge mode and the merge strategy id must stay out of hoodie.properties + // so the reader has to infer them, which is what makes this a pre-1.0 payload-only table. + tableBuilder + .setTableVersion(TABLE_VERSION) + .setPayloadClassName(RankBasedTestPayload.class.getName()); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder + .withWriteTableVersion(TABLE_VERSION.versionCode()) + // Without this the write client silently upgrades the table to the current version on the first commit. + .withAutoUpgradeVersion(false) + .withWritePayLoad(RankBasedTestPayload.class.getName()); + } + + @Override + protected void afterTableInit() + throws IOException + { + stripInferredMergeStrategyId(); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + throws IOException + { + Schema schema = avroSchema(); + // First commit: bulk insert base records (produces base parquet files). + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 5L, 1L), + record(schema, "k2", "k2_base", 100L, 9L, 1L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit: upserts the same keys (produces log files since inline compaction is disabled). + // k1 update has a HIGHER rank (7 > 5) -> the payload keeps the update (99): the LOG record's rank decides. + // k2 update has a LOWER rank (1 < 9) -> the payload keeps the base record (100): the BASE record's rank + // decides, which only works when the base read carries merge_rank despite it never being projected. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 99L, 7L, 2L), + record(schema, "k2", "k2_updated", 4L, 1L, 2L)), secondCommit); + client.commit(secondCommit, secondStatuses); + + // The commits go through the table config; re-check that the writer left the fixture payload-only. + stripInferredMergeStrategyId(); + } + + /** + * Removes the merge strategy id that table creation infers and persists even for a version 6 table, + * then verifies what is left. The merge MODE never reaches disk here: its config is since-version + * 1.0.0, so creation itself drops it for a version 6 table ({@code HoodieTableConfig.dropInvalidConfigs}); + * the {@code checkAbsent} below just pins that. The fixture has to mimic a genuine pre-1.0 table, which + * persists its payload class and nothing else about merging; leaving the inferred id in would hand the + * reader the answer this fixture exists to make it derive. + */ + private void stripInferredMergeStrategyId() + throws IOException + { + Path metaDirectory = stagingTableDirectory().resolve(".hoodie"); + Path propertiesFile = metaDirectory.resolve("hoodie.properties"); + List retainedLines = Files.readAllLines(propertiesFile, UTF_8).stream() + .filter(line -> !isEntryFor(line, HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key())) + .toList(); + Files.write(propertiesFile, retainedLines, UTF_8); + // The Hadoop local filesystem verifies this sidecar when the write client reopens the file, and an + // out-of-band edit leaves it stale; dropping it makes the file checksum-free rather than corrupt. + Files.deleteIfExists(metaDirectory.resolve(".hoodie.properties.crc")); + + Properties properties = new Properties(); + try (InputStream input = Files.newInputStream(propertiesFile)) { + properties.load(input); + } + checkProperty(properties, HoodieTableConfig.VERSION.key(), String.valueOf(TABLE_VERSION.versionCode())); + checkProperty(properties, HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), RankBasedTestPayload.class.getName()); + checkAbsent(properties, HoodieTableConfig.RECORD_MERGE_MODE.key()); + checkAbsent(properties, HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key()); + } + + private static boolean isEntryFor(String line, String key) + { + return line.startsWith(key + "=") || line.startsWith(key + ":") || line.startsWith(key + " "); + } + + private static void checkProperty(Properties properties, String key, String expected) + { + String actual = properties.getProperty(key); + if (!expected.equals(actual)) { + throw new IllegalStateException("Expected %s=%s in hoodie.properties of %s but found %s".formatted(key, expected, TABLE_NAME, actual)); + } + } + + private static void checkAbsent(Properties properties, String key) + { + if (properties.containsKey(key)) { + throw new IllegalStateException("Expected no %s in hoodie.properties of %s but found %s".formatted(key, TABLE_NAME, properties.getProperty(key))); + } + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long rank, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(RANK_FIELD, rank); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/RankBasedTestPayload.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/RankBasedTestPayload.java new file mode 100644 index 0000000000000..c9480a1dc233f --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/RankBasedTestPayload.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.util.Option; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + +import java.io.IOException; + +/** + * Test-only {@link org.apache.hudi.common.model.HoodieRecordPayload} that keeps whichever of the two records + * being merged carries the larger {@code merge_rank}, ties going to the incoming (newer) record. + *

    + * The policy makes payload-based merging distinguishable from both the base-only view (no merging happened at + * all) and the built-in newest-wins behavior (merging happened, but not through the payload), which is what + * makes it usable as the read-side acceptance case for a table whose only merge configuration is its payload + * class. It also forces {@code merge_rank} to be read on BOTH sides of the merge, so a query that does not + * project the column only produces the right answer over a full-table-schema read. + *

    + * Both constructors are required: Hudi instantiates payloads reflectively through + * {@code HoodieRecordUtils.loadPayload}, which looks up either {@code (GenericRecord, Comparable)} or + * {@code (Option)}. + */ +public class RankBasedTestPayload + extends OverwriteWithLatestAvroPayload +{ + /** Name of the column whose value decides the merge, matching the rank-merger fixtures. */ + public static final String RANK_COLUMN = "merge_rank"; + + public RankBasedTestPayload(GenericRecord record, Comparable orderingVal) + { + super(record, orderingVal); + } + + public RankBasedTestPayload(Option record) + { + super(record); + } + + @Override + public Option combineAndGetUpdateValue(IndexedRecord currentValue, Schema schema) + throws IOException + { + // getInsertValue applies the standard empty/delete handling, so an empty result here is a deletion. + Option incomingRecord = getInsertValue(schema); + if (incomingRecord.isEmpty()) { + return incomingRecord; + } + return rankOf(incomingRecord.get()) >= rankOf(currentValue) ? incomingRecord : Option.of(currentValue); + } + + private static long rankOf(IndexedRecord record) + { + GenericRecord genericRecord = (GenericRecord) record; + Schema.Field field = genericRecord.getSchema().getField(RANK_COLUMN); + Object value = field == null ? null : genericRecord.get(field.pos()); + // A null here means the reader failed to supply merge_rank on this side of the merge (the very + // regression this payload exists to catch); fail loudly instead of merging arbitrarily. + if (value == null) { + throw new IllegalStateException("merge_rank is missing from a record of schema " + genericRecord.getSchema()); + } + return ((Number) value).longValue(); + } +} From b60d76f5b1a33ff54a6d78800ca6e5390064107f Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 29 Jul 2026 00:40:04 -0700 Subject: [PATCH 217/255] fix(client): fix NPE in schema conflict resolution on commits with null writer schema (#19388) * fix(client): fix NPE in schema conflict resolution on commits with null writer schema When schema conflict resolution is enabled and a writer commits a batch whose writer schema is the Avro null schema (an ingestion round that writes no data), the resolution strategy resolves the table schema at the current transaction's owner instant. At pre-commit time that instant is inflight with no completion time, so the completion-time filter in the schema getter calls String.compareTo(null) and the commit fails with an NPE. - Expose the per-timeline-version instant ordering as a first-class API on InstantComparator: orderingComparator() / getOrderingTime(instant), requested-time based in v1 and completion-time based in v2. - ConcurrentSchemaEvolutionTableSchemaGetter sorts and bounds the schema evolution timeline with that ordering (completion time for table version 8 and above, requested time for earlier versions, matching 0.x). A target instant without an ordering time no longer bounds the lookup instead of throwing. - SimpleSchemaConflictResolutionStrategy adopts the table schema as of the owner instant on the null-writer-schema path, using the version-appropriate ordering time. - Add regression coverage in TestSimpleSchemaConflictResolutionStrategy, TestConcurrentSchemaEvolutionTableSchemaGetter, and TestInstantComparators. * test(client): parameterize schema conflict resolution over table version 6 and 8; docs(common): ordering-key invariant and upgrade-boundary completion time (cherry picked from commit b4756518990e0091e6a2a312f2887350d0f896d7) --- ...rrentSchemaEvolutionTableSchemaGetter.java | 26 +++- ...impleSchemaConflictResolutionStrategy.java | 10 +- ...rrentSchemaEvolutionTableSchemaGetter.java | 97 +++++++++++++ ...impleSchemaConflictResolutionStrategy.java | 129 ++++++++++++++---- .../table/timeline/InstantComparator.java | 16 +++ .../versioning/v1/InstantComparatorV1.java | 10 ++ .../versioning/v2/InstantComparatorV2.java | 14 ++ .../timeline/TestInstantComparators.java | 35 +++++ 8 files changed, 295 insertions(+), 42 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java index 0bb7db3fa5833..3ff8625e7457f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java @@ -25,7 +25,7 @@ import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.table.timeline.TimelineLayout; +import org.apache.hudi.common.table.timeline.InstantComparator; import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; @@ -60,6 +60,8 @@ class ConcurrentSchemaEvolutionTableSchemaGetter { private final Lazy> tableSchemaCache; + private final InstantComparator instantComparator; + private Option latestCommitWithValidSchema = Option.empty(); @VisibleForTesting @@ -69,10 +71,18 @@ public ConcurrentHashMap getTableSchemaCache() { public ConcurrentSchemaEvolutionTableSchemaGetter(HoodieTableMetaClient metaClient) { this.metaClient = metaClient; + this.instantComparator = metaClient.getTimelineLayout().getInstantComparator(); // Unbounded sized map. Should replace with some caching library. this.tableSchemaCache = Lazy.lazily(ConcurrentHashMap::new); } + /** + * Returns the timestamp ordering the instant in the schema evolution timeline. + */ + String getOrderingTime(HoodieInstant instant) { + return instantComparator.getOrderingTime(instant); + } + /** * Handles partition column logic for a given schema. * @@ -160,9 +170,11 @@ Option> getLastCommitMetadataWithValidSchemaFr // the timeline finding a completed instant containing a valid schema. ConcurrentHashMap tableSchemaAtInstant = new ConcurrentHashMap<>(); Option instantWithTableSchema = Option.fromJavaOptional(reversedTimelineStream - // If a completion time is specified, find the first eligible instant in the schema evolution timeline. - // Should switch to completion time based. - .filter(s -> instant.isEmpty() || compareTimestamps(s.getCompletionTime(), LESSER_THAN_OR_EQUALS, instant.get().getCompletionTime())) + // Find the first eligible instant whose ordering time is no later than the target instant's; + // a target instant without an ordering time (not completed yet, on table version 8 and above) + // does not bound the lookup. + .filter(s -> instant.isEmpty() || StringUtils.isNullOrEmpty(getOrderingTime(instant.get())) + || compareTimestamps(getOrderingTime(s), LESSER_THAN_OR_EQUALS, getOrderingTime(instant.get()))) // Make sure the commit metadata has a valid schema inside. Same caching the result for expensive operation. .filter(s -> { try { @@ -193,6 +205,8 @@ Option> getLastCommitMetadataWithValidSchemaFr /** * Get timeline in REVERSE order that only contains completed instants which POTENTIALLY evolve the table schema. + * The stream follows the timeline layout's instant ordering, newest first (completion time for + * layout v2, requested time for v1). * For types of instants that are included and not reflecting table schema at their instant completion time please refer * comments inside the code. */ @@ -214,9 +228,7 @@ public Stream computeSchemaEvolutionTimelineInReverseOrder() { } // We only care committed instant when it comes to table schema. - TimelineLayout timelineLayout = metaClient.getTimelineLayout(); - // Table schema getter is completion time based ordering. - Comparator reversedComparator = timelineLayout.getInstantComparator().completionTimeOrderedComparator().reversed(); + Comparator reversedComparator = instantComparator.orderingComparator().reversed(); // The timeline still contains DELTA_COMMIT_ACTION/COMMIT_ACTION which might not contain a valid schema // field in their commit metadata. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java index cfcd26362552c..523b21356094c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java @@ -30,8 +30,6 @@ import lombok.extern.slf4j.Slf4j; -import java.util.stream.Stream; - import static org.apache.hudi.client.transaction.SchemaConflictResolutionStrategy.throwConcurrentSchemaEvolutionException; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMPACTION_ACTION; import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN_OR_EQUALS; @@ -77,7 +75,7 @@ public Option resolveConcurrentSchemaEvolution( // schema and writer schema. HoodieInstant lastCompletedInstantAtTxnStart = lastCompletedTxnOwnerInstant.isPresent() ? getInstantInTimelineImmediatelyPriorToTimestamp( - lastCompletedTxnOwnerInstant.get().getCompletionTime(), schemaResolver.computeSchemaEvolutionTimelineInReverseOrder()).orElse(null) + schemaResolver.getOrderingTime(lastCompletedTxnOwnerInstant.get()), schemaResolver).orElse(null) : null; // If lastCompletedInstantAtTxnValidation is null there are 2 possibilities: // - No committed txn at validation starts @@ -157,9 +155,9 @@ public Option resolveConcurrentSchemaEvolution( } private Option getInstantInTimelineImmediatelyPriorToTimestamp( - String timestamp, Stream reverseOrderTimeline) { - return Option.fromJavaOptional(reverseOrderTimeline - .filter(s -> compareTimestamps(s.getCompletionTime(), LESSER_THAN_OR_EQUALS, timestamp)) + String timestamp, ConcurrentSchemaEvolutionTableSchemaGetter schemaResolver) { + return Option.fromJavaOptional(schemaResolver.computeSchemaEvolutionTimelineInReverseOrder() + .filter(s -> compareTimestamps(schemaResolver.getOrderingTime(s), LESSER_THAN_OR_EQUALS, timestamp)) .findFirst()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java index 909ca863bfecb..6178a9847469d 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java @@ -36,6 +36,7 @@ import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanV2MigrationHandler; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; @@ -52,10 +53,17 @@ import org.mockito.Mockito; import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.hudi.common.table.HoodieTableConfig.PARTITION_FIELDS; @@ -70,6 +78,7 @@ import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_SCHEMA; import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; import static org.apache.hudi.common.util.CommitUtils.buildMetadata; +import static org.apache.hudi.config.HoodieWriteConfig.WRITE_TABLE_VERSION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -403,6 +412,82 @@ void testGetTableSchema(HoodieSchema inputSchema, boolean includeMetadataFields, includeMetadataFields, Option.of(instant)).get()); } + @Test + void testTableVersionEightAndAboveOrdersByCompletionTime() throws Exception { + metaClient = HoodieTestUtils.getMetaClientBuilder(HoodieTableType.COPY_ON_WRITE, new Properties(), "") + .initTable(getDefaultStorageConf(), basePath); + // The ordering is driven by the timeline layout version. + assertEquals(TimelineLayoutVersion.VERSION_2, metaClient.getTimelineLayoutVersion().getVersion()); + testTable = HoodieTestTable.of(metaClient); + + // Completion order inverts requested order: requested 001 completes last (at 100) with + // schema 2, requested 009 completes first (at 050) with schema 1. + testTable.addCommit("001", Option.of("100"), Option.of(buildMetadata( + Collections.emptyList(), Collections.emptyMap(), Option.empty(), WriteOperationType.UNKNOWN, + SCHEMA_WITHOUT_METADATA_STR2, COMMIT_ACTION))); + testTable.addCommit("009", Option.of("050"), Option.of(buildMetadata( + Collections.emptyList(), Collections.emptyMap(), Option.empty(), WriteOperationType.UNKNOWN, + SCHEMA_WITHOUT_METADATA_STR, COMMIT_ACTION))); + + ConcurrentSchemaEvolutionTableSchemaGetter resolver = new ConcurrentSchemaEvolutionTableSchemaGetter(metaClient); + // The latest table schema follows completion time: schema 2 of requested 001, completed 100. + assertEquals(SCHEMA_WITHOUT_METADATA2.toString(), + resolver.getTableSchemaIfPresent(false, Option.empty()).get().toString()); + // A target completed at 075 only sees the commit completed at 050 (requested 009, schema 1). + assertEquals(SCHEMA_WITHOUT_METADATA.toString(), + resolver.getTableSchemaIfPresent(false, + Option.of(metaClient.getInstantGenerator().createNewInstant( + HoodieInstant.State.COMPLETED, COMMIT_ACTION, "005", "075"))).get().toString()); + } + + @Test + void testTableVersionSixOrdersByRequestedTime() throws Exception { + Properties properties = new Properties(); + properties.setProperty(WRITE_TABLE_VERSION.key(), "6"); + metaClient = HoodieTestUtils.getMetaClientBuilder(HoodieTableType.COPY_ON_WRITE, properties, "") + .initTable(getDefaultStorageConf(), basePath); + // The ordering is driven by the timeline layout version. + assertEquals(TimelineLayoutVersion.VERSION_1, metaClient.getTimelineLayoutVersion().getVersion()); + testTable = HoodieTestTable.of(metaClient); + + // Same layout as the table-version-8 test: requested 001 carries schema 2, requested 009 + // carries schema 1. The completion times below are ignored by the table-version-6 + // (timeline layout v1) instant file naming. + testTable.addCommit("001", Option.of("100"), Option.of(buildMetadata( + Collections.emptyList(), Collections.emptyMap(), Option.empty(), WriteOperationType.UNKNOWN, + SCHEMA_WITHOUT_METADATA_STR2, COMMIT_ACTION))); + testTable.addCommit("009", Option.of("050"), Option.of(buildMetadata( + Collections.emptyList(), Collections.emptyMap(), Option.empty(), WriteOperationType.UNKNOWN, + SCHEMA_WITHOUT_METADATA_STR, COMMIT_ACTION))); + // Invert the file modification times so that the mtime-derived completion order disagrees + // with the requested order, mirroring the table-version-8 fixture above. + Path timelinePath = Paths.get(metaClient.getTimelinePath().makeQualified(new URI("file:///")).toUri()); + Files.setLastModifiedTime(timelinePath.resolve("001.commit"), FileTime.fromMillis(2_000_000_000_000L)); + Files.setLastModifiedTime(timelinePath.resolve("009.commit"), FileTime.fromMillis(1_000_000_000_000L)); + metaClient.reloadActiveTimeline(); + + // The mtime inversion must stick, otherwise the assertions below also hold under completion-time + // ordering and the test would pass against unfixed code. + Map completionTimeByRequestedTime = metaClient.getActiveTimeline().getInstantsAsStream() + .collect(Collectors.toMap(HoodieInstant::requestedTime, HoodieInstant::getCompletionTime)); + assertTrue(completionTimeByRequestedTime.get("001").compareTo(completionTimeByRequestedTime.get("009")) > 0); + + ConcurrentSchemaEvolutionTableSchemaGetter resolver = new ConcurrentSchemaEvolutionTableSchemaGetter(metaClient); + // The latest table schema follows requested time (schema 1 of requested 009), not the + // mtime-derived completion order which would pick schema 2 of requested 001. + assertEquals(SCHEMA_WITHOUT_METADATA.toString(), + resolver.getTableSchemaIfPresent(false, Option.empty()).get().toString()); + // An inflight target bounds the lookup by its requested time. + assertEquals(SCHEMA_WITHOUT_METADATA2.toString(), + resolver.getTableSchemaIfPresent(false, + Option.of(metaClient.getInstantGenerator().createNewInstant( + HoodieInstant.State.INFLIGHT, COMMIT_ACTION, "005"))).get().toString()); + assertEquals(SCHEMA_WITHOUT_METADATA.toString(), + resolver.getTableSchemaIfPresent(false, + Option.of(metaClient.getInstantGenerator().createNewInstant( + HoodieInstant.State.INFLIGHT, COMMIT_ACTION, "999"))).get().toString()); + } + private static Stream partitionColumnSchemaTestParams() { return Stream.of( Arguments.of(false, SCHEMA_WITHOUT_METADATA), // Schema with metadata fields, don't include metadata @@ -528,6 +613,18 @@ void testGetTableSchemaInternalWithSpecificInstant(HoodieTableType tableType) th assertTrue(schema2Option.isPresent()); assertEquals(schema2.toString(), schema2Option.get().toString()); + // A target instant without a completion time (e.g., an inflight instant at pre-commit time) + // does not bound the lookup; the latest table schema is returned. + String inflightTimestamp = padWithLeadingZeros(Integer.toString(startCommitTime), REQUEST_TIME_LENGTH); + Option schemaAtInstantWithoutCompletionTime = resolver.getTableSchemaIfPresent( + false, + Option.of(metaClient.getInstantGenerator().createNewInstant( + HoodieInstant.State.INFLIGHT, + tableType.equals(HoodieTableType.COPY_ON_WRITE) ? COMMIT_ACTION : DELTA_COMMIT_ACTION, + inflightTimestamp))); + assertTrue(schemaAtInstantWithoutCompletionTime.isPresent()); + assertEquals(schema2.toString(), schemaAtInstantWithoutCompletionTime.get().toString()); + // Now follow with more disqualified instants and try to get table schema with their request time, we should back track to instant 2. int endCommitTime = createExhaustiveDisqualifiedInstants(startCommitTime, tableType); metaClient.reloadActiveTimeline(); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java index 44759a5743ddd..2af94c07bdbd6 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java @@ -34,6 +34,7 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.view.FileSystemViewManager; import org.apache.hudi.common.testutils.HoodieTestTable; @@ -46,12 +47,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.stream.Stream; import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLUSTERING_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION; @@ -61,6 +67,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; import static org.apache.hudi.common.util.CommitUtils.buildMetadata; import static org.apache.hudi.config.HoodieWriteConfig.ENABLE_SCHEMA_CONFLICT_RESOLUTION; +import static org.apache.hudi.config.HoodieWriteConfig.WRITE_TABLE_VERSION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -93,9 +100,25 @@ public class TestSimpleSchemaConflictResolutionStrategy { private static final String NULL_SCHEMA = "{\"type\":\"null\"}"; private void setupInstants(String tableSchemaAtTxnStart, String tableSchemaAtTxnValidation, - String writerSchemaOfTxn, Boolean enableResolution, boolean setupLegacyClustering) throws Exception { - metaClient = HoodieTestUtils.getMetaClientBuilder(HoodieTableType.COPY_ON_WRITE, new Properties(), "") - .setTableCreateSchema(SCHEMA1) + String writerSchemaOfTxn, boolean enableResolution, boolean setupLegacyClustering) throws Exception { + setupInstants(SCHEMA1, tableSchemaAtTxnStart, tableSchemaAtTxnValidation, writerSchemaOfTxn, + enableResolution, setupLegacyClustering, HoodieTableVersion.current().versionCode()); + } + + private void setupInstants(String tableSchemaAtTxnStart, String tableSchemaAtTxnValidation, + String writerSchemaOfTxn, boolean enableResolution, boolean setupLegacyClustering, + int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, tableSchemaAtTxnStart, tableSchemaAtTxnValidation, writerSchemaOfTxn, + enableResolution, setupLegacyClustering, writeTableVersion); + } + + private void setupInstants(String tableCreateSchema, String tableSchemaAtTxnStart, String tableSchemaAtTxnValidation, + String writerSchemaOfTxn, boolean enableResolution, boolean setupLegacyClustering, + int writeTableVersion) throws Exception { + Properties tableProperties = new Properties(); + tableProperties.setProperty(WRITE_TABLE_VERSION.key(), String.valueOf(writeTableVersion)); + metaClient = HoodieTestUtils.getMetaClientBuilder(HoodieTableType.COPY_ON_WRITE, tableProperties, "") + .setTableCreateSchema(tableCreateSchema) .initTable(getDefaultStorageConf(), basePath.toString()); dummyInstantGenerator = HoodieTestTable.of(metaClient); @@ -134,71 +157,119 @@ private void setupInstants(String tableSchemaAtTxnStart, String tableSchemaAtTxn strategy = new SimpleSchemaConflictResolutionStrategy(); } - @Test - void testNoConflictFirstCommit() throws Exception { - setupInstants(null, null, SCHEMA1, true, false); + // The schema evolution timeline ordering follows the table version (requested time for table + // version 6, completion time for 8 and above), so the RFC-82 resolution cases run on both. The + // fixture's requested and completion orders agree, so the outcomes are the same on both versions. + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNoConflictFirstCommit(int writeTableVersion) throws Exception { + setupInstants(null, null, SCHEMA1, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, Option.empty(), nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA1), result); } - @Test - void testNullWriterSchema() throws Exception { - setupInstants(SCHEMA1, SCHEMA1, "", true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNullWriterSchema(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA1, "", true, false, writeTableVersion); assertFalse(strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).isPresent()); } - @Test - void testNullTypeWriterSchema() throws Exception { - setupInstants(SCHEMA1, SCHEMA1, NULL_SCHEMA, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNullTypeWriterSchema(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA1, NULL_SCHEMA, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA1), result); } @Test - void testConflictSecondCommitDifferentSchema() throws Exception { - setupInstants(null, SCHEMA1, SCHEMA2, true, false); + void testNullTypeWriterSchemaCurrTxnInstantWithoutCompletionTime() throws Exception { + setupInstants(SCHEMA1, SCHEMA2, NULL_SCHEMA, true, false, 8); + // At pre-commit time the curr txn owner instant is inflight and has no completion time; + // on table version 8 and above the resolution falls back to the latest table schema. + Option currTxnOwnerInstant = Option.of( + metaClient.createNewInstant(HoodieInstant.State.INFLIGHT, COMMIT_ACTION, "0040")); + HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( + table, config, lastCompletedTxnOwnerInstant, currTxnOwnerInstant).get(); + assertEquals(HoodieSchema.parse(SCHEMA2), result); + } + + @ParameterizedTest + @MethodSource("tableVersionSixNullSchemaCases") + void testNullTypeWriterSchemaTableVersionSix(String currTxnInstantTime, String expectedSchema) throws Exception { + // Table version 6 orders the schema evolution timeline by requested time, so the curr txn owner + // instant's requested time bounds the null-writer-schema lookup. A create schema (SCHEMA3) + // distinct from the commit schemas makes the before-all-commits fallback observable. + setupInstants(SCHEMA3, SCHEMA1, SCHEMA2, NULL_SCHEMA, true, false, 6); + Option currTxnOwnerInstant = Option.of( + metaClient.createNewInstant(HoodieInstant.State.INFLIGHT, COMMIT_ACTION, currTxnInstantTime)); + HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( + table, config, lastCompletedTxnOwnerInstant, currTxnOwnerInstant).get(); + assertEquals(HoodieSchema.parse(expectedSchema), result); + } + + private static Stream tableVersionSixNullSchemaCases() { + return Stream.of( + // curr txn requested between the two commits: adopt the earlier commit's schema + Arguments.of("0015", SCHEMA1), + // curr txn requested after all commits: adopt the latest commit's schema + Arguments.of("0040", SCHEMA2), + // curr txn requested before all commits: fall back to the table create schema + Arguments.of("0005", SCHEMA3)); + } + + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testConflictSecondCommitDifferentSchema(int writeTableVersion) throws Exception { + setupInstants(null, SCHEMA1, SCHEMA2, true, false, writeTableVersion); assertThrows(HoodieSchemaEvolutionConflictException.class, () -> strategy.resolveConcurrentSchemaEvolution(table, config, Option.empty(), nonTableCompactionInstant)); } - @Test - void testConflictSecondCommitSameSchema() throws Exception { - setupInstants(null, SCHEMA1, SCHEMA1, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testConflictSecondCommitSameSchema(int writeTableVersion) throws Exception { + setupInstants(null, SCHEMA1, SCHEMA1, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, Option.empty(), nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA1), result); } - @Test - void testNoConflictSameSchema() throws Exception { - setupInstants(SCHEMA1, SCHEMA1, SCHEMA1, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNoConflictSameSchema(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA1, SCHEMA1, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA1), result); } - @Test - void testNoConflictBackwardsCompatible1() throws Exception { - setupInstants(SCHEMA1, SCHEMA2, SCHEMA1, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNoConflictBackwardsCompatible1(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA2, SCHEMA1, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA2), result); } - @Test - void testNoConflictBackwardsCompatible2() throws Exception { - setupInstants(SCHEMA1, SCHEMA1, SCHEMA2, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNoConflictBackwardsCompatible2(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA1, SCHEMA2, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA2), result); } - @Test - void testNoConflictConcurrentEvolutionSameSchema() throws Exception { - setupInstants(SCHEMA1, SCHEMA2, SCHEMA2, true, false); + @ParameterizedTest + @ValueSource(ints = {6, 8}) + void testNoConflictConcurrentEvolutionSameSchema(int writeTableVersion) throws Exception { + setupInstants(SCHEMA1, SCHEMA2, SCHEMA2, true, false, writeTableVersion); HoodieSchema result = strategy.resolveConcurrentSchemaEvolution( table, config, lastCompletedTxnOwnerInstant, nonTableCompactionInstant).get(); assertEquals(HoodieSchema.parse(SCHEMA2), result); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/InstantComparator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/InstantComparator.java index 4be5f941b9d86..2824866706265 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/InstantComparator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/InstantComparator.java @@ -37,4 +37,20 @@ public interface InstantComparator extends Serializable { * @return {@link Comparator} that orders primarily based on completion time and secondary ordering based on {@link #requestedTimeOrderedComparator()}. */ Comparator completionTimeOrderedComparator(); + + /** + * Returns the comparator implementing the instant ordering of this timeline version: + * completion-time based for v2, requested-time based for v1. + * + *

    Implementations must keep this consistent with {@link #getOrderingTime(HoodieInstant)}, + * which returns the primary timestamp this comparator orders by: a timeline walk that sorts by + * this comparator and then bounds instants by {@code getOrderingTime} relies on the two agreeing. + */ + Comparator orderingComparator(); + + /** + * Returns the timestamp ordering the given instant in this timeline version: completion time + * for v2 (null if the instant is not completed yet), requested time for v1. + */ + String getOrderingTime(HoodieInstant instant); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/InstantComparatorV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/InstantComparatorV1.java index 8b21e672b7974..2c4c150a1416e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/InstantComparatorV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/InstantComparatorV1.java @@ -74,4 +74,14 @@ public Comparator requestedTimeOrderedComparator() { public Comparator completionTimeOrderedComparator() { return COMPLETION_TIME_BASED_COMPARATOR; } + + @Override + public Comparator orderingComparator() { + return REQUESTED_TIME_BASED_COMPARATOR; + } + + @Override + public String getOrderingTime(HoodieInstant instant) { + return instant.requestedTime(); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/InstantComparatorV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/InstantComparatorV2.java index 4b20708789bef..f6b51292009cf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/InstantComparatorV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/InstantComparatorV2.java @@ -70,4 +70,18 @@ public Comparator requestedTimeOrderedComparator() { public Comparator completionTimeOrderedComparator() { return COMPLETION_TIME_BASED_COMPARATOR; } + + @Override + public Comparator orderingComparator() { + return COMPLETION_TIME_BASED_COMPARATOR; + } + + // On tables upgraded from version 6, completion times of instants before the upgrade boundary are + // backfilled from the meta file modification time and are not guaranteed durable ordering keys. + // This is safe: the upgrade runs a full compaction with no concurrent writers, so those pre-upgrade + // completion times no longer affect concurrency or file-slicing decisions. + @Override + public String getOrderingTime(HoodieInstant instant) { + return instant.getCompletionTime(); + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestInstantComparators.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestInstantComparators.java index f8b51f79ff692..c01cf1cf681ff 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestInstantComparators.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestInstantComparators.java @@ -20,6 +20,7 @@ package org.apache.hudi.common.table.timeline; import org.apache.hudi.common.table.timeline.versioning.common.InstantComparators; +import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1; import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2; import org.junit.jupiter.api.Test; @@ -30,6 +31,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; class TestInstantComparators { @Test @@ -46,6 +48,39 @@ void testCompletionTimeOrdering() { assertEquals(Arrays.asList(instant1, instant3, instant2, instant4, instant5), instants); } + @Test + void testOrderingComparatorPerTimelineVersion() { + // Completion order (001 completes at 005, 002 completes at 003) inverts requested order. + HoodieInstant instant1 = createCompletedHoodieInstant("001", "005"); + HoodieInstant instant2 = createCompletedHoodieInstant("002", "003"); + + // Timeline layout v1 orders by requested time. + List instants = Arrays.asList(instant2, instant1); + instants.sort(new InstantComparatorV1().orderingComparator()); + assertEquals(Arrays.asList(instant1, instant2), instants); + + // Timeline layout v2 orders by completion time. + instants = Arrays.asList(instant1, instant2); + instants.sort(new InstantComparatorV2().orderingComparator()); + assertEquals(Arrays.asList(instant2, instant1), instants); + } + + @Test + void testGetOrderingTimePerTimelineVersion() { + HoodieInstant completed = createCompletedHoodieInstant("001", "005"); + HoodieInstant inflight = createInflightHoodieInstant("002"); + + // Timeline layout v1 orders by requested time. + InstantComparator comparatorV1 = new InstantComparatorV1(); + assertEquals("001", comparatorV1.getOrderingTime(completed)); + assertEquals("002", comparatorV1.getOrderingTime(inflight)); + + // Timeline layout v2 orders by completion time, which an inflight instant does not have yet. + InstantComparator comparatorV2 = new InstantComparatorV2(); + assertEquals("005", comparatorV2.getOrderingTime(completed)); + assertNull(comparatorV2.getOrderingTime(inflight)); + } + private static HoodieInstant createCompletedHoodieInstant(String requestedTime, String completionTime) { return new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, requestedTime, completionTime, InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR); } From 7b1f070087bbbec33735b4fdb810771a0248d971 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Wed, 29 Jul 2026 17:34:41 +0800 Subject: [PATCH 218/255] test(flink): improve streamer config and schema provider coverage (#19392) Adapted for release-1.2.1: TestOptionsResolver#testConcurrencyControlModes keeps only the NON_BLOCKING_CONCURRENCY_CONTROL assertion. The OPTIMISTIC_CONCURRENCY_CONTROL half calls OptionsResolver#isOptimisticConcurrencyControl, which this branch does not have; master re-added that method in 348e7f13a7fb (#18946, "feat(flink): Add validation to reject multiple writers for flink RLI writes"), which is not backported. A comment in the test records the omission. The conflict also offered a HoodieTableConfig import that #19392 does not add and nothing here uses, so it was left out to avoid an unused import. Everything else applies unchanged, including both production changes: the instantRetryInterval seconds-to-millis conversion in FlinkStreamerConfig and the --config key=value parsing in StreamerUtil. (cherry picked from commit ed7327b325894405a49fb9ab1f7730f851a51b10) --- .../hudi/streamer/FlinkStreamerConfig.java | 4 +- .../org/apache/hudi/util/StreamerUtil.java | 8 +- .../configuration/TestOptionsInference.java | 98 ++++++++ .../configuration/TestOptionsResolver.java | 160 ++++++++++++- .../schema/TestFilebasedSchemaProvider.java | 92 +++++++ .../schema/TestSchemaRegistryProvider.java | 127 ++++++++++ .../clustering/TestFlinkClusteringConfig.java | 110 +++++++++ .../compact/TestFlinkCompactionConfig.java | 79 ++++++ .../streamer/TestFlinkStreamerConfig.java | 146 ++++++++++++ .../streamer/TestHoodieFlinkStreamer.java | 224 ++++++++++++++++++ 10 files changed, 1045 insertions(+), 3 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestFilebasedSchemaProvider.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestSchemaRegistryProvider.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/clustering/TestFlinkClusteringConfig.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestFlinkCompactionConfig.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestFlinkStreamerConfig.java create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestHoodieFlinkStreamer.java diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java index 278daacb081e3..52b8b9b6ca9ea 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java @@ -42,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH; import static org.apache.hudi.configuration.FlinkOptions.PARTITION_FORMAT_DAY; @@ -433,7 +434,8 @@ public static org.apache.flink.configuration.Configuration toFlinkConfig(FlinkSt conf.set(FlinkOptions.RECORD_MERGER_STRATEGY_ID, config.recordMergerStrategy); conf.set(FlinkOptions.PRE_COMBINE, config.preCombine); conf.set(FlinkOptions.RETRY_TIMES, Integer.parseInt(config.instantRetryTimes)); - conf.set(FlinkOptions.RETRY_INTERVAL_MS, Long.parseLong(config.instantRetryInterval)); + conf.set(FlinkOptions.RETRY_INTERVAL_MS, + TimeUnit.SECONDS.toMillis(Long.parseLong(config.instantRetryInterval))); conf.set(FlinkOptions.IGNORE_FAILED, config.commitOnErrors); conf.set(FlinkOptions.RECORD_KEY_FIELD, config.recordKeyField); conf.set(FlinkOptions.PARTITION_PATH_FIELD, config.partitionPathField); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index c17e0528e4586..2f9c46b590b40 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -137,7 +137,13 @@ public static TypedProperties appendKafkaProps(FlinkStreamerConfig config) { public static TypedProperties getProps(FlinkStreamerConfig cfg) { if (cfg.propsFilePath.isEmpty()) { - return new TypedProperties(); + TypedProperties properties = new TypedProperties(); + cfg.configs.forEach(x -> { + String[] kv = x.split("="); + ValidationUtils.checkArgument(kv.length == 2); + properties.setProperty(kv[0], kv[1]); + }); + return properties; } return readConfig( HadoopConfigurations.getHadoopConf(cfg), diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java index 5d13ae0e67ea8..0bc69ae910b87 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java @@ -22,7 +22,12 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.util.ClientIds; +import org.apache.flink.FlinkVersion; +import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ExecutionOptions; +import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.SchedulerExecutionMode; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -30,6 +35,9 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test cases for {@link OptionsInference}. @@ -38,6 +46,83 @@ public class TestOptionsInference { @TempDir File tempFile; + @Test + void testSetupSourceAndSinkTasks() { + Configuration conf = new Configuration(); + + OptionsInference.setupSourceTasks(conf, 3); + OptionsInference.setupSinkTasks(conf, 4); + + assertEquals(3, conf.get(FlinkOptions.READ_TASKS)); + assertEquals(4, conf.get(FlinkOptions.WRITE_TASKS)); + assertEquals(4, conf.get(FlinkOptions.BUCKET_ASSIGN_TASKS)); + assertEquals(4, conf.get(FlinkOptions.COMPACTION_TASKS)); + assertEquals(4, conf.get(FlinkOptions.CLUSTERING_TASKS)); + assertEquals(4, conf.get(FlinkOptions.INDEX_WRITE_TASKS)); + + conf.set(FlinkOptions.READ_TASKS, 7); + conf.set(FlinkOptions.WRITE_TASKS, 8); + conf.set(FlinkOptions.BUCKET_ASSIGN_TASKS, 9); + conf.set(FlinkOptions.COMPACTION_TASKS, 10); + conf.set(FlinkOptions.CLUSTERING_TASKS, 11); + conf.set(FlinkOptions.INDEX_WRITE_TASKS, 12); + + OptionsInference.setupSourceTasks(conf, 20); + OptionsInference.setupSinkTasks(conf, 20); + + assertEquals(7, conf.get(FlinkOptions.READ_TASKS)); + assertEquals(8, conf.get(FlinkOptions.WRITE_TASKS)); + assertEquals(9, conf.get(FlinkOptions.BUCKET_ASSIGN_TASKS)); + assertEquals(10, conf.get(FlinkOptions.COMPACTION_TASKS)); + assertEquals(11, conf.get(FlinkOptions.CLUSTERING_TASKS)); + assertEquals(12, conf.get(FlinkOptions.INDEX_WRITE_TASKS)); + } + + @Test + void testSetupRuntimeConfigurations() { + Configuration conf = new Configuration(); + conf.set(JobManagerOptions.SCHEDULER, JobManagerOptions.SchedulerType.AdaptiveBatch); + Configuration runtimeConf = new Configuration(); + runtimeConf.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH); + + OptionsInference.setupRuntimeConfigs(conf, runtimeConf); + + if (FlinkVersion.current().toString().compareTo("2.0") >= 0) { + assertTrue(conf.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + } else { + assertFalse(conf.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + } + + conf.set(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION, false); + runtimeConf.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.STREAMING); + OptionsInference.setupRuntimeConfigs(conf, runtimeConf); + assertFalse(conf.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + } + + @Test + void testSchedulerTypeResolutionThroughRuntimeSetup() { + Configuration runtimeConf = new Configuration(); + runtimeConf.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH); + boolean isFlink2 = FlinkVersion.current().toString().compareTo("2.0") >= 0; + + Configuration reactive = new Configuration(); + reactive.set(JobManagerOptions.SCHEDULER_MODE, SchedulerExecutionMode.REACTIVE); + OptionsInference.setupRuntimeConfigs(reactive, runtimeConf); + assertEquals(isFlink2, + reactive.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + + Configuration adaptive = new Configuration(); + adaptive.set(JobManagerOptions.SCHEDULER, JobManagerOptions.SchedulerType.Adaptive); + OptionsInference.setupRuntimeConfigs(adaptive, runtimeConf); + assertEquals(isFlink2, + adaptive.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + + Configuration defaultScheduler = new Configuration(); + defaultScheduler.set(JobManagerOptions.SCHEDULER, JobManagerOptions.SchedulerType.Default); + OptionsInference.setupRuntimeConfigs(defaultScheduler, runtimeConf); + assertFalse(defaultScheduler.get(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION)); + } + @Test void testSetupClientId() throws Exception { Configuration conf = getConf(); @@ -75,4 +160,17 @@ private Configuration getConf() { conf.set(FlinkOptions.PATH, tempFile.getAbsolutePath()); return conf; } + + @Test + void testClientIdAndIndexSetupAreNoOpsWhenNotApplicable() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.PATH, tempFile.getAbsolutePath()); + conf.set(FlinkOptions.INDEX_TYPE, "BLOOM"); + + OptionsInference.setupClientId(conf); + OptionsInference.setupIndexConfigs(conf); + + assertFalse(conf.contains(FlinkOptions.WRITE_CLIENT_ID)); + assertFalse(conf.contains(FlinkOptions.BUCKET_INDEX_PARTITION_EXPRESSIONS)); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java index a70e7dc9f7df5..ac679dba8c184 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java @@ -18,26 +18,39 @@ package org.apache.hudi.configuration; +import org.apache.hudi.client.transaction.BucketIndexConcurrentFileWritesConflictResolutionStrategy; +import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode; +import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.sink.buffer.BufferMemoryType; import org.apache.hudi.utils.TestConfigurations; +import org.apache.flink.api.common.functions.Partitioner; import org.apache.flink.configuration.Configuration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -46,7 +59,7 @@ public class TestOptionsResolver { @TempDir File tempFile; - + @Test void testGetIndexType() { Configuration conf = getConf(); @@ -225,4 +238,149 @@ void testEstimateFileGroupCountForGlobalRLI() { conf.setString(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key(), "11"); assertEquals(11, OptionsResolver.estimateFileGroupCountForRLI(conf)); } + + @Test + void testIncrementalJobGraphPredicate() { + Configuration conf = new Configuration(); + assertFalse(OptionsResolver.isIncrementalJobGraph(conf)); + conf.set(FlinkOptions.WRITE_INCREMENTAL_JOB_GRAPH_GENERATION, true); + assertTrue(OptionsResolver.isIncrementalJobGraph(conf)); + } + + @Test + void testTableTypePredicates() { + Configuration conf = new Configuration(); + assertTrue(OptionsResolver.isCowTable(conf)); + assertFalse(OptionsResolver.isMorTable(conf)); + assertFalse(OptionsResolver.isMorTable(Collections.emptyMap())); + conf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name().toLowerCase()); + assertTrue(OptionsResolver.isMorTable(conf)); + assertTrue(OptionsResolver.isMorTable( + Collections.singletonMap(FlinkOptions.TABLE_TYPE.key(), HoodieTableType.MERGE_ON_READ.name()))); + } + + @Test + void testOperationTypePredicates() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.OPERATION, WriteOperationType.INSERT.value()); + assertTrue(OptionsResolver.isInsertOperation(conf)); + conf.set(FlinkOptions.OPERATION, WriteOperationType.UPSERT.value()); + assertTrue(OptionsResolver.isUpsertOperation(conf)); + conf.set(FlinkOptions.OPERATION, WriteOperationType.BULK_INSERT.value()); + assertTrue(OptionsResolver.isBulkInsertOperation(conf)); + } + + @Test + void testPayloadAndCompactionPredicates() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.PAYLOAD_CLASS_NAME, DefaultHoodieRecordPayload.class.getName()); + assertTrue(OptionsResolver.isDefaultHoodieRecordPayloadClazz(conf)); + conf.set(FlinkOptions.COMPACTION_TRIGGER_STRATEGY, FlinkOptions.TIME_ELAPSED.toUpperCase()); + assertTrue(OptionsResolver.isDeltaTimeCompaction(conf)); + conf.set(FlinkOptions.COMPACTION_TRIGGER_STRATEGY, FlinkOptions.NUM_COMMITS); + assertFalse(OptionsResolver.isDeltaTimeCompaction(conf)); + } + + @Test + void testReadCommitsLimit() { + Configuration conf = new Configuration(); + assertEquals(-1, OptionsResolver.getReadCommitsLimit(conf)); + conf.set(FlinkOptions.READ_COMMITS_LIMIT, 5); + assertEquals(5, OptionsResolver.getReadCommitsLimit(conf)); + } + + @Test + void testCdcOptions() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.SUPPLEMENTAL_LOGGING_MODE, + HoodieCDCSupplementalLoggingMode.DATA_BEFORE_AFTER.name().toLowerCase()); + assertEquals(HoodieCDCSupplementalLoggingMode.DATA_BEFORE_AFTER, + OptionsResolver.getCDCSupplementalLoggingMode(conf)); + + conf.set(FlinkOptions.READ_CDC_FROM_CHANGELOG, false); + assertFalse(OptionsResolver.readCDCFromChangelog(conf)); + } + + @Test + void testIndexKeyFields() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.RECORD_KEY_FIELD, "id,tenant"); + assertEquals("id", OptionsResolver.getIndexKeyFields(conf).get(0)); + assertEquals("tenant", OptionsResolver.getIndexKeyFields(conf).get(1)); + } + + @Test + void testSchemaAndTimestampOptions() { + Configuration conf = new Configuration(); + conf.setString(HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key(), "true"); + assertTrue(OptionsResolver.isSchemaEvolutionEnabled(conf)); + conf.setString(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), "true"); + assertTrue(OptionsResolver.isConsistentLogicalTimestampEnabled(conf)); + conf.setString(HoodieCommonConfig.INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT.key(), + HollowCommitHandling.USE_TRANSITION_TIME.name()); + assertTrue(OptionsResolver.isReadByTxnCompletionTime(conf)); + } + + @Test + void testWriteFlags() { + Configuration conf = new Configuration(); + conf.setString(HoodieWriteConfig.ALLOW_EMPTY_COMMIT.key(), "true"); + assertTrue(OptionsResolver.allowCommitOnEmptyBatch(conf)); + conf.setString(HoodieWriteConfig.COMPLEX_KEYGEN_NEW_ENCODING.key(), "true"); + assertTrue(OptionsResolver.useComplexKeygenNewEncoding(conf)); + } + + @Test + void testConcurrencyControlModes() { + Configuration conf = new Configuration(); + conf.setString(HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key(), + WriteConcurrencyMode.NON_BLOCKING_CONCURRENCY_CONTROL.name()); + assertTrue(OptionsResolver.isNonBlockingConcurrencyControl(conf)); + // The OPTIMISTIC_CONCURRENCY_CONTROL assertion is omitted: OptionsResolver + // #isOptimisticConcurrencyControl arrives with 348e7f13a7fb (#18946) and does not exist here. + } + + @Test + void testInsertPartitioner() { + Configuration conf = new Configuration(); + assertFalse(OptionsResolver.getInsertPartitioner(conf).isPresent()); + conf.set(FlinkOptions.INSERT_PARTITIONER_CLASS_NAME, TestPartitioner.class.getName()); + assertTrue(OptionsResolver.getInsertPartitioner(conf).isPresent()); + conf.set(FlinkOptions.INSERT_PARTITIONER_CLASS_NAME, String.class.getName()); + assertThrows(HoodieException.class, () -> OptionsResolver.getInsertPartitioner(conf)); + } + + @Test + void testConflictResolutionStrategies() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BLOOM.name()); + assertInstanceOf(SimpleConcurrentFileWritesConflictResolutionStrategy.class, + OptionsResolver.getConflictResolutionStrategy(conf)); + conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name()); + assertInstanceOf(BucketIndexConcurrentFileWritesConflictResolutionStrategy.class, + OptionsResolver.getConflictResolutionStrategy(conf)); + } + + @Test + void testWriteBufferSizingAndManagedMemory() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.WRITE_TASK_MAX_SIZE, 300D); + conf.set(FlinkOptions.WRITE_MERGE_MAX_MEMORY, 50); + assertEquals(150L * 1024 * 1024, OptionsResolver.getWriteBufferSizeInBytes(conf)); + conf.set(FlinkOptions.WRITE_TASK_MAX_SIZE, 100D); + assertThrows(IllegalStateException.class, () -> OptionsResolver.getWriteBufferSizeInBytes(conf)); + + conf.set(FlinkOptions.WRITE_BUFFER_MEMORY_TYPE, BufferMemoryType.MANAGED.name().toLowerCase()); + assertTrue(OptionsResolver.isManagedMemoryBufferEnabled(conf)); + } + + public static class TestPartitioner implements Partitioner { + public TestPartitioner(Configuration conf) { + } + + @Override + public int partition(String key, int numPartitions) { + return 0; + } + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestFilebasedSchemaProvider.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestFilebasedSchemaProvider.java new file mode 100644 index 0000000000000..ebcbee01d93f1 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestFilebasedSchemaProvider.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.schema; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.flink.configuration.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link FilebasedSchemaProvider}. + */ +class TestFilebasedSchemaProvider { + + private static final String SOURCE_SCHEMA_KEY = "hoodie.streamer.schemaprovider.source.schema.file"; + private static final String TARGET_SCHEMA_KEY = "hoodie.streamer.schemaprovider.target.schema.file"; + private static final String SOURCE_SCHEMA = + "{\"type\":\"record\",\"name\":\"SourceRecord\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"; + private static final String TARGET_SCHEMA = + "{\"type\":\"record\",\"name\":\"TargetRecord\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"ts\",\"type\":\"long\",\"default\":0}]}"; + + @TempDir + Path tempDir; + + @Test + void testConfigurationConstructorReturnsSourceSchema() throws IOException { + Path sourcePath = writeSchema("source.avsc", SOURCE_SCHEMA); + Configuration conf = new Configuration(); + conf.set(FlinkOptions.SOURCE_AVRO_SCHEMA_PATH, sourcePath.toString()); + + FilebasedSchemaProvider provider = new FilebasedSchemaProvider(conf); + + assertEquals("SourceRecord", provider.getSourceSchema().getName()); + assertEquals("SourceRecord", provider.getTargetSchema().getName()); + } + + @Test + void testTypedPropertiesConstructorReturnsSeparateTargetSchema() throws IOException { + Path sourcePath = writeSchema("source.avsc", SOURCE_SCHEMA); + Path targetPath = writeSchema("target.avsc", TARGET_SCHEMA); + TypedProperties props = new TypedProperties(); + props.setProperty(SOURCE_SCHEMA_KEY, sourcePath.toString()); + props.setProperty(TARGET_SCHEMA_KEY, targetPath.toString()); + + FilebasedSchemaProvider provider = new FilebasedSchemaProvider(props); + + assertEquals("SourceRecord", provider.getSourceSchema().getName()); + assertEquals("TargetRecord", provider.getTargetSchema().getName()); + } + + @Test + void testReadFailureIsWrapped() { + Configuration conf = new Configuration(); + conf.set(FlinkOptions.SOURCE_AVRO_SCHEMA_PATH, tempDir.resolve("missing.avsc").toString()); + + assertThrows(HoodieIOException.class, () -> new FilebasedSchemaProvider(conf)); + } + + private Path writeSchema(String fileName, String schema) throws IOException { + Path path = tempDir.resolve(fileName); + Files.write(path, schema.getBytes(StandardCharsets.UTF_8)); + return path; + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestSchemaRegistryProvider.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestSchemaRegistryProvider.java new file mode 100644 index 0000000000000..d73fdf1a9bf8c --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/schema/TestSchemaRegistryProvider.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.schema; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * Tests for {@link SchemaRegistryProvider}. + */ +class TestSchemaRegistryProvider { + + private static final String SOURCE_URL_KEY = "hoodie.streamer.schemaprovider.registry.url"; + private static final String TARGET_URL_KEY = "hoodie.streamer.schemaprovider.registry.targetUrl"; + private static final String SOURCE_SCHEMA = + "{\"type\":\"record\",\"name\":\"SourceRecord\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"; + private static final String TARGET_SCHEMA = + "{\"type\":\"record\",\"name\":\"TargetRecord\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"ts\",\"type\":\"long\",\"default\":0}]}"; + + @Test + void testReturnsSourceAndTargetSchemasFromRegistryResponses() { + TypedProperties props = new TypedProperties(); + props.setProperty(SOURCE_URL_KEY, "http://source:secret@localhost/source"); + props.setProperty(TARGET_URL_KEY, "http://localhost/target"); + StubSchemaRegistryProvider provider = new StubSchemaRegistryProvider(props); + + assertEquals("SourceRecord", provider.getSourceSchema().getName()); + assertEquals("source:secret", provider.authorizationCredentials); + assertEquals("TargetRecord", provider.getTargetSchema().getName()); + assertEquals("TargetRecord", provider.getTargetHoodieSchema().getName()); + } + + @Test + void testAuthorizationHeaderIsBase64Encoded() { + TypedProperties props = new TypedProperties(); + props.setProperty(SOURCE_URL_KEY, "http://localhost/source"); + StubSchemaRegistryProvider provider = new StubSchemaRegistryProvider(props); + HttpURLConnection connection = mock(HttpURLConnection.class); + + provider.setAuthorizationHeader("source:secret", connection); + + verify(connection).setRequestProperty("Authorization", "Basic c291cmNlOnNlY3JldA=="); + } + + @Test + void testTargetDefaultsToSourceRegistry() { + TypedProperties props = new TypedProperties(); + props.setProperty(SOURCE_URL_KEY, "http://localhost/source"); + StubSchemaRegistryProvider provider = new StubSchemaRegistryProvider(props); + + Schema targetSchema = provider.getTargetSchema(); + + assertEquals("SourceRecord", targetSchema.getName()); + assertNotNull(provider.getTargetHoodieSchema()); + } + + @Test + void testRegistryReadFailureIsWrapped() { + TypedProperties props = new TypedProperties(); + props.setProperty(SOURCE_URL_KEY, "http://localhost/failure"); + StubSchemaRegistryProvider provider = new StubSchemaRegistryProvider(props); + + assertThrows(HoodieIOException.class, provider::getSourceSchema); + assertThrows(HoodieIOException.class, provider::getTargetSchema); + } + + private static class StubSchemaRegistryProvider extends SchemaRegistryProvider { + private String authorizationCredentials; + + StubSchemaRegistryProvider(TypedProperties props) { + super(props); + } + + @Override + protected void setAuthorizationHeader(String creds, HttpURLConnection connection) { + super.setAuthorizationHeader(creds, connection); + authorizationCredentials = creds; + } + + @Override + protected InputStream getStream(HttpURLConnection connection) throws IOException { + String path = connection.getURL().getPath(); + if ("/failure".equals(path)) { + throw new IOException("schema registry unavailable"); + } + String schema = "/target".equals(path) ? TARGET_SCHEMA : SOURCE_SCHEMA; + String response = "{\"schema\":" + quote(schema) + "}"; + return new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)); + } + + private static String quote(String value) { + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/clustering/TestFlinkClusteringConfig.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/clustering/TestFlinkClusteringConfig.java new file mode 100644 index 0000000000000..b4340ceae20b0 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/clustering/TestFlinkClusteringConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.clustering; + +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.util.StreamerUtil; +import org.apache.hudi.utils.TestConfigurations; + +import com.beust.jcommander.JCommander; +import org.apache.flink.configuration.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link FlinkClusteringConfig}. + */ +class TestFlinkClusteringConfig { + + @TempDir + Path tempDir; + + @Test + void testParseAndDeriveFlinkConfiguration() throws Exception { + Configuration tableConf = TestConfigurations.getDefaultConf(tempDir.toString()); + tableConf.set(FlinkOptions.URL_ENCODE_PARTITIONING, true); + tableConf.set(FlinkOptions.HIVE_STYLE_PARTITIONING, true); + StreamerUtil.initTableIfNotExists(tableConf); + + FlinkClusteringConfig config = new FlinkClusteringConfig(); + JCommander.newBuilder().addObject(config).build().parse( + "--path", tempDir.toString(), + "--clustering-delta-commits", "6", + "--clustering-tasks", "4", + "--clean-retain-commits", "12", + "--clean-retain-hours", "36", + "--clean-retain-file-versions", "7", + "--archive-min-commits", "25", + "--archive-max-commits", "40", + "--schedule", + "--clean-async-enabled", + "--plan-partition-filter-mode", "RECENT_DAYS", + "--target-file-max-bytes", "1048576", + "--small-file-limit", "524288", + "--skip-from-latest-partitions", "2", + "--sort-columns", "event_ts,order_id", + "--sort-memory", "256", + "--max-num-groups", "10", + "--target-partitions", "5", + "--cluster-begin-partition", "2026-01-01", + "--cluster-end-partition", "2026-01-31", + "--partition-regex-pattern", "2026-01-.*", + "--partition-selected", "2026-01-01,2026-01-02", + "--hoodie-conf", "hoodie.test.clustering.option=from-cli"); + + Configuration conf = FlinkClusteringConfig.toFlinkConfig(config); + + assertEquals(tempDir.toString(), conf.get(FlinkOptions.PATH)); + assertEquals(6, conf.get(FlinkOptions.CLUSTERING_DELTA_COMMITS)); + assertEquals(4, conf.get(FlinkOptions.CLUSTERING_TASKS)); + assertEquals(12, conf.get(FlinkOptions.CLEAN_RETAIN_COMMITS)); + assertEquals(36, conf.get(FlinkOptions.CLEAN_RETAIN_HOURS)); + assertEquals(7, conf.get(FlinkOptions.CLEAN_RETAIN_FILE_VERSIONS)); + assertEquals(25, conf.get(FlinkOptions.ARCHIVE_MIN_COMMITS)); + assertEquals(40, conf.get(FlinkOptions.ARCHIVE_MAX_COMMITS)); + assertEquals("RECENT_DAYS", conf.get(FlinkOptions.CLUSTERING_PLAN_PARTITION_FILTER_MODE_NAME)); + assertEquals(1048576L, conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_TARGET_FILE_MAX_BYTES)); + assertEquals(524288L, conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_SMALL_FILE_LIMIT)); + assertEquals(2, conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST)); + assertEquals("event_ts,order_id", conf.get(FlinkOptions.CLUSTERING_SORT_COLUMNS)); + assertEquals(256, conf.get(FlinkOptions.WRITE_SORT_MEMORY)); + assertEquals(10, conf.get(FlinkOptions.CLUSTERING_MAX_NUM_GROUPS)); + assertEquals(5, conf.get(FlinkOptions.CLUSTERING_TARGET_PARTITIONS)); + assertEquals("2026-01-01", + conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_CLUSTER_BEGIN_PARTITION)); + assertEquals("2026-01-31", + conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_CLUSTER_END_PARTITION)); + assertEquals("2026-01-.*", + conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_PARTITION_REGEX_PATTERN)); + assertEquals("2026-01-01,2026-01-02", + conf.get(FlinkOptions.CLUSTERING_PLAN_STRATEGY_PARTITION_SELECTED)); + assertTrue(conf.get(FlinkOptions.CLEAN_ASYNC_ENABLED)); + assertFalse(conf.get(FlinkOptions.CLUSTERING_ASYNC_ENABLED)); + assertTrue(conf.get(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED)); + assertTrue(conf.get(FlinkOptions.URL_ENCODE_PARTITIONING)); + assertTrue(conf.get(FlinkOptions.HIVE_STYLE_PARTITIONING)); + assertEquals("from-cli", conf.getString("hoodie.test.clustering.option", null)); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestFlinkCompactionConfig.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestFlinkCompactionConfig.java new file mode 100644 index 0000000000000..10ac0f7357dc1 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestFlinkCompactionConfig.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.sink.compact; + +import org.apache.hudi.common.config.HoodieMemoryConfig; +import org.apache.hudi.common.config.HoodieReaderConfig; +import org.apache.hudi.configuration.FlinkOptions; + +import com.beust.jcommander.JCommander; +import org.apache.flink.configuration.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link FlinkCompactionConfig}. + */ +class TestFlinkCompactionConfig { + + @TempDir + Path tempDir; + + @Test + void testParseAndDeriveFlinkConfiguration() { + FlinkCompactionConfig config = new FlinkCompactionConfig(); + JCommander.newBuilder().addObject(config).build().parse( + "--path", tempDir.toString(), + "--compaction-trigger-strategy", FlinkCompactionConfig.NUM_OR_TIME, + "--disable-file-group-reader", + "--compaction-delta-commits", "8", + "--compaction-delta-seconds", "900", + "--clean-async-enabled", + "--compaction-max-memory", "256", + "--compaction-target-io", "2048", + "--compaction-tasks", "4", + "--schedule", + "--spillable_map_path", tempDir.resolve("spill").toString(), + "--hoodie-conf", "hoodie.test.compaction.option=from-cli"); + + Configuration conf = FlinkCompactionConfig.toFlinkConfig(config); + + assertEquals(tempDir.toString(), conf.get(FlinkOptions.PATH)); + assertEquals(FlinkCompactionConfig.NUM_OR_TIME, conf.get(FlinkOptions.COMPACTION_TRIGGER_STRATEGY)); + assertEquals(8, conf.get(FlinkOptions.COMPACTION_DELTA_COMMITS)); + assertEquals(900, conf.get(FlinkOptions.COMPACTION_DELTA_SECONDS)); + assertEquals(256, conf.get(FlinkOptions.COMPACTION_MAX_MEMORY)); + assertEquals(256, conf.get(FlinkOptions.WRITE_MERGE_MAX_MEMORY)); + assertEquals(2048L, conf.get(FlinkOptions.COMPACTION_TARGET_IO)); + assertEquals(4, conf.get(FlinkOptions.COMPACTION_TASKS)); + assertTrue(conf.get(FlinkOptions.CLEAN_ASYNC_ENABLED)); + assertFalse(conf.get(FlinkOptions.COMPACTION_OPERATION_EXECUTE_ASYNC_ENABLED)); + assertTrue(conf.get(FlinkOptions.COMPACTION_SCHEDULE_ENABLED)); + assertEquals(tempDir.resolve("spill").toString(), + conf.getString(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.key(), null)); + assertEquals("false", conf.getString(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), null)); + assertEquals("from-cli", conf.getString("hoodie.test.compaction.option", null)); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestFlinkStreamerConfig.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestFlinkStreamerConfig.java new file mode 100644 index 0000000000000..446b95558c577 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestFlinkStreamerConfig.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.streamer; + +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.configuration.FlinkOptions; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import org.apache.flink.configuration.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link FlinkStreamerConfig}. + */ +class TestFlinkStreamerConfig { + + @TempDir + Path tempDir; + + @Test + void testParseAndDeriveFlinkConfiguration() { + FlinkStreamerConfig config = parse( + "--kafka-topic", "orders", + "--kafka-group-id", "flink-writers", + "--kafka-bootstrap-servers", "broker:9092", + "--target-base-path", tempDir.toString(), + "--target-table", "orders_hudi", + "--table-type", "merge_on_read", + "--op", "INSERT", + "--record-key-field", "order_id", + "--partition-path-field", "order_date", + "--source-ordering-fields", "event_ts,seq_no", + "--instant-retry-times", "7", + "--instant-retry-interval", "2", + "--filter-dupes", + "--commit-on-errors", + "--metadata-enabled", + "--write-rate-limit", "500", + "--write-task-num", "6", + "--bucket-assign-num", "5", + "--index-bootstrap-num", "4", + "--source-avro-schema-path", "file:///tmp/source.avsc", + "--source-avro-schema", "{\"type\":\"record\",\"name\":\"order\",\"fields\":[]}", + "--compaction-tasks", "3", + "--clustering-tasks", "2", + "--hive-sync-enable", + "--hive-sync-db", "analytics", + "--hive-sync-table", "orders", + "--hoodie-conf", "hoodie.datasource.write.drop.partition.columns=true"); + + Configuration conf = FlinkStreamerConfig.toFlinkConfig(config); + + assertEquals(tempDir.toString(), conf.get(FlinkOptions.PATH)); + assertEquals("orders_hudi", conf.get(FlinkOptions.TABLE_NAME)); + assertEquals("MERGE_ON_READ", conf.get(FlinkOptions.TABLE_TYPE)); + assertEquals(WriteOperationType.INSERT.value(), conf.get(FlinkOptions.OPERATION)); + assertEquals("order_id", conf.get(FlinkOptions.RECORD_KEY_FIELD)); + assertEquals("order_date", conf.get(FlinkOptions.PARTITION_PATH_FIELD)); + assertEquals("event_ts,seq_no", conf.get(FlinkOptions.ORDERING_FIELDS)); + assertEquals(7, conf.get(FlinkOptions.RETRY_TIMES)); + assertEquals(2_000L, conf.get(FlinkOptions.RETRY_INTERVAL_MS)); + assertTrue(conf.get(FlinkOptions.PRE_COMBINE)); + assertTrue(conf.get(FlinkOptions.IGNORE_FAILED)); + assertTrue(conf.get(FlinkOptions.METADATA_ENABLED)); + assertEquals(500L, conf.get(FlinkOptions.WRITE_RATE_LIMIT)); + assertEquals(6, conf.get(FlinkOptions.WRITE_TASKS)); + assertEquals(5, conf.get(FlinkOptions.BUCKET_ASSIGN_TASKS)); + assertEquals(4, conf.get(FlinkOptions.INDEX_BOOTSTRAP_TASKS)); + assertEquals("file:///tmp/source.avsc", conf.get(FlinkOptions.SOURCE_AVRO_SCHEMA_PATH)); + assertEquals(3, conf.get(FlinkOptions.COMPACTION_TASKS)); + assertEquals(2, conf.get(FlinkOptions.CLUSTERING_TASKS)); + assertTrue(conf.get(FlinkOptions.HIVE_SYNC_ENABLED)); + assertEquals("analytics", conf.get(FlinkOptions.HIVE_SYNC_DB)); + assertEquals("orders", conf.get(FlinkOptions.HIVE_SYNC_TABLE)); + assertEquals("true", + conf.getString("hoodie.datasource.write.drop.partition.columns", null)); + } + + @Test + void testCustomKeyGeneratorTakesPrecedence() { + FlinkStreamerConfig config = parseRequiredOptions( + "--keygen-class", "org.example.CustomKeyGenerator", + "--keygen-type", "COMPLEX"); + + Configuration conf = FlinkStreamerConfig.toFlinkConfig(config); + + assertEquals("org.example.CustomKeyGenerator", conf.get(FlinkOptions.KEYGEN_CLASS_NAME)); + assertFalse(conf.contains(FlinkOptions.KEYGEN_TYPE)); + } + + @Test + void testRequiredOptionsAndNumericValuesAreValidated() { + FlinkStreamerConfig missingRequired = new FlinkStreamerConfig(); + assertThrows(ParameterException.class, + () -> JCommander.newBuilder().addObject(missingRequired).build().parse("--kafka-topic", "orders")); + + FlinkStreamerConfig invalidRetry = parseRequiredOptions("--instant-retry-times", "not-a-number"); + assertThrows(NumberFormatException.class, () -> FlinkStreamerConfig.toFlinkConfig(invalidRetry)); + } + + private FlinkStreamerConfig parseRequiredOptions(String... additionalArgs) { + String[] requiredArgs = { + "--kafka-topic", "orders", + "--kafka-group-id", "flink-writers", + "--kafka-bootstrap-servers", "broker:9092", + "--target-base-path", tempDir.toString(), + "--target-table", "orders_hudi", + "--table-type", "copy_on_write" + }; + String[] args = new String[requiredArgs.length + additionalArgs.length]; + System.arraycopy(requiredArgs, 0, args, 0, requiredArgs.length); + System.arraycopy(additionalArgs, 0, args, requiredArgs.length, additionalArgs.length); + return parse(args); + } + + private static FlinkStreamerConfig parse(String... args) { + FlinkStreamerConfig config = new FlinkStreamerConfig(); + JCommander.newBuilder().addObject(config).build().parse(args); + return config; + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestHoodieFlinkStreamer.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestHoodieFlinkStreamer.java new file mode 100644 index 0000000000000..f848faa0d6ee2 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/streamer/TestHoodieFlinkStreamer.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.streamer; + +import org.apache.hudi.client.model.HoodieFlinkInternalRow; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.configuration.OptionsInference; +import org.apache.hudi.configuration.OptionsResolver; +import org.apache.hudi.sink.transform.Transformer; +import org.apache.hudi.sink.utils.Pipelines; +import org.apache.hudi.util.StreamerUtil; +import org.apache.hudi.utils.StreamerUtils; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.StateBackendOptions; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.CheckpointConfig; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests the argument and pipeline wiring in {@link HoodieFlinkStreamer}. + */ +class TestHoodieFlinkStreamer { + + private static final String SOURCE_SCHEMA = + "{\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"; + + @Test + void testAppendPipelineWiringWithTransformer() throws Exception { + StreamExecutionEnvironment env = mockEnvironment(); + DataStream source = mock(DataStream.class); + DataStream transformed = mock(DataStream.class); + DataStream pipeline = mock(DataStream.class); + Transformer transformer = mock(Transformer.class); + when(transformer.apply(source)).thenReturn(transformed); + AtomicReference envConf = new AtomicReference<>(); + + // Static mocks intercept config inference and resolution so this test only exercises entry-point wiring. + try (MockedStatic environments = mockStatic(StreamExecutionEnvironment.class); + MockedStatic streamerUtils = mockStatic(StreamerUtils.class); + MockedStatic streamerUtil = mockStatic(StreamerUtil.class, CALLS_REAL_METHODS); + MockedStatic inference = mockStatic(OptionsInference.class); + MockedStatic resolver = mockStatic(OptionsResolver.class); + MockedStatic pipelines = mockStatic(Pipelines.class)) { + environments.when(() -> StreamExecutionEnvironment.getExecutionEnvironment(any(Configuration.class))) + .thenAnswer(invocation -> { + envConf.set(invocation.getArgument(0)); + return env; + }); + streamerUtils.when(() -> StreamerUtils.createKafkaStream( + same(env), any(RowType.class), eq("orders"), any())).thenReturn(source); + streamerUtil.when(() -> StreamerUtil.createTransformer(anyList())).thenReturn(Option.of(transformer)); + resolver.when(() -> OptionsResolver.isAppendMode(any())).thenReturn(true); + resolver.when(() -> OptionsResolver.needsAsyncClustering(any())).thenReturn(true); + pipelines.when(() -> Pipelines.append(any(), any(RowType.class), same(transformed))).thenReturn(pipeline); + + HoodieFlinkStreamer.main(args( + "--table-type", "COPY_ON_WRITE", + "--op", "INSERT", + "--transformer-class", "org.example.Transformer", + "--flink-checkpoint-path", "file:///tmp/checkpoints")); + + assertEquals("HashMapStateBackend", envConf.get().get(StateBackendOptions.STATE_BACKEND)); + assertEquals("filesystem", envConf.get().get(CheckpointingOptions.CHECKPOINT_STORAGE)); + assertEquals("file:///tmp/checkpoints", + envConf.get().get(CheckpointingOptions.CHECKPOINTS_DIRECTORY)); + pipelines.verify(() -> Pipelines.cluster(any(), any(RowType.class), same(pipeline))); + verify(env).execute("orders_hudi"); + } + } + + @Test + void testAppendPipelineFallbackWiring() throws Exception { + StreamExecutionEnvironment env = mockEnvironment(); + DataStream source = mock(DataStream.class); + DataStream pipeline = mock(DataStream.class); + + try (MockedStatic environments = mockStatic(StreamExecutionEnvironment.class); + MockedStatic streamerUtils = mockStatic(StreamerUtils.class); + MockedStatic inference = mockStatic(OptionsInference.class); + MockedStatic resolver = mockStatic(OptionsResolver.class); + MockedStatic pipelines = mockStatic(Pipelines.class)) { + environments.when(() -> StreamExecutionEnvironment.getExecutionEnvironment(any(Configuration.class))) + .thenReturn(env); + streamerUtils.when(() -> StreamerUtils.createKafkaStream( + same(env), any(RowType.class), eq("orders"), any())).thenReturn(source); + resolver.when(() -> OptionsResolver.isAppendMode(any())).thenReturn(true); + resolver.when(() -> OptionsResolver.needsAsyncClustering(any())).thenReturn(false); + pipelines.when(() -> Pipelines.append(any(), any(RowType.class), same(source))).thenReturn(pipeline); + + resolver.when(() -> OptionsResolver.isLazyFailedWritesCleaning(any())).thenReturn(true); + HoodieFlinkStreamer.main(args("--table-type", "COPY_ON_WRITE", "--op", "INSERT")); + pipelines.verify(() -> Pipelines.clean(any(), same(pipeline))); + + resolver.when(() -> OptionsResolver.isLazyFailedWritesCleaning(any())).thenReturn(false); + HoodieFlinkStreamer.main(args("--table-type", "COPY_ON_WRITE", "--op", "INSERT")); + pipelines.verify(() -> Pipelines.dummySink(same(pipeline))); + } + } + + @Test + void testUpsertPipelineWiringWithCompaction() throws Exception { + StreamExecutionEnvironment env = mockEnvironment(); + DataStream source = mock(DataStream.class); + DataStream bootstrapped = mock(DataStream.class); + DataStream pipeline = mock(DataStream.class); + + // Static mocks intercept config inference and resolution so this test only exercises entry-point wiring. + try (MockedStatic environments = mockStatic(StreamExecutionEnvironment.class); + MockedStatic streamerUtils = mockStatic(StreamerUtils.class); + MockedStatic inference = mockStatic(OptionsInference.class); + MockedStatic resolver = mockStatic(OptionsResolver.class); + MockedStatic pipelines = mockStatic(Pipelines.class)) { + environments.when(() -> StreamExecutionEnvironment.getExecutionEnvironment(any(Configuration.class))) + .thenReturn(env); + streamerUtils.when(() -> StreamerUtils.createKafkaStream( + same(env), any(RowType.class), eq("orders"), any())).thenReturn(source); + resolver.when(() -> OptionsResolver.isAppendMode(any())).thenReturn(false); + resolver.when(() -> OptionsResolver.needsAsyncCompaction(any())).thenReturn(true); + pipelines.when(() -> Pipelines.bootstrap(any(), any(RowType.class), same(source))) + .thenReturn(bootstrapped); + pipelines.when(() -> Pipelines.hoodieStreamWrite(any(), any(RowType.class), same(bootstrapped))) + .thenReturn(pipeline); + + HoodieFlinkStreamer.main(args("--table-type", "MERGE_ON_READ", "--op", "UPSERT")); + + pipelines.verify(() -> Pipelines.compact(any(), same(pipeline))); + verify(env).execute("orders_hudi"); + } + } + + @Test + void testUpsertPipelineFallbackWiring() throws Exception { + StreamExecutionEnvironment env = mockEnvironment(); + DataStream source = mock(DataStream.class); + DataStream bootstrapped = mock(DataStream.class); + DataStream pipeline = mock(DataStream.class); + + try (MockedStatic environments = mockStatic(StreamExecutionEnvironment.class); + MockedStatic streamerUtils = mockStatic(StreamerUtils.class); + MockedStatic inference = mockStatic(OptionsInference.class); + MockedStatic resolver = mockStatic(OptionsResolver.class); + MockedStatic pipelines = mockStatic(Pipelines.class)) { + environments.when(() -> StreamExecutionEnvironment.getExecutionEnvironment(any(Configuration.class))) + .thenReturn(env); + streamerUtils.when(() -> StreamerUtils.createKafkaStream( + same(env), any(RowType.class), eq("orders"), any())).thenReturn(source); + resolver.when(() -> OptionsResolver.isAppendMode(any())).thenReturn(false); + resolver.when(() -> OptionsResolver.needsAsyncCompaction(any())).thenReturn(false); + pipelines.when(() -> Pipelines.bootstrap(any(), any(RowType.class), same(source))) + .thenReturn(bootstrapped); + pipelines.when(() -> Pipelines.hoodieStreamWrite(any(), any(RowType.class), same(bootstrapped))) + .thenReturn(pipeline); + + resolver.when(() -> OptionsResolver.needsAsyncCleaning(any())).thenReturn(true); + HoodieFlinkStreamer.main(args("--table-type", "MERGE_ON_READ", "--op", "UPSERT")); + pipelines.verify(() -> Pipelines.clean(any(), same(pipeline))); + + resolver.when(() -> OptionsResolver.needsAsyncCleaning(any())).thenReturn(false); + HoodieFlinkStreamer.main(args("--table-type", "MERGE_ON_READ", "--op", "UPSERT")); + pipelines.verify(() -> Pipelines.dummySink(same(pipeline))); + } + } + + private static StreamExecutionEnvironment mockEnvironment() { + StreamExecutionEnvironment env = mock(StreamExecutionEnvironment.class); + CheckpointConfig checkpointConfig = mock(CheckpointConfig.class); + ExecutionConfig executionConfig = mock(ExecutionConfig.class); + when(env.getCheckpointConfig()).thenReturn(checkpointConfig); + when(env.getConfig()).thenReturn(executionConfig); + when(checkpointConfig.getCheckpointTimeout()).thenReturn(600_000L); + return env; + } + + private static String[] args(String... additionalArgs) { + String[] requiredArgs = { + "--kafka-topic", "orders", + "--kafka-group-id", "flink-writers", + "--kafka-bootstrap-servers", "broker:9092", + "--target-base-path", "file:///tmp/orders", + "--target-table", "orders_hudi", + "--source-avro-schema", SOURCE_SCHEMA + }; + String[] args = new String[requiredArgs.length + additionalArgs.length]; + System.arraycopy(requiredArgs, 0, args, 0, requiredArgs.length); + System.arraycopy(additionalArgs, 0, args, requiredArgs.length, additionalArgs.length); + return args; + } +} From 1f92b195964717a36aacb08cfb373c4f97ec9b1d Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 30 Jul 2026 22:21:05 +0800 Subject: [PATCH 219/255] feat(release): add a JDK 25 staging path for org.apache.hudi:hudi-trino (#19410) Adapted for release-1.2.1: - validate_staged_bundles.sh: added the hudi-trino entry as upstream does, but kept this branch's bundle list otherwise; upstream's line also introduces hudi-spark4.2-bundle_2.13, and there is no Spark 4.2 module here. - bot.yml: skipped. Upstream's change is a comment reword inside the validate-source job's "stage the source copy outside the workspace" block. This branch's validate-source job never received that restructure (it still creates hudi-tmp-repo inside the workspace from the Check Copyright step), so the comment being edited does not exist here and there is nothing to apply. Taking the conflict's incoming side would have imported the whole job restructure from an unrelated, unbackported commit. Everything else applies unchanged: the new deploy_staging_jars_java25.sh, the hudi-trino javadoc source/failOnError override, the release_guide.md steps, the source-copyright prune rationale, and the hudi_trino_ci.yml javadoc check. (cherry picked from commit d43cc5c054c5409e387f745d2cc4e3dcb5beefa5) --- .github/workflows/hudi_trino_ci.yml | 6 ++ hudi-trino/pom.xml | 11 +++ release/release_guide.md | 26 ++++-- scripts/release/deploy_staging_jars_java25.sh | 93 +++++++++++++++++++ scripts/release/validate_source_copyright.sh | 5 +- scripts/release/validate_staged_bundles.sh | 2 + 6 files changed, 131 insertions(+), 12 deletions(-) create mode 100755 scripts/release/deploy_staging_jars_java25.sh diff --git a/.github/workflows/hudi_trino_ci.yml b/.github/workflows/hudi_trino_ci.yml index 0cfd1684d817d..cbd7b851f42cb 100644 --- a/.github/workflows/hudi_trino_ci.yml +++ b/.github/workflows/hudi_trino_ci.yml @@ -142,6 +142,12 @@ jobs: - name: Build connector (JDK 25) if: needs.changes.outputs.trino == 'true' run: mvn $MVN_ARGS -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + # The release deploy (-DdeployArtifacts=true) runs javadoc:jar with doclint=none, + # and the module pom keeps failOnError=true. Reproduce that exact configuration here + # so a javadoc break fails the PR instead of surfacing during the staging deploy. + - name: Javadoc check (JDK 25) + if: needs.changes.outputs.trino == 'true' + run: mvn $MVN_ARGS -Phudi-trino -pl hudi-trino javadoc:jar -Ddoclint=none - name: Test connector (JDK 25) if: needs.changes.outputs.trino == 'true' run: mvn $MVN_ARGS -Phudi-trino,hudi-trino-tests -pl hudi-trino test diff --git a/hudi-trino/pom.xml b/hudi-trino/pom.xml index a272cd1865a39..4a488ee7309b6 100644 --- a/hudi-trino/pom.xml +++ b/hudi-trino/pom.xml @@ -595,6 +595,17 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + + ${hudi.trino.java.version} + + true + + org.apache.maven.plugins maven-enforcer-plugin diff --git a/release/release_guide.md b/release/release_guide.md index 14efd12860afd..ed78098f66c1a 100644 --- a/release/release_guide.md +++ b/release/release_guide.md @@ -408,8 +408,8 @@ Set up a few environment variables to simplify Maven commands that follow. This 1. This will deploy jar artifacts to the Apache Nexus Repository, which is the staging area for deploying jars to Maven Central. 2. Review all staged artifacts (https://repository.apache.org/). They should contain all relevant parts for each module, including pom.xml, jar, test jar, source, test source, javadoc, etc. Carefully review any new artifacts. 3. git checkout ${RELEASE_BRANCH} - 4. Given that certain bundle jars are built by Java 17 (Spark 4 bundle), multiple - scripts need to be run. Run each from the repository root directory (the one containing `packaging/`). + 4. Given that certain artifacts require newer JDKs (Java 17 for the Spark 4 bundles, Java 25 for hudi-trino), + multiple scripts need to be run. Run each from the repository root directory (the one containing `packaging/`). 1. For most modules with Java 11 build, run `export JAVA_HOME=$(/usr/libexec/java_home -v 11)` and `./scripts/release/deploy_staging_jars.sh 2>&1 | tee -a "/tmp/${RELEASE_VERSION}-${RC_NUM}.deploy1.log"` 1. when prompted for the passphrase, if you have multiple gpg keys in your keyring, make sure that you enter @@ -425,23 +425,29 @@ Set up a few environment variables to simplify Maven commands that follow. This module. See [checklist](#checklist-to-proceed-to-the-next-step). 2. Continue with Java 17 build for Spark 4 bundle, run `export JAVA_HOME=$(/usr/libexec/java_home -v 17)` and `./scripts/release/deploy_staging_jars_java17.sh 2>&1 | tee -a "/tmp/${RELEASE_VERSION}-${RC_NUM}.deploy2.log"` - 5. Note that the artifacts from Java 17 build are uploaded to a separate staging repo. Use the - `copy_staging_repo.sh` script to copy all artifacts from the Java 17 staging repo into the Java 11 staging repo + 3. Continue with Java 25 build for the hudi-trino connector, run `export JAVA_HOME=$(/usr/libexec/java_home -v 25)` + and `./scripts/release/deploy_staging_jars_java25.sh 2>&1 | tee -a "/tmp/${RELEASE_VERSION}-${RC_NUM}.deploy3.log"`. + This step must run after the Java 11 step in 9.4.1, which installs the upstream Hudi modules that hudi-trino + resolves from the local m2 (the script does not pass `-am` because Lombok cannot run on JDK 25). + 5. Note that each of the Java 17 and Java 25 builds uploads its artifacts to its own separate staging repo. Use the + `copy_staging_repo.sh` script once per extra staging repo to copy all artifacts into the Java 11 staging repo so that all artifacts stay in the same repo. - 1. Identify both staging repo IDs from [Apache Nexus Staging Repositories](https://repository.apache.org/#stagingRepositories) - (e.g., `orgapachehudi-1177` for Java 17, `orgapachehudi-1176` for Java 11). Make sure both repos are still in - the "open" state (not closed). + 1. Identify all staging repo IDs from [Apache Nexus Staging Repositories](https://repository.apache.org/#stagingRepositories) + (e.g., `orgapachehudi-1177` for Java 17, `orgapachehudi-1178` for Java 25, `orgapachehudi-1176` for Java 11). + Make sure all repos are still in the "open" state (not closed). 2. First do a dry-run to verify the list of artifacts to be copied: ```shell ./scripts/release/copy_staging_repo.sh --dry-run + ./scripts/release/copy_staging_repo.sh --dry-run ``` - 3. Then run the actual copy: + 3. Then run the actual copies: ```shell ./scripts/release/copy_staging_repo.sh 2>&1 | tee -a "/tmp/${RELEASE_VERSION}-${RC_NUM}.copy_staging.log" + ./scripts/release/copy_staging_repo.sh 2>&1 | tee -a "/tmp/${RELEASE_VERSION}-${RC_NUM}.copy_staging.log" ``` 4. The script reads Nexus credentials from `~/.m2/settings.xml` (server id `apache.releases.https`), downloads - every artifact from the source repo, and re-uploads them to the target repo. After it finishes, drop the - Java 17 staging repo on Apache Nexus. + every artifact from the source repo, and re-uploads them to the target repo. After it finishes, drop both the + Java 17 and the Java 25 staging repos on Apache Nexus. 6. Review all staged artifacts by logging into Apache Nexus and clicking on "Staging Repositories" link on left pane. Then find a "open" entry for apachehudi 7. Ensure it contains all 2 (2.12 and 2.13) artifacts, mainly hudi-spark-bundle-2.12/2.13, diff --git a/scripts/release/deploy_staging_jars_java25.sh b/scripts/release/deploy_staging_jars_java25.sh new file mode 100755 index 0000000000000..bd58f63b08a59 --- /dev/null +++ b/scripts/release/deploy_staging_jars_java25.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +MVN=${MVN:-mvn} +# fail immediately +set -o errexit +set -o nounset + +CURR_DIR=$(pwd) +if [ ! -d "$CURR_DIR/packaging" ] ; then + echo "You have to call the script from the repository root dir that contains 'packaging/'" + exit 1 +fi + +# Validate Java version (hudi-trino requires Java 25) +EXPECTED_JAVA_VERSION=25 +JAVA_VERSION_OUTPUT=$("${JAVA_HOME:+$JAVA_HOME/bin/}java" -version 2>&1) +JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_OUTPUT" | awk -F[\".] '/version/ {print ($2 == "1" ? $3 : $2); exit}') +if [ "$JAVA_MAJOR_VERSION" != "$EXPECTED_JAVA_VERSION" ]; then + echo "Error: Java $EXPECTED_JAVA_VERSION is required for this script, but found:" + echo "$JAVA_VERSION_OUTPUT" + echo "Set JAVA_HOME to a Java $EXPECTED_JAVA_VERSION installation and retry." + exit 1 +fi + +if [ "$#" -gt "1" ]; then + echo "Only accept 0 or 1 argument. Use -h to see examples." + exit 1 +fi + +declare -a ALL_VERSION_OPTS=( +# org.apache.hudi:hudi-trino (RFC-105), a plain library jar compiled with JDK 25. +# No -am: Lombok cannot run on JDK 25, so upstream Hudi modules (hudi-common, +# hudi-hive-sync, hudi-io:shaded, hudi-sync-common) must already be in the local m2 +# from a prior deploy_staging_jars.sh (JDK 11) run. +"-Phudi-trino -pl hudi-trino" +) +printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}" + +if [ "${1:-}" == "-h" ]; then + echo " +Usage: $(basename "$0") [OPTIONS] + +Options: + One of the version options below +${joined} +-h, --help +" + exit 0 +fi + +VERSION_OPT=${1:-} +valid_version_opt=false +for v in "${ALL_VERSION_OPTS[@]}"; do + [[ $VERSION_OPT == "$v" ]] && valid_version_opt=true +done + +if [ "$valid_version_opt" = true ]; then + # run deploy for only specified version option + ALL_VERSION_OPTS=("$VERSION_OPT") +elif [ "$#" == "1" ]; then + echo "Version option $VERSION_OPT is invalid. Use -h to see examples." + exit 1 +fi + +# -Dmaven.test.skip=true, not -DskipTests: test-compile needs hudi-trino-tests profile deps that are absent here. +COMMON_OPTIONS="-DdeployArtifacts=true -Dmaven.test.skip=true -DretryFailedDeploymentCount=10" +for v in "${ALL_VERSION_OPTS[@]}" +do + echo "Deploying to repository.apache.org with options ${v}" + # Single pass: unlike deploy_staging_jars.sh there is no -am here, so a separate + # install pass would rebuild exactly what deploy builds. + $MVN clean deploy $COMMON_OPTIONS ${v} +done diff --git a/scripts/release/validate_source_copyright.sh b/scripts/release/validate_source_copyright.sh index 67f32e2d9a7b8..5068e588f2bb5 100755 --- a/scripts/release/validate_source_copyright.sh +++ b/scripts/release/validate_source_copyright.sh @@ -47,8 +47,9 @@ echo -e "\t\tNotice file exists ? [OK]\n" ### Licensing Check echo "Performing custom Licensing Check " # --- -# Exclude the 'hudi-trino' directory. Its license checks are handled by airlift: -# https://github.com/airlift/airbase/blob/823101482dbc60600d7862f0f5c93aded6190996/airbase/pom.xml#L1239 +# Exclude the 'hudi-trino' directory: its files carry the short AL header (Trino convention), +# which the ASF wording this grep matches does not cover. RAT still checks the module via its +# default AL matchers. Drop this prune once apache/hudi#19412 converts the module to ASF headers. # --- numfilesWithNoLicense=$(find . -path './hudi-trino' -prune -o -type f -iname '*' | grep -v './hudi-trino' | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.zip' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'banner.txt' | grep -v '.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)") # Check if the variable holding the list of files is non-empty diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh index 1e1ee7c4565c8..2257d2fd6cbb6 100755 --- a/scripts/release/validate_staged_bundles.sh +++ b/scripts/release/validate_staged_bundles.sh @@ -83,12 +83,14 @@ declare -a extensions=("-javadoc.jar" "-javadoc.jar.asc" "-javadoc.jar.md5" "-ja "-sources.jar.asc" "-sources.jar.md5" "-sources.jar.sha1" ".jar" ".jar.asc" ".jar.md5" ".jar.sha1" ".pom" ".pom.asc" ".pom.md5" ".pom.sha1") +# hudi-trino is a plain library jar, not a bundle, but it is staged and validated like the bundles. declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle" "hudi-cli-bundle_2.12" "hudi-cli-bundle_2.13" "hudi-datahub-sync-bundle" "hudi-flink1.17-bundle" "hudi-flink1.18-bundle" "hudi-flink1.19-bundle" "hudi-flink1.20-bundle" "hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle" "hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle" "hudi-presto-bundle" "hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12" "hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" +"hudi-trino" "hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13" "hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13") From 0ee5be70747bf0d59528ed37c8accca7e452d691 Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 30 Jul 2026 22:46:42 +0800 Subject: [PATCH 220/255] test(trino): add MoR read tests for delete markers, custom payloads and commit-time ordering (#19295) * test(trino): add MoR read tests for delete markers, custom payloads and commit-time ordering Fixes apache/hudi#18898 (follow-up from the RFC-105 review of HudiTrinoReaderContext.getRecordMerger): the merge-mode dispatch had no end-to-end coverage where the merger choice matters. Adds runtime-written v10 MOR tables and snapshot-read tests for: - Deletes under EVENT_TIME_ORDERING (mor_deletes): hard deletes -- which at v10 are native delete log files read back through the connector's own getFileRecordIterator with the synthetic delete-log schema, a previously untested path enabled by the required-column resolution this branch stacks on -- plus _hoodie_is_deleted soft deletes, an OBSOLETE soft delete and an OBSOLETE update (lower ordering value) that must LOSE against the base row. - COMMIT_TIME_ORDERING (mor_commit_time): a lower-ordering update that must WIN (latest write), the exact mirror of the event-time case, discriminating OverwriteWithLatestMerger from event-time merging. - Payload-driven semantics without any hudi.record-merger-impls property: AWSDmsAvroPayload (v10-translated to COMMIT_TIME + delete-key props; Op='D' log records delete rows at merge time), OverwriteNonDefaultsWithLatestAvroPayload (v10-translated to PARTIAL_UPDATE_MODE=IGNORE_DEFAULTS; null update columns keep stored values), and a user-defined SummingTestPayload riding the payload-based CUSTOM strategy whose merged value (old + new) proves the payload's combineAndGetUpdateValue executed at read time. Test rows are written with the pass-through HoodieAvroPayload (not a BaseAvroPayload) so delete-flagged rows land as data records and every merge decision happens at read time from the table config. Bugs found by these tests and fixed here: - HudiUtil.mergeRequiredColumnNames read the custom delete key/marker from raw table props, but v9+ table creation persists them PREFIXED (hoodie.record.merge.property.*). The file-group reader strips the prefix via getTableMergeProperties before DeleteContext reads the plain keys, so its required schema included the delete column while the connector's base-read prediction missed it and the base-read projection guard failed every narrow projection on such tables. Now resolved the same way the reader does: table merge properties first, raw props as fallback. Unit-tested in TestHudiMergeRequiredColumns. - buildColumnHandles matched predicted merge columns against metastore data columns case-sensitively; the prediction carries the table schema's casing (AWSDms hardcodes 'Op') while metastore names are lowercased, so the handle was never built. Matching is now case-insensitive. - buildRequiredColumnHandles emitted projection handles with the lowercased metastore name on the log projection, but hudi-common reads merge fields off log records by the table schema's exact field name (Avro lookups are case-sensitive), so e.g. the DMS delete marker was never seen on log records. Handles are now re-labeled with the required-schema field's exact name; parquet columns resolve case-insensitively either way. - count(*) over a MoR split with log files failed with "Failed to construct Avro schema": the query projects no columns, and HudiPageSource's serializer fed the empty projection to AvroHiveFileUtils.determineSchemaOrThrowException, which rejects an empty column list. HudiUtil.constructSchema now short-circuits an empty projection to an empty record schema; the page source already counts positions fine with zero channels. HudiUtilTest's empty-columns test now pins the empty-record schema instead of the former exception. - Payload-based merging read garbage on the summing table ([, -44] instead of [k1, 109]): records handed to hudi-common carried a schema reconstructed from Hive metastore types (every column a nullable union, fields in projection order), while the file-group reader tracks its required schema for those records. BaseAvroPayload round-trips the newer record through Avro binary with the tracked schema, so the structural mismatch misaligned the binary decode. HudiTrinoReaderContext now builds the log-side AND base-side serializers with requiredSchema itself, via a new HudiAvroSerializer constructor that takes the record schema explicitly and maps page channels to record fields BY NAME (the base projection's order can differ from the required schema's field order; containment holds in both directions through the existing base-read guard and generateRequiredSchema always containing the requested schema). This also aligns record layouts with the positions hudi-common derives from the tracked schema (partial-update merging, _hoodie_operation lookup). - (hudi-common, engine-agnostic) The file-group reader passed its raw input props to initRecordMerger, the MERGE_TYPE lookup and the schema handler instead of the ConfigUtils.getMergeProps view, which layers the table's prefixed merge properties (hoodie.record.merge.property.*) in as plain keys -- a regression from commit 19961e77d539 replacing the previous in-place putAll with a copy. DeleteContext therefore missed the translated DMS delete key/marker, the delete column was not a mandatory merge field, and Op='D' log records merged as regular data updates instead of deletes on snapshot reads. Fixed in HoodieFileGroupReader and HoodieLsmFileGroupReader alike by passing this.props downstream. Also re-ports a shim-lineage guard (trino repo commit 78209f83b50) that the RFC-105 migration squash predates: HudiAvroSerializer's default constructor now maps channels past hidden (synthesized, split-prefilled) columns instead of relying on serialize() never being called with them; with identity positions a hidden column would shift every later value into the wrong field and overrun the record. Also replaces the TODO(apache/hudi#18898) in getRecordMerger with a pointer to the new suites. * test(trino): address review comments on MoR merge-semantics read tests - pass the payload class to getTableMergeProperties (the no-arg overload does not exist; the module profile being off by default hid the break) - drop the buildColumnHandles case-folding and the log-projection handle re-label: appendMissingMergeRequiredColumns and the requiredSchema-stamping serializer already cover both - stop building a record schema in the page-building serializer and revert the constructSchema empty-list special case; the count(*) path stays pinned by TestHudiMorMergeModeSemantics - scope the getRecordMerger comment to the CUSTOM arm, the only return value the file-group reader dereferences for merging - split the two monolithic fixtures into per-table subclasses of AbstractMergerHudiTablesInitializer composed with CompositeHudiTablesInitializer * test(trino): address round-2 review nits on MoR merge-semantics tests - qualify every fixture's TABLE_NAME/RT_TABLE_NAME in both suites so each assertion names the table it checks (no mixed static imports) - replace the duplicated PARTITION_PATH constants in the delete-writing fixtures with a hoodieKey(String) helper on AbstractMergerHudiTablesInitializer - correct the getRecordMerger comment: the ordering arms ARE reachable via partialMerge on IS_PARTIAL log blocks, just not covered by these suites; restore the TODO, now pointing at follow-up issue apache/hudi#19413 * test(trino): address round-3 review comments on MoR merge-semantics tests - rename testPrefixedDeleteKeyAndMarkerAreRequested to testPrefixedDeleteKeyIsRequested: only the delete key is requested, the marker is a precondition - reuse AWSDmsAvroPayload.OP_FIELD / DELETE_OPERATION_VALUE in the DMS fixture instead of local Op / D literals - add a non-marker Op='U' log record for k1 in the DMS fixture so a broken marker comparison fails the suite (k1 must update, not delete) - flip the five new table names to trail with _mor (deletes_mor, commit_time_mor, dms_mor, overwrite_non_defaults_mor, summing_mor) to match the existing fixtures in the package - cover the CUSTOM merge arm's delete path: the summing fixture gains a k2 base row and a hard-delete commit, pinned by testSummingPayloadHardDeleteRemovesRowOnSnapshotRead * test(trino): address round-4 review comment on MoR merge-semantics tests The hard-delete comments claimed the delete reaches the payload arm as an empty payload whose combineAndGetUpdateValue returns empty. In fact HoodieAvroRecordMerger.merge returns the delete on its isCommitTimeOrderingDelete short-circuit (writeClient.delete records carry the sentinel ordering value) before loadPayload runs, so no payload is constructed. Say so in the initializer javadoc, the commit comment and the test method comment. (cherry picked from commit 633d1427f35fd2d4a21c6e02363dc8dd90239237) --- .../java/io/trino/plugin/hudi/HudiUtil.java | 15 +- .../hudi/reader/HudiTrinoReaderContext.java | 33 +++- .../plugin/hudi/util/HudiAvroSerializer.java | 41 ++++- .../hudi/TestHudiMergeRequiredColumns.java | 15 ++ .../hudi/TestHudiMorMergeModeSemantics.java | 135 +++++++++++++++ .../hudi/TestHudiMorPayloadSemantics.java | 129 +++++++++++++++ .../AbstractMergerHudiTablesInitializer.java | 8 +- ...mmitTimeOrderingHudiTablesInitializer.java | 129 +++++++++++++++ .../DmsPayloadHudiTablesInitializer.java | 135 +++++++++++++++ ...EventTimeDeletesHudiTablesInitializer.java | 156 ++++++++++++++++++ ...nDefaultsPayloadHudiTablesInitializer.java | 123 ++++++++++++++ .../SummingPayloadHudiTablesInitializer.java | 131 +++++++++++++++ .../hudi/testing/SummingTestPayload.java | 72 ++++++++ 13 files changed, 1103 insertions(+), 19 deletions(-) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorPayloadSemantics.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/DmsPayloadHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/EventTimeDeletesHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OverwriteNonDefaultsPayloadHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingPayloadHudiTablesInitializer.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingTestPayload.java diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java index 4d339e7d56c42..4bcd3a65ec203 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiUtil.java @@ -651,8 +651,14 @@ static LinkedHashSet mergeRequiredColumnNames(HoodieTableConfig tableCon // Delete markers and the operation field decide record deletion at merge time requiredColumnNames.add(HOODIE_IS_DELETED_FIELD); requiredColumnNames.add(OPERATION_METADATA_FIELD); - String deleteKey = tableConfig.getProps().getProperty(DELETE_KEY); - String deleteMarker = tableConfig.getProps().getProperty(DELETE_MARKER); + // Resolve the delete key/marker the way the file-group reader does (ConfigUtils.getMergeProps): + // table merge properties first -- v9+ tables persist these PREFIXED (hoodie.record.merge.property.*) + // and getTableMergeProperties strips the prefix and bridges legacy delete payloads (keyed on the + // payload class, which for the read path resolves from the table config) -- falling back to the raw + // table props, where pre-prefix tables and reader/write configs carry the plain keys. + Map tableMergeProps = tableConfig.getTableMergeProperties(tableConfig.getPayloadClass()); + String deleteKey = tableMergeProps.getOrDefault(DELETE_KEY, tableConfig.getProps().getProperty(DELETE_KEY)); + String deleteMarker = tableMergeProps.getOrDefault(DELETE_MARKER, tableConfig.getProps().getProperty(DELETE_MARKER)); // DeleteContext only honors a custom delete key when the marker value is also set if (!StringUtils.isNullOrEmpty(deleteKey) && !StringUtils.isNullOrEmpty(deleteMarker)) { requiredColumnNames.add(deleteKey); @@ -715,7 +721,10 @@ public static List appendMissingMergeRequiredColumns( /** * Builds {@link HiveColumnHandle}s, preserving physical (data-column) index, for the data columns whose names * appear in {@code columnNames}. Names that are not data columns (e.g. Hudi meta fields) or whose types are not - * supported by the storage format are skipped. + * supported by the storage format are skipped. Matching is case-sensitive: a predicted merge column that carries + * the table schema's field casing (e.g. the {@code Op} delete key of DMS tables) misses the lowercased metastore + * name here and is recovered from the table schema by {@link #appendMissingMergeRequiredColumns} on the merge + * read path. */ private static List buildColumnHandles(Table table, TypeManager typeManager, Set columnNames, HiveTimestampPrecision timestampPrecision) { diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java index 57e8ad5d288c2..cc812a9a466fe 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java @@ -61,7 +61,7 @@ public class HudiTrinoReaderContext extends HoodieReaderContext { private final ConnectorPageSource pageSource; - private final HudiAvroSerializer avroSerializer; + private final List columnHandles; private final PrefilledColumnValues prefilledColumnValues; private final LogFileParquetPageSourceFactory logPageSourceFactory; private final Map colNameToHandle; @@ -88,7 +88,7 @@ public HudiTrinoReaderContext( super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), new AvroRecordContext(tableConfig, tableConfig.getPayloadClass())); this.pageSource = pageSource; this.prefilledColumnValues = prefilledColumnValues; - this.avroSerializer = new HudiAvroSerializer(columnHandles, prefilledColumnValues); + this.columnHandles = columnHandles; this.logPageSourceFactory = logPageSourceFactory; this.colNameToHandle = new HashMap<>(); for (HiveColumnHandle handle : columnHandles) { @@ -129,6 +129,11 @@ public ClosableIterator getFileRecordIterator( * fresh page source is built on demand with predicate pushdown disabled so every log record is read and * merged; for the base file the pre-built base page source is reused. Classic Avro log blocks never reach * here (they deserialize inline). + *

    + * Both paths emit records that CARRY {@code requiredSchema}: the file-group reader tracks that schema for + * every buffered record, and payload-based merging round-trips records through Avro binary with it + * ({@code BaseAvroPayload}), so a record whose own schema differs (in field order or nullability) would + * decode into garbage values there. */ private ClosableIterator getFileRecordIterator( StoragePath path, @@ -143,7 +148,7 @@ private ClosableIterator getFileRecordIterator( } List logProjection = buildRequiredColumnHandles(requiredSchema); ConnectorPageSource logSource = logPageSourceFactory.create(path.toString(), start, length, logProjection); - HudiAvroSerializer logSerializer = new HudiAvroSerializer(logProjection, prefilledColumnValues); + HudiAvroSerializer logSerializer = new HudiAvroSerializer(logProjection, prefilledColumnValues, requiredSchema.toAvroSchema()); return createRecordIterator(logSource, logSerializer); } // The base read reuses the pre-built page source, so it can only satisfy requiredSchema fields the @@ -161,7 +166,10 @@ private ClosableIterator getFileRecordIterator( + "FileGroupReaderSchemaHandler.generateRequiredSchema.", missingColumns)); } - return createRecordIterator(pageSource, avroSerializer); + // Every requiredSchema field is in the base projection (checked above) and every projected column is + // a requiredSchema field (the file-group reader's required schema always contains the full requested + // schema), so the by-name channel mapping resolves in both directions. + return createRecordIterator(pageSource, new HudiAvroSerializer(columnHandles, prefilledColumnValues, requiredSchema.toAvroSchema())); } /** @@ -176,6 +184,11 @@ private List buildRequiredColumnHandles(HoodieSchema requiredS { List handles = new ArrayList<>(); for (HoodieSchemaField field : requiredSchema.getFields()) { + // A projection handle may carry the lowercased metastore column name (e.g. 'op' for the + // DMS delete key 'Op'); that is fine on this path: parquet columns resolve + // case-insensitively, the serializer maps channels into requiredSchema positions + // case-insensitively, and hudi-common reads merge fields off the record's own schema, + // which serialize() stamps with the table casing. handles.add(colNameToHandle.computeIfAbsent( field.name().toLowerCase(Locale.ROOT), _ -> HudiUtil.toColumnHandle(field))); @@ -241,11 +254,13 @@ public IndexedRecord next() protected Option getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) { // Dispatch on the table's merge mode, mirroring HoodieAvroReaderContext. The Trino reader - // operates on IndexedRecord, so the Avro mergers apply directly. Using the read-time merger - // (combineAndGetUpdateValue) rather than a fixed preCombine merger keeps COMMIT_TIME_ORDERING - // and custom-payload tables correct on MoR reads. - // TODO(apache/hudi#18898): add MoR read tests for delete markers and custom payloads to - // exercise the EVENT_TIME_ORDERING (combineAndGetUpdateValue) and CUSTOM branches below. + // operates on IndexedRecord, so the Avro mergers apply directly. The CUSTOM arm's return + // value drives merging (combineAndGetUpdateValue at read time, covered end-to-end by + // TestHudiMorPayloadSemantics; apache/hudi#18898). The ordering arms' mergers are reached + // through partialMerge when a log block carries IS_PARTIAL (BufferedRecordMergerFactory), + // which these suites do not cover; TestHudiMorMergeModeSemantics pins the mode semantics + // themselves. TODO(apache/hudi#19413): cover the ordering arms' partialMerge path + // (IS_PARTIAL log blocks) once the Avro mergers implement it. switch (mergeMode) { case EVENT_TIME_ORDERING: return Option.of(new HoodieAvroRecordMerger()); diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index 4791507489286..0d9bc9f5978b3 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -60,9 +60,9 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.hudi.HudiUtil.constructSchema; import static io.trino.plugin.hudi.HudiUtil.getFieldFromSchema; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static io.trino.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE; @@ -111,24 +111,53 @@ public class HudiAvroSerializer private final List columnHandles; private final List columnTypes; + // Both are null for a page-building-only serializer (the two-arg constructor): buildRecordInPage + // reads field positions off each record's own schema, so no record schema is needed there -- and + // none could be built for hidden (synthesized) columns, which are answered from the split by + // PrefilledColumnValues rather than read from the file. serialize() requires the three-arg + // constructor, which maps page channel i to record position channelToFieldPosition[i]. private final Schema schema; + private final int[] channelToFieldPosition; public HudiAvroSerializer(List columnHandles, PrefilledColumnValues prefilledColumnValues) { this.columnHandles = columnHandles; this.columnTypes = columnHandles.stream().map(HiveColumnHandle::getType).toList(); - // Fetches projected schema - this.schema = constructSchema(columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getName).toList(), - columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getHiveType).toList()); this.prefilledColumnValues = prefilledColumnValues; + this.schema = null; + this.channelToFieldPosition = null; + } + + /** + * Builds a serializer whose {@link #serialize} records carry {@code recordSchema} -- the exact + * schema hudi-common tracks for the records of this read (the file-group reader's required + * schema) -- instead of a schema reconstructed from the projection's Hive types. The + * reconstruction differs from the table's real schema (every Hive column becomes a nullable + * union, fields follow projection order), and payload-based merging round-trips the record + * through Avro BINARY with the tracked schema ({@code BaseAvroPayload}), where any structural + * difference misaligns the decode and yields garbage values. Page channels are matched to + * record fields BY NAME, so the projection may order columns differently from the schema; + * every projected column must be a field of {@code recordSchema}. + */ + public HudiAvroSerializer(List columnHandles, PrefilledColumnValues prefilledColumnValues, Schema recordSchema) + { + this.columnHandles = columnHandles; + this.columnTypes = columnHandles.stream().map(HiveColumnHandle::getType).toList(); + this.schema = recordSchema; + this.prefilledColumnValues = prefilledColumnValues; + int[] mapping = new int[columnHandles.size()]; + for (int i = 0; i < columnHandles.size(); i++) { + mapping[i] = getFieldFromSchema(columnHandles.get(i).getName(), recordSchema).pos(); + } + this.channelToFieldPosition = mapping; } public IndexedRecord serialize(Page sourcePage, int position) { + checkState(schema != null, "serialize() requires a serializer built with a record schema"); IndexedRecord record = new GenericData.Record(schema); for (int i = 0; i < columnTypes.size(); i++) { - Object value = getValue(sourcePage, i, position); - record.put(i, value); + record.put(channelToFieldPosition[i], getValue(sourcePage, i, position)); } return record; } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java index f6a4ffb874d06..2abc2c680b08e 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMergeRequiredColumns.java @@ -89,6 +89,21 @@ public void testCustomDeleteKeyRequiresMarkerToo() .doesNotContain("op"); } + @Test + public void testPrefixedDeleteKeyIsRequested() + { + // v9+ table creation persists the delete key/marker under the hoodie.record.merge.property. + // prefix (e.g. for AWSDmsAvroPayload tables); the file-group reader strips the prefix via + // getTableMergeProperties() before DeleteContext reads the plain keys, and the connector's + // prediction must see the same values or the base-read projection guard fires on narrow queries + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY, "Op"); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER, "D"); + + assertThat(mergeRequiredColumnNames(tableConfig, RecordMergeMode.COMMIT_TIME_ORDERING)) + .contains("Op"); + } + @Test public void testRecordKeyFieldsRequestedWithoutPopulatedMetaFields() { diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java new file mode 100644 index 0000000000000..ba21f2b34a679 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.plugin.hudi.testing.CommitTimeOrderingHudiTablesInitializer; +import io.trino.plugin.hudi.testing.CompositeHudiTablesInitializer; +import io.trino.plugin.hudi.testing.EventTimeDeletesHudiTablesInitializer; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * End-to-end MoR snapshot-read tests for the merge-mode dispatch in + * {@code HudiTrinoReaderContext.getRecordMerger} with deletes (issue apache/hudi#18898), on tables + * written by {@link EventTimeDeletesHudiTablesInitializer} and + * {@link CommitTimeOrderingHudiTablesInitializer}: + *

      + *
    • EVENT_TIME_ORDERING: updates and soft deletes apply only when their ordering value wins; + * obsolete (lower-ordering) updates and soft deletes must LOSE against the base row.
      • + *
    • Hard deletes (native delete log files, read back through the connector's own + * {@code getFileRecordIterator}) always win.
      • + *
    • COMMIT_TIME_ORDERING: the latest write wins even with a LOWER ordering value -- the exact + * mirror of the event-time obsolete-update case, discriminating the two merger dispatches.
      • + *
    + */ +public class TestHudiMorMergeModeSemantics + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return HudiQueryRunner.builder() + .setDataLoader(new CompositeHudiTablesInitializer( + new EventTimeDeletesHudiTablesInitializer(), + new CommitTimeOrderingHudiTablesInitializer())) + .build(); + } + + @Test + public void testReadOptimizedShowsAllBaseRows() + { + // Deletes and updates live in log files only; the read-optimized tables reflect the base commit + assertQuery( + "SELECT key, name, value FROM " + EventTimeDeletesHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT)), ('k2', 'k2_base', 20), ('k3', 'k3_base', 30)," + + " ('k4', 'k4_base', 40), ('k5', 'k5_base', 50), ('k6', 'k6_base', 60)"); + assertQuery( + "SELECT key, name, value FROM " + CommitTimeOrderingHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT)), ('k2', 'k2_base', 20), ('k3', 'k3_base', 30)"); + } + + @Test + public void testEventTimeMergeWithDeletes() + { + // k1: higher-ts update wins; k2: hard-deleted; k3: soft-deleted (higher ts); + // k4: OBSOLETE soft delete (lower ts) -> base row survives; + // k5: untouched; k6: OBSOLETE update (lower ts) -> base row survives + assertQuery( + "SELECT key, name, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(11 AS BIGINT)), ('k4', 'k4_base', 40)," + + " ('k5', 'k5_base', 50), ('k6', 'k6_base', 60)"); + } + + @Test + public void testHardDeleteRemovesRowOnSnapshotRead() + { + // The hard delete is a native delete log file, resolved through the connector's + // getFileRecordIterator with the synthetic delete-log schema (record key + ordering field) + assertThat(computeScalar("SELECT count(*) FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " WHERE key = 'k2'")) + .isEqualTo(0L); + assertThat(computeScalar("SELECT count(*) FROM " + EventTimeDeletesHudiTablesInitializer.TABLE_NAME + " WHERE key = 'k2'")) + .isEqualTo(1L); + } + + @Test + public void testSoftDeleteRemovesRowOnSnapshotRead() + { + // _hoodie_is_deleted=true log record with a winning (higher) ordering value + assertThat(computeScalar("SELECT count(*) FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " WHERE key = 'k3'")) + .isEqualTo(0L); + assertThat(computeScalar("SELECT count(*) FROM " + EventTimeDeletesHudiTablesInitializer.TABLE_NAME + " WHERE key = 'k3'")) + .isEqualTo(1L); + } + + @Test + public void testObsoleteSoftDeleteLosesUnderEventTimeOrdering() + { + // The k4 soft delete carries ts=50 < base ts=100: event-time merging must keep the base row + assertQuery( + "SELECT key, name, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " WHERE key = 'k4'", + "VALUES ('k4', 'k4_base', CAST(40 AS BIGINT))"); + } + + @Test + public void testCommitTimeOrderingKeepsLatestWrite() + { + // k1's update carries ts=50 < base ts=100. Under COMMIT_TIME_ORDERING the LATEST WRITE wins + // regardless of the ordering value -- the mirror of the event-time k6 case, where the same + // shape keeps the BASE row. Together they discriminate the two merger dispatches. + assertQuery( + "SELECT key, name, value FROM " + CommitTimeOrderingHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(11 AS BIGINT)), ('k3', 'k3_base', 30)"); + } + + @Test + public void testCountAfterDeletes() + { + assertThat(computeScalar("SELECT count(*) FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME)).isEqualTo(4L); + assertThat(computeScalar("SELECT count(*) FROM " + CommitTimeOrderingHudiTablesInitializer.RT_TABLE_NAME)).isEqualTo(2L); + } + + @Test + public void testNarrowProjectionMergesCorrectly() + { + // Neither the ordering field nor _hoodie_is_deleted is projected; the connector must still read + // them on both the base and log sides for the merge to resolve updates and deletes correctly + assertQuery( + "SELECT key, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', CAST(11 AS BIGINT)), ('k4', 40), ('k5', 50), ('k6', 60)"); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorPayloadSemantics.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorPayloadSemantics.java new file mode 100644 index 0000000000000..2183d80fe6867 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorPayloadSemantics.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.plugin.hudi.testing.CompositeHudiTablesInitializer; +import io.trino.plugin.hudi.testing.DmsPayloadHudiTablesInitializer; +import io.trino.plugin.hudi.testing.OverwriteNonDefaultsPayloadHudiTablesInitializer; +import io.trino.plugin.hudi.testing.SummingPayloadHudiTablesInitializer; +import io.trino.plugin.hudi.testing.SummingTestPayload; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * End-to-end MoR snapshot-read tests for PAYLOAD-driven merge semantics (issue apache/hudi#18898), on + * tables written by {@link DmsPayloadHudiTablesInitializer}, + * {@link OverwriteNonDefaultsPayloadHudiTablesInitializer} and {@link SummingPayloadHudiTablesInitializer}. + * No {@code hudi.record-merger-impls} connector property is set anywhere -- every behavior below must + * resolve purely from the table config: + *
      + *
    • AWSDms: a log record with {@code Op='D'} deletes the row at merge time via the translated + * delete-key/marker table properties, while a log record with the non-marker {@code Op='U'} must + * apply as an update (the narrow-projection case pins the fix that reads those properties with + * their {@code hoodie.record.merge.property.} prefix).
      • + *
    • OverwriteNonDefaults: IGNORE_DEFAULTS partial merging keeps the stored value for update columns + * equal to the schema default (null).
      • + *
    • {@link SummingTestPayload}: a user-defined payload rides the payload-based CUSTOM merge + * strategy; the merged value is the SUM of stored and incoming values, which proves the payload's + * {@code combineAndGetUpdateValue} executed (overwrite would yield the incoming value), and a hard + * delete routed through the same arm must remove its row.
      • + *
    + */ +public class TestHudiMorPayloadSemantics + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return HudiQueryRunner.builder() + .setDataLoader(new CompositeHudiTablesInitializer( + new DmsPayloadHudiTablesInitializer(), + new OverwriteNonDefaultsPayloadHudiTablesInitializer(), + new SummingPayloadHudiTablesInitializer())) + .build(); + } + + @Test + public void testDmsDeleteMarkerRemovesRowOnSnapshotRead() + { + // Read-optimized: both rows (the log records are not merged) + assertQuery( + "SELECT key, name, value, Op FROM " + DmsPayloadHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_base', CAST(10 AS BIGINT), 'I'), ('k2', 'k2_base', 20, 'I')"); + // Snapshot: k2 is deleted by the Op='D' log record via the delete-key/marker table properties, + // while k1's NON-marker Op='U' log record must apply as an update -- a marker comparison that + // fires on any non-null Op would wrongly delete k1 too + assertQuery( + "SELECT key, name, value, Op FROM " + DmsPayloadHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', 'k1_updated', CAST(11 AS BIGINT), 'U')"); + } + + @Test + public void testDmsNarrowProjectionMergesCorrectly() + { + // The Op column is NOT projected, so the connector must predict it as a merge-required column + // from the PREFIXED table properties (hoodie.record.merge.property.hoodie.payload.delete.field) + // for the base read -- the regression this suite pins for HudiUtil.mergeRequiredColumnNames + assertQuery( + "SELECT key, value FROM " + DmsPayloadHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', CAST(11 AS BIGINT))"); + assertThat(computeScalar("SELECT count(*) FROM " + DmsPayloadHudiTablesInitializer.RT_TABLE_NAME)).isEqualTo(1L); + } + + @Test + public void testOverwriteNonDefaultsKeepsStoredValueForDefaultColumns() + { + // Read-optimized: base values + assertQuery( + "SELECT key, a, b FROM " + OverwriteNonDefaultsPayloadHudiTablesInitializer.TABLE_NAME, + "VALUES ('k1', 'base_a', 'base_b')"); + // Snapshot: the update carried a='new_a' and b=null (the schema default); IGNORE_DEFAULTS + // partial merging takes the update's a but keeps the STORED b + assertQuery( + "SELECT key, a, b FROM " + OverwriteNonDefaultsPayloadHudiTablesInitializer.RT_TABLE_NAME, + "VALUES ('k1', 'new_a', 'base_b')"); + } + + @Test + public void testSummingPayloadRunsCombineAndGetUpdateValueOnRead() + { + // Read-optimized: the base values + assertQuery( + "SELECT key, value FROM " + SummingPayloadHudiTablesInitializer.TABLE_NAME + " ORDER BY key", + "VALUES ('k1', CAST(10 AS BIGINT)), ('k2', 20)"); + // Snapshot: 10 + 99 = 109 -- only the payload's combineAndGetUpdateValue can produce this + // (newest-wins would yield 99, base-only 10), proving the CUSTOM payload-strategy branch ran; + // k2 is hard-deleted + assertQuery( + "SELECT key, value FROM " + SummingPayloadHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", + "VALUES ('k1', CAST(109 AS BIGINT))"); + } + + @Test + public void testSummingPayloadHardDeleteRemovesRowOnSnapshotRead() + { + // The hard delete is a native delete log record routed to the payload-based CUSTOM merge arm, + // where it wins on HoodieAvroRecordMerger's isCommitTimeOrderingDelete short-circuit (the + // delete carries the sentinel ordering value) -- the delete path of the user-merger dispatch, + // which both ordering arms already cover + assertThat(computeScalar("SELECT count(*) FROM " + SummingPayloadHudiTablesInitializer.RT_TABLE_NAME + " WHERE key = 'k2'")) + .isEqualTo(0L); + assertThat(computeScalar("SELECT count(*) FROM " + SummingPayloadHudiTablesInitializer.TABLE_NAME + " WHERE key = 'k2'")) + .isEqualTo(1L); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java index 045e8a3386eaf..2aa39bf1cffa8 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java @@ -206,7 +206,13 @@ protected HoodieJavaWriteClient writeClient() protected static HoodieRecord avroRecord(GenericRecord record, String key) { - return new HoodieAvroRecord<>(new HoodieKey(key, PARTITION_PATH), new HoodieAvroPayload(Option.of(record)), null); + return new HoodieAvroRecord<>(hoodieKey(key), new HoodieAvroPayload(Option.of(record)), null); + } + + /** Addresses a record in the single unnamed partition, e.g. for hard deletes via {@code writeClient.delete}. */ + protected static HoodieKey hoodieKey(String key) + { + return new HoodieKey(key, PARTITION_PATH); } /** Mirrors the staged table into the Trino filesystem so the connector observes the commits written so far. */ diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java new file mode 100644 index 0000000000000..8fa5c7fb3d790 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table in {@link RecordMergeMode#COMMIT_TIME_ORDERING} that + * exercises the read-side merge-mode dispatch (issue apache/hudi#18898). ONLY a record merge mode is set + * (no payload class), so table creation persists the mode as-is, which is exactly the dispatch input + * {@code HudiTrinoReaderContext.getRecordMerger} switches on. + *

    + * A base commit is followed by a log commit whose update carries an ordering value LOWER than the base + * row's: latest-write-wins must KEEP the update, the exact mirror of the event-time obsolete-update case + * in {@link EventTimeDeletesHudiTablesInitializer}, which is what discriminates the two merger dispatches. + * A final commit hard-deletes a key ({@code writeClient.delete}); commit-time deletes always win. See + * {@code TestHudiMorMergeModeSemantics}. + */ +public class CommitTimeOrderingHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "commit_time_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public CommitTimeOrderingHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setRecordMergeMode(RecordMergeMode.COMMIT_TIME_ORDERING); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withRecordMergeMode(RecordMergeMode.COMMIT_TIME_ORDERING); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: base parquet file with 3 keys at ts 100. + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 100L), + record(schema, "k2", "k2_base", 20L, 100L), + record(schema, "k3", "k3_base", 30L, 100L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit (log file): update k1 with a LOWER ts (50). Commit-time ordering keeps the + // LATEST WRITE regardless of the ordering value -- the exact mirror of the event-time k6 + // case, which discriminates OverwriteWithLatestMerger from event-time merging. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 11L, 50L)), secondCommit); + client.commit(secondCommit, secondStatuses); + + // Third commit: hard delete of k2 (commit-time deletes always win). + String deleteCommit = client.startCommit(); + List deleteStatuses = client.delete( + ImmutableList.of(hoodieKey("k2")), deleteCommit); + client.commit(deleteCommit, deleteStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/DmsPayloadHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/DmsPayloadHudiTablesInitializer.java new file mode 100644 index 0000000000000..0d1de49d5a3a1 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/DmsPayloadHudiTablesInitializer.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.AWSDmsAvroPayload; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static org.apache.hudi.common.model.AWSDmsAvroPayload.DELETE_OPERATION_VALUE; +import static org.apache.hudi.common.model.AWSDmsAvroPayload.OP_FIELD; + +/** + * Creates a non-partitioned Merge-On-Read table whose merge semantics come from the + * {@link AWSDmsAvroPayload} class persisted in the table config (issue apache/hudi#18898). ONLY the payload + * class is set (no merge mode / strategy id), so table creation translates it exactly as a real writer + * would: at the current table version this "deprecated" payload becomes COMMIT_TIME_ORDERING plus PREFIXED + * delete-key props ({@code hoodie.record.merge.property.hoodie.payload.delete.field=Op}, marker {@code D}). + *

    + * A base commit is followed by a log record with {@code Op='D'}, which deletes the row at merge time via + * {@code DeleteContext}, with the payload never executing at read, plus a log record with the non-marker + * {@code Op='U'} whose update must APPLY rather than delete -- pinning the marker-value comparison itself. + *

    + * Records are wrapped in {@link HoodieAvroPayload} (a pass-through that is NOT a {@code BaseAvroPayload}), + * so rows a semantic payload would drop at write time land as DATA records and every merge decision happens + * at read time. See {@code TestHudiMorPayloadSemantics}. + */ +public class DmsPayloadHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "dms_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public DmsPayloadHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + // The Avro/parquet field is 'Op' (AWSDms hardcodes that casing), but a real Hive + // metastore lowercases column names on DDL -- exactly the case mismatch the connector's + // merge-column matching must bridge + new Column(OP_FIELD.toLowerCase(Locale.ROOT), HIVE_STRING, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(OP_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setPayloadClassName(AWSDmsAvroPayload.class.getName()); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withWritePayLoad(AWSDmsAvroPayload.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, "I", 100L), + record(schema, "k2", "k2_base", 20L, "I", 100L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Log records, both written as DATA records by the pass-through HoodieAvroPayload. Only k2's + // marker value deletes at merge time via DeleteContext (delete key Op, marker D from the + // translated table config); k1 carries the NON-marker Op='U' and its update must apply, so a + // marker comparison that fires on any non-null Op fails the suite. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 11L, "U", 200L), + record(schema, "k2", "k2_deleted", 22L, DELETE_OPERATION_VALUE, 200L)), secondCommit); + client.commit(secondCommit, secondStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, String op, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(OP_FIELD, op); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/EventTimeDeletesHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/EventTimeDeletesHudiTablesInitializer.java new file mode 100644 index 0000000000000..12cb59371b99b --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/EventTimeDeletesHudiTablesInitializer.java @@ -0,0 +1,156 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.JsonProperties; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_BOOLEAN; +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static org.apache.hudi.common.model.HoodieRecord.HOODIE_IS_DELETED_FIELD; + +/** + * Creates a non-partitioned Merge-On-Read table in {@link RecordMergeMode#EVENT_TIME_ORDERING} that + * exercises the read-side merge-mode dispatch with deletes (issue apache/hudi#18898). ONLY a record merge + * mode is set (no payload class), so table creation persists the mode as-is, which is exactly the dispatch + * input {@code HudiTrinoReaderContext.getRecordMerger} switches on. + *

    + * A base commit is followed by a log commit carrying an update, a soft delete + * ({@code _hoodie_is_deleted=true}), an OBSOLETE soft delete and an OBSOLETE update (both with an ordering + * value LOWER than the base row's, so event-time merging must keep the base row), and then a hard-delete + * commit ({@code writeClient.delete}) that produces a native delete log file read back through the + * connector's {@code getFileRecordIterator}. + *

    + * Records are wrapped in {@link HoodieAvroPayload}, which implements {@code HoodieRecordPayload} directly + * (NOT {@code BaseAvroPayload}), so rows with {@code _hoodie_is_deleted=true} are written as DATA records + * and delete semantics are evaluated at READ time. See {@code TestHudiMorMergeModeSemantics}. + */ +public class EventTimeDeletesHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "deletes_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public EventTimeDeletesHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("value", HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of()), + new Column(HOODIE_IS_DELETED_FIELD, HIVE_BOOLEAN, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("value", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field( + HOODIE_IS_DELETED_FIELD, + Schema.createUnion(Schema.create(Schema.Type.NULL), Schema.create(Schema.Type.BOOLEAN)), + null, + JsonProperties.NULL_VALUE)); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setRecordMergeMode(RecordMergeMode.EVENT_TIME_ORDERING); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withRecordMergeMode(RecordMergeMode.EVENT_TIME_ORDERING); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + // First commit: base parquet file with 6 keys, all at ordering value (ts) 100. + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "k1_base", 10L, 100L, false), + record(schema, "k2", "k2_base", 20L, 100L, false), + record(schema, "k3", "k3_base", 30L, 100L, false), + record(schema, "k4", "k4_base", 40L, 100L, false), + record(schema, "k5", "k5_base", 50L, 100L, false), + record(schema, "k6", "k6_base", 60L, 100L, false)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Second commit (log file). Event-time merging must resolve each key by ordering value: + // - k1: update with HIGHER ts (200) -> update wins + // - k3: soft delete with HIGHER ts (200) -> row deleted at read time + // - k4: soft delete with LOWER ts (50) -> OBSOLETE delete, base row survives + // - k6: update with LOWER ts (50) -> OBSOLETE update, base row survives + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "k1_updated", 11L, 200L, false), + record(schema, "k3", "k3_deleted", 33L, 200L, true), + record(schema, "k4", "k4_deleted", 44L, 50L, true), + record(schema, "k6", "k6_updated", 66L, 50L, false)), secondCommit); + client.commit(secondCommit, secondStatuses); + + // Third commit: HARD delete of k2. At the current table version this produces a native + // delete log file, which the file-group reader reads back through the connector's + // getFileRecordIterator with the synthetic delete-log schema (record key + ordering). + // Hard deletes carry the sentinel ordering value and win regardless of merge mode. + String deleteCommit = client.startCommit(); + List deleteStatuses = client.delete( + ImmutableList.of(hoodieKey("k2")), deleteCommit); + client.commit(deleteCommit, deleteStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String name, long value, long ts, boolean deleted) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("value", value); + record.put(ORDERING_FIELD, ts); + record.put(HOODIE_IS_DELETED_FIELD, deleted); + // HoodieAvroPayload passes the record through untouched (it is not a BaseAvroPayload), so a row + // with _hoodie_is_deleted=true is WRITTEN as a data record and only deleted at merge/read time. + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OverwriteNonDefaultsPayloadHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OverwriteNonDefaultsPayloadHudiTablesInitializer.java new file mode 100644 index 0000000000000..ba460111d5d30 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OverwriteNonDefaultsPayloadHudiTablesInitializer.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.JsonProperties; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table whose merge semantics come from the + * {@link OverwriteNonDefaultsWithLatestAvroPayload} class persisted in the table config (issue + * apache/hudi#18898). ONLY the payload class is set (no merge mode / strategy id), so table creation + * translates it exactly as a real writer would: into COMMIT_TIME_ORDERING plus + * {@code PARTIAL_UPDATE_MODE=IGNORE_DEFAULTS}. + *

    + * A base commit is followed by an update whose column is null (the schema default), which must keep the + * STORED value for that column at merge time. + *

    + * Records are wrapped in {@link HoodieAvroPayload} (a pass-through that is NOT a {@code BaseAvroPayload}), + * so every merge decision happens at read time from the table config. See + * {@code TestHudiMorPayloadSemantics}. + */ +public class OverwriteNonDefaultsPayloadHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "overwrite_non_defaults_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + public OverwriteNonDefaultsPayloadHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column("a", HIVE_STRING, Optional.empty(), Map.of()), + new Column("b", HIVE_STRING, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + Schema nullableString = Schema.createUnion(Schema.create(Schema.Type.NULL), Schema.create(Schema.Type.STRING)); + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("a", nullableString, null, JsonProperties.NULL_VALUE), + new Schema.Field("b", nullableString, null, JsonProperties.NULL_VALUE), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setPayloadClassName(OverwriteNonDefaultsWithLatestAvroPayload.class.getName()); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withWritePayLoad(OverwriteNonDefaultsWithLatestAvroPayload.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", "base_a", "base_b", 100L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // Update with b=null (the schema default): IGNORE_DEFAULTS partial merging must keep the + // stored 'base_b' while taking the updated 'new_a'. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", "new_a", null, 200L)), secondCommit); + client.commit(secondCommit, secondStatuses); + } + + private static HoodieRecord record(Schema schema, String key, String a, String b, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("a", a); + record.put("b", b); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingPayloadHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingPayloadHudiTablesInitializer.java new file mode 100644 index 0000000000000..07d437decb134 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingPayloadHudiTablesInitializer.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned Merge-On-Read table whose merge semantics come from the {@link SummingTestPayload} + * class persisted in the table config (issue apache/hudi#18898). ONLY the payload class is set (no merge mode + * / strategy id), so table creation translates it exactly as a real writer would: this user-defined payload is + * NOT in the deprecation set, so it is persisted as RECORD_MERGE_MODE=CUSTOM with the payload-based merge + * strategy id. Reads resolve {@code HoodieAvroRecordMerger} (no {@code hudi.record-merger-impls} needed) and + * run the payload's {@code combineAndGetUpdateValue}, observable as SUMMED values. + *

    + * A final commit hard-deletes a key ({@code writeClient.delete}): the native delete log record routes to + * {@code HoodieAvroRecordMerger} but wins on its {@code isCommitTimeOrderingDelete} short-circuit (the + * delete carries the sentinel ordering value), before any payload is constructed -- the delete coverage + * both ordering arms already have. + *

    + * Records are wrapped in {@link HoodieAvroPayload} (a pass-through that is NOT a {@code BaseAvroPayload}), so + * every merge decision happens at read time from the table config. See {@code TestHudiMorPayloadSemantics}. + */ +public class SummingPayloadHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "summing_mor"; + public static final String RT_TABLE_NAME = TABLE_NAME + "_rt"; + + private static final String SUM_FIELD = SummingTestPayload.SUM_COLUMN; + + public SummingPayloadHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected List dataColumns() + { + return ImmutableList.of( + new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), + new Column(SUM_FIELD, HIVE_LONG, Optional.empty(), Map.of()), + new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + } + + @Override + protected Schema avroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field(SUM_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setPayloadClassName(SummingTestPayload.class.getName()); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withWritePayLoad(SummingTestPayload.class.getName()); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + String firstCommit = client.startCommit(); + List firstStatuses = client.bulkInsert(ImmutableList.of( + record(schema, "k1", 10L, 100L), + record(schema, "k2", 20L, 100L)), firstCommit); + client.commit(firstCommit, firstStatuses); + + // The payload's combineAndGetUpdateValue SUMS stored and incoming values: 10 + 99 = 109 -- + // a result neither overwrite (99) nor base-only (10) can produce. + String secondCommit = client.startCommit(); + List secondStatuses = client.upsert(ImmutableList.of( + record(schema, "k1", 99L, 200L)), secondCommit); + client.commit(secondCommit, secondStatuses); + + // Third commit: hard delete of k2. The native delete log record reaches the payload-based + // CUSTOM merge arm, where it wins on HoodieAvroRecordMerger's isCommitTimeOrderingDelete + // short-circuit (writeClient.delete records carry the sentinel ordering value), before any + // payload is constructed -- the delete path of the user-merger dispatch. + String deleteCommit = client.startCommit(); + List deleteStatuses = client.delete( + ImmutableList.of(hoodieKey("k2")), deleteCommit); + client.commit(deleteCommit, deleteStatuses); + } + + private static HoodieRecord record(Schema schema, String key, long value, long ts) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put(SUM_FIELD, value); + record.put(ORDERING_FIELD, ts); + return avroRecord(record, key); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingTestPayload.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingTestPayload.java new file mode 100644 index 0000000000000..7a009fbeab22b --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/SummingTestPayload.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.util.Option; + +import java.io.IOException; + +/** + * Test-only user-defined {@link org.apache.hudi.common.model.HoodieRecordPayload} whose read-side merge + * SUMS the {@code value} column of the stored and incoming records. A merged row therefore carries a + * value no built-in merge policy can produce (overwrite yields the incoming value, base-only the stored + * value), which proves end-to-end that the connector's CUSTOM merge branch executed this payload's + * {@code combineAndGetUpdateValue} (issue apache/hudi#18898). + *

    + * Unlike the built-in payloads named in the issue, this class is NOT in hudi's payloads-under-deprecation + * set, so v9+ table creation persists it as {@code RECORD_MERGE_MODE=CUSTOM} with the payload-based merge + * strategy id -- the configuration that routes reads through {@code HoodieAvroRecordMerger} and this + * payload, with no {@code hudi.record-merger-impls} connector property involved. + */ +public class SummingTestPayload + extends OverwriteWithLatestAvroPayload +{ + /** Name of the column whose stored and incoming values are summed at merge time. */ + public static final String SUM_COLUMN = "value"; + + public SummingTestPayload(GenericRecord record, Comparable orderingVal) + { + super(record, orderingVal); + } + + public SummingTestPayload(Option record) + { + super(record); + } + + @Override + public Option combineAndGetUpdateValue(IndexedRecord currentValue, Schema schema) + throws IOException + { + Option incoming = getInsertValue(schema); + if (incoming.isEmpty()) { + return Option.empty(); + } + GenericRecord newer = (GenericRecord) incoming.get(); + GenericRecord older = (GenericRecord) currentValue; + + long sum = ((Number) older.get(SUM_COLUMN)).longValue() + ((Number) newer.get(SUM_COLUMN)).longValue(); + GenericRecord merged = new GenericData.Record(newer.getSchema()); + for (Schema.Field field : newer.getSchema().getFields()) { + merged.put(field.pos(), newer.get(field.pos())); + } + merged.put(SUM_COLUMN, sum); + return Option.of(merged); + } +} From 233bf0d12803ee63e0c167272a14c200ee41ac8f Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Fri, 31 Jul 2026 13:23:25 +0530 Subject: [PATCH 221/255] feat(client): enrich write commit callback message and fire it for table-service commits (#18988) Adapted for release-1.2.1, which keeps the callback helpers in org.apache.hudi.callback.util where master has them in org.apache.hudi.callback: - BaseHoodieClient: HoodieCommitCallbackFactory imported from callback.util. - HoodieWriteCommitCallbackMessage and its new test: HoodieWriteCommitCallbackUtil imported from callback.util; Lazy imported from org.apache.hudi.util rather than master's common.util. - TestHoodieWriteCommitCallbackUtil placed under callback/util/ to match the package of the class it tests. Git relocated the util itself automatically, but the new test arrived at master's path. - BaseHoodieClient: took only the BaseFileOnlyView import from the conflict. The block also offered common.util.HoodieStorageUtils, which this commit does not add and which is already imported here from org.apache.hudi.storage. - BaseHoodieWriteClient: dropped the now-unused HoodieCommitCallbackFactory import along with the two the commit removes, since the callback firing moves to BaseHoodieClient. (cherry picked from commit bf3cc4393981149167973e700927f22748ae1dcf) --- .../HoodieWriteCommitCallbackMessage.java | 118 ++++++++++- .../util/HoodieWriteCommitCallbackUtil.java | 54 +++++ .../apache/hudi/client/BaseHoodieClient.java | 45 ++++ .../client/BaseHoodieTableServiceClient.java | 6 + .../hudi/client/BaseHoodieWriteClient.java | 27 +-- .../TestHoodieWriteCommitCallbackMessage.java | 198 ++++++++++++++++++ .../TestHoodieWriteCommitCallbackUtil.java | 146 +++++++++++++ .../client/HoodieFlinkTableServiceClient.java | 4 + ...tHoodieJavaClientOnMergeOnReadStorage.java | 121 +++++++++++ 9 files changed, 700 insertions(+), 19 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/common/TestHoodieWriteCommitCallbackMessage.java create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/util/TestHoodieWriteCommitCallbackUtil.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/common/HoodieWriteCommitCallbackMessage.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/common/HoodieWriteCommitCallbackMessage.java index 713427b52c01f..23a1e08b86c26 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/common/HoodieWriteCommitCallbackMessage.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/common/HoodieWriteCommitCallbackMessage.java @@ -19,20 +19,26 @@ import org.apache.hudi.ApiMaturityLevel; import org.apache.hudi.PublicAPIClass; +import org.apache.hudi.callback.util.HoodieWriteCommitCallbackUtil; import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.util.Lazy; import org.apache.hudi.common.util.Option; -import lombok.AllArgsConstructor; +import lombok.AccessLevel; import lombok.Getter; +import java.io.IOException; +import java.io.ObjectOutputStream; import java.io.Serializable; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Base callback message, which contains commitTime and tableName only for now. */ -@AllArgsConstructor @Getter @PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING) public class HoodieWriteCommitCallbackMessage implements Serializable { @@ -69,10 +75,116 @@ public class HoodieWriteCommitCallbackMessage implements Serializable { */ private final Option> extraMetadata; + /** + * Previous base file paths keyed by fileId, derived from {@link #hoodieWriteStat} and the + * {@link BaseFileOnlyView} handed over by the write client, so that callback + * implementations don't have to rebuild a view themselves. Empty for inserts and for + * callers that don't supply a view. + * + *

    Holds the resolved map once {@link #getPrevFilePaths()} has run, and stays null until + * then. Not transient: this is the copy that crosses Java serialization, which is why + * {@link #writeObject} forces resolution before writing. Excluded from the generated + * getters so it is published only through {@link #getPrevFilePaths()}. + */ + @Getter(AccessLevel.NONE) + private volatile Map prevFilePaths; + + /** + * Resolves {@link #prevFilePaths} on demand. Resolution is deferred until the first + * {@link #getPrevFilePaths()} call, so a callback that never reads the previous paths pays + * nothing (no FileSystemView access at all). Transient because it captures a + * FileSystemView supplier, which is not serializable: on a deserialized instance this is + * null and the already-resolved {@link #prevFilePaths} is used instead. Excluded from the + * generated getters so the {@link Lazy} wrapper never leaks into JSON. + */ + @Getter(AccessLevel.NONE) + private final transient Lazy> prevFilePathsResolver; + + /** + * Free-form context that producers can attach for downstream callback consumers. + * The OSS write client populates this as empty; specialized callsites or wrappers + * may populate it with whatever context their callbacks need. + */ + private final Map extraContext; + + public HoodieWriteCommitCallbackMessage(String commitTime, + String tableName, + String basePath, + List hoodieWriteStat, + Option commitActionType, + Option> extraMetadata, + Supplier fsViewSupplier, + Map extraContext) { + this.commitTime = commitTime; + this.tableName = tableName; + this.basePath = basePath; + this.hoodieWriteStat = hoodieWriteStat; + this.commitActionType = commitActionType; + this.extraMetadata = extraMetadata; + this.prevFilePathsResolver = Lazy.lazily(() -> HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + hoodieWriteStat, fsViewSupplier == null ? null : fsViewSupplier.get())); + this.extraContext = extraContext; + } + public HoodieWriteCommitCallbackMessage(String commitTime, String tableName, String basePath, List hoodieWriteStat) { - this(commitTime, tableName, basePath, hoodieWriteStat, Option.empty(), Option.empty()); + this(commitTime, tableName, basePath, hoodieWriteStat, Option.empty(), Option.empty(), + null, Collections.emptyMap()); + } + + public HoodieWriteCommitCallbackMessage(String commitTime, + String tableName, + String basePath, + List hoodieWriteStat, + Option commitActionType, + Option> extraMetadata) { + this(commitTime, tableName, basePath, hoodieWriteStat, commitActionType, extraMetadata, + null, Collections.emptyMap()); + } + + /** + * Returns the previous base file paths keyed by fileId, resolving them from the file-system + * view on first access and memoizing the result. A consumer that never calls this triggers + * no FileSystemView lookup. Never null: empty when no view was supplied and when the commit + * only inserted. + */ + public Map getPrevFilePaths() { + Map paths = prevFilePaths; + if (paths == null) { + // The resolver is null only on an instance restored from Java serialization, and there + // the resolved map has already been read back into prevFilePaths (see writeObject). + paths = prevFilePathsResolver == null ? Collections.emptyMap() : prevFilePathsResolver.get(); + prevFilePaths = paths; + } + return paths; + } + + /** + * A {@link BaseFileOnlyView} cannot cross a serialization boundary, so materialize the + * paths at the last possible moment and let the resolved map travel in their place. + */ + private void writeObject(ObjectOutputStream out) throws IOException { + getPrevFilePaths(); + out.defaultWriteObject(); + } + + /** + * Container for previously-existing file paths associated with a single fileId in a + * commit. {@link #baseFilePath} is the base file the new write replaces, and + * {@link #bootstrapBaseFilePath} is the bootstrap-source file the previous + * base file referenced (null for non-bootstrap tables). + */ + @Getter + public static class PrevFilePaths implements Serializable { + private static final long serialVersionUID = 1L; + private final String baseFilePath; + private final String bootstrapBaseFilePath; + + public PrevFilePaths(String baseFilePath, String bootstrapBaseFilePath) { + this.baseFilePath = baseFilePath; + this.bootstrapBaseFilePath = bootstrapBaseFilePath; + } } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieWriteCommitCallbackUtil.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieWriteCommitCallbackUtil.java index cd05b78dfcf2b..c255e31b9fc8e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieWriteCommitCallbackUtil.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieWriteCommitCallbackUtil.java @@ -17,15 +17,27 @@ package org.apache.hudi.callback.util; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage.PrevFilePaths; +import org.apache.hudi.common.model.BaseFile; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieCommitCallbackException; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Util helps to prepare callback message. */ +@Slf4j public class HoodieWriteCommitCallbackUtil { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -41,4 +53,46 @@ public static String convertToJsonString(Object obj) { } } + /** + * Resolve the previous base file (and bootstrap base file, if any) for every + * {@link HoodieWriteStat} that represents an update, using a populated + * {@link BaseFileOnlyView}. The lookup is O(1) per stat against the cached view, so + * this adds no I/O on top of what the writer already paid. + * + *

    Feeds {@link org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage#getPrevFilePaths()} + * so the callback message can ship actual file paths rather than forcing each callback + * impl to rebuild a {@code FileSystemView}. + */ + public static Map resolvePrevFilePaths(List stats, + BaseFileOnlyView fsView) { + Map pathsByFileId = new HashMap<>(); + if (stats == null || fsView == null) { + return pathsByFileId; + } + for (HoodieWriteStat stat : stats) { + String prevCommit = stat.getPrevCommit(); + if (StringUtils.isNullOrEmpty(prevCommit) || HoodieWriteStat.NULL_COMMIT.equals(prevCommit)) { + continue; + } + Option prev; + try { + prev = fsView.getBaseFileOn(stat.getPartitionPath(), prevCommit, stat.getFileId()); + } catch (Exception e) { + // Best-effort: a remote view 4xx/5xx, a stale view, or a replaced file group must not + // fail the commit. Drop the prev path for this stat and keep going. + log.warn("Could not resolve prev base file for fileId={} prevCommit={}; skipping", + stat.getFileId(), prevCommit, e); + continue; + } + if (!prev.isPresent()) { + continue; + } + HoodieBaseFile prevBaseFile = prev.get(); + Option bootstrapBaseFile = prevBaseFile.getBootstrapBaseFile(); + String prevPath = prevBaseFile.getPath(); + String bootstrapPath = bootstrapBaseFile.isPresent() ? bootstrapBaseFile.get().getPath() : null; + pathsByFileId.put(stat.getFileId(), new PrevFilePaths(prevPath, bootstrapPath)); + } + return pathsByFileId; + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java index c04454bf08570..fbd0d5e4f5794 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java @@ -20,6 +20,9 @@ import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.callback.HoodieClientInitCallback; +import org.apache.hudi.callback.HoodieWriteCommitCallback; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; +import org.apache.hudi.callback.util.HoodieCommitCallbackFactory; import org.apache.hudi.client.embedded.EmbeddedTimelineServerHelper; import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient; @@ -34,6 +37,7 @@ import org.apache.hudi.common.table.timeline.TimeGenerator; import org.apache.hudi.common.table.timeline.TimeGenerators; import org.apache.hudi.common.table.timeline.TimelineUtils; +import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; @@ -65,6 +69,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -87,6 +92,14 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable { protected final TransactionManager txnManager; protected final TimeGenerator timeGenerator; + /** + * Lazily-initialized commit callback (HoodieWriteCommitCallback). Lifted from + * {@link BaseHoodieWriteClient} so that {@link BaseHoodieTableServiceClient} can also + * fire callbacks for compaction and clustering completions. Transient is fine + * because the callback is only ever invoked from the driver after a commit. + */ + protected transient HoodieWriteCommitCallback commitCallback; + /** * Timeline Server has the same lifetime as that of Client. Any operations done on the same timeline service will be * able to take advantage of the cached file-system view. New completed actions will be synced automatically in an @@ -462,4 +475,36 @@ private static Map collectRollingMetadataFromTimeline( protected Option> updateExtraMetadata(Option> extraMetadata) { return CommitMetadataProperties.enrich(extraMetadata, config, context); } + + /** + * Fire {@link HoodieWriteCommitCallback} for a commit, if enabled. Shared by + * {@link BaseHoodieWriteClient#postCommit} (regular auto- and explicit-commit paths) + * and {@link BaseHoodieTableServiceClient} (compaction and clustering completions). + * Lazily constructs the callback instance from {@code hoodie.write.commit.callback.class}. + * + *

    Best-effort: catches and logs any exception from the user-supplied callback so a + * misbehaving observer cannot fail the commit. + */ + protected void fireCommitCallbackIfNecessary(String commitTime, + String commitActionType, + List stats, + Supplier fsViewSupplier, + Option> extraMetadata) { + if (!config.writeCommitCallbackOn()) { + return; + } + try { + if (commitCallback == null) { + commitCallback = HoodieCommitCallbackFactory.create(config); + } + commitCallback.call(new HoodieWriteCommitCallbackMessage( + commitTime, config.getTableName(), config.getBasePath(), + stats, Option.of(commitActionType), extraMetadata, + fsViewSupplier, + Collections.emptyMap())); + } catch (Exception e) { + log.warn("HoodieWriteCommitCallback failed for commit {} ({}); ignoring", + commitTime, commitActionType, e); + } + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index 739e11933a38b..cbd6c7562c1db 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -425,6 +425,8 @@ protected void completeCompaction(HoodieCommitMetadata metadata, HoodieTable tab ); } log.info("Compacted successfully on commit {}", compactionCommitTime); + fireCommitCallbackIfNecessary(compactionCommitTime, HoodieTimeline.COMMIT_ACTION, + writeStats, table::getBaseFileOnlyView, Option.empty()); } finally { if (config.getWriteConcurrencyMode().supportsMultiWriter()) { this.heartbeatClient.stop(compactionCommitTime); @@ -497,6 +499,8 @@ protected void completeLogCompaction(HoodieCommitMetadata metadata, HoodieTable ); } log.info("Log Compacted successfully on commit {}", logCompactionCommitTime); + fireCommitCallbackIfNecessary(logCompactionCommitTime, HoodieTimeline.DELTA_COMMIT_ACTION, + writeStats, table::getBaseFileOnlyView, Option.empty()); } /** @@ -641,6 +645,8 @@ private void completeClustering(HoodieReplaceCommitMetadata replaceCommitMetadat heartbeatClient.stop(clusteringCommitTime); } log.info("Clustering successfully on commit {} for table {}", clusteringCommitTime, table.getConfig().getBasePath()); + fireCommitCallbackIfNecessary(clusteringCommitTime, clusteringInstant.getAction(), + writeStats, table::getBaseFileOnlyView, Option.empty()); } protected void runTableServicesInline(HoodieTable table, HoodieCommitMetadata metadata, Option> extraMetadata) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 07b08c44768fa..d97ea7dbb7cd1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -24,10 +24,7 @@ import org.apache.hudi.avro.model.HoodieRestoreMetadata; import org.apache.hudi.avro.model.HoodieRestorePlan; import org.apache.hudi.avro.model.HoodieRollbackMetadata; -import org.apache.hudi.callback.HoodieWriteCommitCallback; -import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; import org.apache.hudi.callback.common.WriteStatusValidator; -import org.apache.hudi.callback.util.HoodieCommitCallbackFactory; import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.client.heartbeat.HeartbeatUtils; import org.apache.hudi.client.transaction.TransactionManager; @@ -147,7 +144,6 @@ public abstract class BaseHoodieWriteClient extends BaseHoodieClient @Getter @Setter private transient WriteOperationType operationType; - private transient HoodieWriteCommitCallback commitCallback; protected transient Timer.Context writeTimer = null; @@ -288,7 +284,7 @@ public boolean commitStats(String instantTime, TableWriteStats tableWriteStats, boolean postCommitStatus = true; HoodieTimer postCommitTimer = HoodieTimer.start(); try { - postCommit(table, metadata, instantTime, extraMetadata); + postCommit(table, metadata, instantTime, commitActionType, extraMetadata); mayBeCleanAndArchive(table); runTableServicesInline(table, metadata, extraMetadata); } catch (Exception e) { @@ -304,15 +300,6 @@ public boolean commitStats(String instantTime, TableWriteStats tableWriteStats, } emitCommitMetrics(instantTime, metadata, commitActionType); - - // callback if needed. - if (config.writeCommitCallbackOn()) { - if (null == commitCallback) { - commitCallback = HoodieCommitCallbackFactory.create(config); - } - commitCallback.call(new HoodieWriteCommitCallbackMessage( - instantTime, config.getTableName(), config.getBasePath(), tableWriteStats.getDataTableWriteStats(), Option.of(commitActionType), extraMetadata)); - } return true; } @@ -643,7 +630,9 @@ public O postWrite(HoodieWriteMetadata result, String instantTime, HoodieTabl boolean postCommitStatus = true; HoodieTimer postCommitTimer = HoodieTimer.start(); try { - postCommit(hoodieTable, result.getCommitMetadata().get(), instantTime, Option.empty()); + String commitActionType = CommitUtils.getCommitActionType(operationType, hoodieTable.getMetaClient().getTableType()); + postCommit(hoodieTable, result.getCommitMetadata().get(), instantTime, + commitActionType, Option.empty()); mayBeCleanAndArchive(hoodieTable); } catch (Exception e) { postCommitStatus = false; @@ -670,7 +659,7 @@ public O postWrite(HoodieWriteMetadata result, String instantTime, HoodieTabl * @param instantTime Instant Time * @param extraMetadata Additional Metadata passed by user */ - protected void postCommit(HoodieTable table, HoodieCommitMetadata metadata, String instantTime, Option> extraMetadata) { + protected void postCommit(HoodieTable table, HoodieCommitMetadata metadata, String instantTime, String commitActionType, Option> extraMetadata) { try { context.setJobStatus(this.getClass().getSimpleName(), "Cleaning up marker directories for commit " + instantTime + " in table " + config.getTableName()); @@ -678,6 +667,12 @@ protected void postCommit(HoodieTable table, HoodieCommitMetadata metadata, Stri WriteMarkersFactory.get(config.getMarkersType(), table, instantTime) .quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism()); metrics.updateTableServiceInstantMetrics(table.getActiveTimeline()); + // Fire write commit callback if a callback class is registered. postCommit() is reached + // by both auto-commit and explicit-commit paths; compaction and clustering have their own + // explicit fireCommitCallbackIfNecessary call sites in BaseHoodieTableServiceClient. + List stats = metadata.getWriteStats(); + fireCommitCallbackIfNecessary(instantTime, commitActionType, stats, + table::getBaseFileOnlyView, extraMetadata); } finally { this.heartbeatClient.stop(instantTime); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/common/TestHoodieWriteCommitCallbackMessage.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/common/TestHoodieWriteCommitCallbackMessage.java new file mode 100644 index 0000000000000..1b27ef50000e8 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/common/TestHoodieWriteCommitCallbackMessage.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.callback.common; + +import org.apache.hudi.callback.util.HoodieWriteCommitCallbackUtil; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage.PrevFilePaths; +import org.apache.hudi.common.model.BaseFile; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.util.Option; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the {@link HoodieWriteCommitCallbackMessage} contract: default (never-null) + * collections, lazy one-shot resolution of {@code prevFilePaths} off the supplied file-system + * view, and the Java-serialization round trip (the view cannot be shipped, so the resolved + * paths must be materialized at the boundary and carried across in its place). + */ +public class TestHoodieWriteCommitCallbackMessage { + + private static final String COMMIT_TIME = "002"; + private static final String PARTITION = "2024/01/01"; + private static final String PREV_COMMIT = "001"; + private static final String PREV_PATH = "/tbl/" + PARTITION + "/f0_0-1-1_" + PREV_COMMIT + ".parquet"; + private static final String BOOTSTRAP_PATH = "/bootstrap/source/f0.parquet"; + + private static List updateStat() { + HoodieWriteStat writeStat = new HoodieWriteStat(); + writeStat.setFileId("f0"); + writeStat.setPartitionPath(PARTITION); + writeStat.setPrevCommit(PREV_COMMIT); + return Collections.singletonList(writeStat); + } + + private static BaseFileOnlyView viewResolving(String prevBaseFilePath) { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "f0")) + .thenReturn(Option.of(new HoodieBaseFile(prevBaseFilePath))); + return view; + } + + private static BaseFileOnlyView viewResolvingWithBootstrap(String prevBaseFilePath, String bootstrapPath) { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "f0")) + .thenReturn(Option.of(new HoodieBaseFile(prevBaseFilePath, new BaseFile(bootstrapPath)))); + return view; + } + + @Test + public void callbackMessageDefaultsCollectionsToEmpty() { + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", Collections.emptyList()); + + assertFalse(message.getCommitActionType().isPresent()); + assertFalse(message.getExtraMetadata().isPresent()); + assertTrue(message.getPrevFilePaths().isEmpty(), "prevFilePaths must default to an empty map, never null"); + assertTrue(message.getExtraContext().isEmpty(), "extraContext must default to an empty map, never null"); + } + + @Test + public void callbackMessageResolvesPrevFilePathsFromViewAndRetainsContext() { + Map extraContext = Collections.singletonMap("file_id", "f0"); + + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", updateStat(), + Option.of("commit"), Option.empty(), () -> viewResolving(PREV_PATH), extraContext); + + assertEquals("commit", message.getCommitActionType().get()); + PrevFilePaths resolved = message.getPrevFilePaths().get("f0"); + assertEquals(PREV_PATH, resolved.getBaseFilePath()); + assertEquals(extraContext, message.getExtraContext()); + } + + @Test + public void nullFileSystemViewSupplierYieldsEmptyPrevFilePaths() { + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", updateStat(), + Option.of("commit"), Option.empty(), null, Collections.emptyMap()); + + assertTrue(message.getPrevFilePaths().isEmpty(), + "a message built without a file-system view must yield an empty map, never null"); + } + + @Test + public void prevFilePathsAreResolvedLazilyAndMemoized() { + AtomicInteger viewLookups = new AtomicInteger(); + BaseFileOnlyView view = viewResolving(PREV_PATH); + Supplier viewSupplier = () -> { + viewLookups.incrementAndGet(); + return view; + }; + + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", updateStat(), + Option.empty(), Option.empty(), viewSupplier, Collections.emptyMap()); + + // Constructing the message must not touch the file-system view. + assertEquals(0, viewLookups.get(), + "prevFilePaths must not be resolved until a consumer reads them"); + verify(view, never()).getBaseFileOn(anyString(), anyString(), anyString()); + + assertEquals(PREV_PATH, message.getPrevFilePaths().get("f0").getBaseFilePath()); + // A second read must reuse the memoized result rather than resolve again. + assertEquals(PREV_PATH, message.getPrevFilePaths().get("f0").getBaseFilePath()); + assertEquals(1, viewLookups.get(), "prevFilePaths must be resolved at most once and memoized"); + verify(view).getBaseFileOn(PARTITION, PREV_COMMIT, "f0"); + } + + @Test + public void javaSerializationResolvesAndPreservesPrevFilePaths() throws IOException, ClassNotFoundException { + AtomicInteger viewLookups = new AtomicInteger(); + // The view supplier cannot be shipped, so writeObject has to materialize the paths first. + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", updateStat(), + Option.of("commit"), Option.empty(), + () -> { + viewLookups.incrementAndGet(); + return viewResolvingWithBootstrap(PREV_PATH, BOOTSTRAP_PATH); + }, + Collections.emptyMap()); + + assertEquals(0, viewLookups.get(), "building the message must not touch the file-system view"); + + HoodieWriteCommitCallbackMessage roundTripped = serializeAndDeserialize(message); + + assertEquals(1, viewLookups.get(), "serialization must force resolution exactly once"); + assertEquals(COMMIT_TIME, roundTripped.getCommitTime()); + assertEquals("commit", roundTripped.getCommitActionType().get()); + assertEquals(1, roundTripped.getHoodieWriteStat().size()); + assertEquals(PREV_PATH, roundTripped.getPrevFilePaths().get("f0").getBaseFilePath()); + assertEquals(BOOTSTRAP_PATH, roundTripped.getPrevFilePaths().get("f0").getBootstrapBaseFilePath(), + "the bootstrap source path must survive the round trip too"); + } + + @Test + public void jsonPayloadExposesPrevFilePathsAndNotTheResolver() { + HoodieWriteCommitCallbackMessage message = new HoodieWriteCommitCallbackMessage( + COMMIT_TIME, "table", "/base", updateStat(), + Option.of("commit"), Option.empty(), () -> viewResolving(PREV_PATH), Collections.emptyMap()); + + // This is the payload the built-in HTTP/Kafka/Pulsar callbacks put on the wire. + String json = HoodieWriteCommitCallbackUtil.convertToJsonString(message); + + assertTrue(json.contains("\"prevFilePaths\""), "prevFilePaths must be part of the callback payload"); + assertTrue(json.contains(PREV_PATH), "Jackson must see the resolved paths, not the lazy holder"); + assertFalse(json.contains("prevFilePathsResolver"), + "the lazy resolver is an implementation detail and must never reach the payload"); + } + + private static HoodieWriteCommitCallbackMessage serializeAndDeserialize( + HoodieWriteCommitCallbackMessage message) throws IOException, ClassNotFoundException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(message); + } + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (HoodieWriteCommitCallbackMessage) in.readObject(); + } + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/util/TestHoodieWriteCommitCallbackUtil.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/util/TestHoodieWriteCommitCallbackUtil.java new file mode 100644 index 0000000000000..cb05b378461d6 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/util/TestHoodieWriteCommitCallbackUtil.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.callback.util; + +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage.PrevFilePaths; +import org.apache.hudi.common.model.BaseFile; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.util.Option; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for + * {@link HoodieWriteCommitCallbackUtil#resolvePrevFilePaths(List, BaseFileOnlyView)}, which + * pre-resolves the previous base file (and bootstrap source, if any) for each updated file + * group from a cached {@link BaseFileOnlyView}, so callback implementations receive the + * read/write file pairing without rebuilding a file-system view. + */ +public class TestHoodieWriteCommitCallbackUtil { + + private static final String PARTITION = "2024/01/01"; + private static final String PREV_COMMIT = "001"; + + private static HoodieWriteStat stat(String fileId, String partitionPath, String prevCommit) { + HoodieWriteStat writeStat = new HoodieWriteStat(); + writeStat.setFileId(fileId); + writeStat.setPartitionPath(partitionPath); + writeStat.setPrevCommit(prevCommit); + return writeStat; + } + + @Test + public void resolvePrevFilePathsReturnsEmptyForNullInputs() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + assertTrue(HoodieWriteCommitCallbackUtil.resolvePrevFilePaths(null, view).isEmpty(), + "null stats must yield an empty map"); + assertTrue(HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + Collections.singletonList(stat("f0", PARTITION, PREV_COMMIT)), null).isEmpty(), + "null file-system view must yield an empty map"); + } + + @Test + public void resolvePrevFilePathsSkipsStatsWithoutAPrevCommit() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + List inserts = Arrays.asList( + stat("f-null", PARTITION, null), + stat("f-empty", PARTITION, ""), + stat("f-nullcommit", PARTITION, HoodieWriteStat.NULL_COMMIT)); + + Map resolved = + HoodieWriteCommitCallbackUtil.resolvePrevFilePaths(inserts, view); + + assertTrue(resolved.isEmpty(), "inserts (no prevCommit) must not resolve a prev base file"); + // The view must not even be consulted for inserts. + verify(view, never()).getBaseFileOn(anyString(), anyString(), anyString()); + } + + @Test + public void resolvePrevFilePathsResolvesUpdatePrevBaseFile() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + HoodieBaseFile prevBase = new HoodieBaseFile("/tbl/" + PARTITION + "/f0_0-1-1_" + PREV_COMMIT + ".parquet"); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "f0")).thenReturn(Option.of(prevBase)); + + Map resolved = HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + Collections.singletonList(stat("f0", PARTITION, PREV_COMMIT)), view); + + assertEquals(1, resolved.size()); + assertEquals(prevBase.getPath(), resolved.get("f0").getBaseFilePath()); + assertNull(resolved.get("f0").getBootstrapBaseFilePath(), "non-bootstrap update has no bootstrap path"); + } + + @Test + public void resolvePrevFilePathsCapturesBootstrapBaseFile() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + BaseFile bootstrap = new BaseFile("/bootstrap/source/f0.parquet"); + HoodieBaseFile prevBase = new HoodieBaseFile("/tbl/" + PARTITION + "/f0_0-1-1_" + PREV_COMMIT + ".parquet", bootstrap); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "f0")).thenReturn(Option.of(prevBase)); + + Map resolved = HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + Collections.singletonList(stat("f0", PARTITION, PREV_COMMIT)), view); + + assertEquals(prevBase.getPath(), resolved.get("f0").getBaseFilePath()); + assertEquals(bootstrap.getPath(), resolved.get("f0").getBootstrapBaseFilePath(), + "bootstrap source path must be carried through for bootstrapped file groups"); + } + + @Test + public void resolvePrevFilePathsSkipsWhenBaseFileAbsent() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "f0")).thenReturn(Option.empty()); + + Map resolved = HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + Collections.singletonList(stat("f0", PARTITION, PREV_COMMIT)), view); + + assertTrue(resolved.isEmpty(), "a missing prev base file must be skipped, not mapped to null"); + } + + @Test + public void resolvePrevFilePathsIsBestEffortOnViewFailure() { + BaseFileOnlyView view = mock(BaseFileOnlyView.class); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "boom")) + .thenThrow(new RuntimeException("stale or remote view error")); + HoodieBaseFile prevBase = new HoodieBaseFile("/tbl/" + PARTITION + "/ok_0-1-1_" + PREV_COMMIT + ".parquet"); + when(view.getBaseFileOn(PARTITION, PREV_COMMIT, "ok")).thenReturn(Option.of(prevBase)); + + Map resolved = HoodieWriteCommitCallbackUtil.resolvePrevFilePaths( + Arrays.asList(stat("boom", PARTITION, PREV_COMMIT), stat("ok", PARTITION, PREV_COMMIT)), view); + + // The failing file group is dropped; resolution continues for the rest (must not fail the commit). + assertFalse(resolved.containsKey("boom")); + assertEquals(prevBase.getPath(), resolved.get("ok").getBaseFilePath()); + } +} diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java index 9295151156277..0ab62d498dddd 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java @@ -100,6 +100,8 @@ protected void completeCompaction(HoodieCommitMetadata metadata, HoodieTable tab } } log.info("Compacted successfully on commit {}", compactionCommitTime); + fireCommitCallbackIfNecessary(compactionCommitTime, HoodieActiveTimeline.COMMIT_ACTION, + metadata.getWriteStats(), table::getBaseFileOnlyView, Option.empty()); } finally { if (config.getWriteConcurrencyMode().supportsMultiWriter()) { this.heartbeatClient.stop(compactionCommitTime); @@ -160,6 +162,8 @@ protected void completeClustering( } } log.info("Clustering successfully on commit {}", clusteringCommitTime); + fireCommitCallbackIfNecessary(clusteringCommitTime, clusteringInstant.getAction(), + writeStats, table::getBaseFileOnlyView, Option.empty()); } @Override diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java index 8f4a73f51b088..3964dd69a517f 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java @@ -18,14 +18,21 @@ package org.apache.hudi.client.functional; +import org.apache.hudi.callback.HoodieWriteCommitCallback; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; import org.apache.hudi.client.HoodieJavaWriteClient; import org.apache.hudi.client.WriteClientTestUtils; +import org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy; +import org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.SyncableFileSystemView; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteCommitCallbackConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.action.HoodieWriteMetadata; @@ -37,12 +44,16 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.apache.hudi.common.testutils.HoodieTestUtils.TIMELINE_FACTORY; import static org.apache.hudi.testutils.GenericRecordValidationTestUtils.assertDataInMORTable; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieJavaClientOnMergeOnReadStorage extends HoodieJavaClientTestHarness { @@ -180,4 +191,114 @@ protected HoodieTableType getTableType() { return HoodieTableType.MERGE_ON_READ; } + @Test + public void testWriteCommitCallbackFiresOnCompaction() throws Exception { + RecordingCommitCallback.MESSAGES.clear(); + HoodieWriteConfig config = getConfigBuilder(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA, + HoodieIndex.IndexType.INMEMORY) + .withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(2).build()) + .withCallbackConfig(HoodieWriteCommitCallbackConfig.newBuilder() + .writeCommitCallbackOn("true") + .withCallbackClass(RecordingCommitCallback.class.getName()) + .build()) + .build(); + HoodieJavaWriteClient client = getHoodieWriteClient(config); + + // Two delta commits through the auto-commit path. + String commitTime = WriteClientTestUtils.createNewInstantTime(); + insertBatch(config, client, commitTime, "000", 100, HoodieJavaWriteClient::insert, + false, false, 100, 100, 1, Option.empty(), INSTANT_GENERATOR); + String prevCommit = commitTime; + commitTime = WriteClientTestUtils.createNewInstantTime(); + updateBatch(config, client, commitTime, prevCommit, + Option.of(Arrays.asList(prevCommit)), "000", 50, HoodieJavaWriteClient::upsert, + false, false, 5, 100, 2, config.populateMetaFields(), INSTANT_GENERATOR); + + // The callback must fire for the auto-committed delta commits with the deltacommit action. + assertTrue(RecordingCommitCallback.MESSAGES.stream().anyMatch(m -> + HoodieTimeline.DELTA_COMMIT_ACTION.equals(m.getCommitActionType().orElse(null))), + "callback must fire for delta commits"); + + // Schedule, execute and commit compaction. + Option compactionTime = client.scheduleCompaction(Option.empty()); + assertTrue(compactionTime.isPresent()); + HoodieWriteMetadata writeMetadata = client.compact(compactionTime.get()); + client.commitCompaction(compactionTime.get(), writeMetadata, Option.empty()); + assertTrue(metaClient.reloadActiveTimeline().filterCompletedInstants().containsInstant(compactionTime.get())); + + // The callback must fire exactly once for the compaction completion, reporting the completed + // timeline action (commit). + List compactionMessages = RecordingCommitCallback.MESSAGES.stream() + .filter(m -> m.getCommitTime().equals(compactionTime.get())) + .collect(Collectors.toList()); + assertEquals(1, compactionMessages.size(), "callback must fire once for the compaction commit"); + assertEquals(HoodieTimeline.COMMIT_ACTION, compactionMessages.get(0).getCommitActionType().orElse(null)); + assertNotNull(compactionMessages.get(0).getPrevFilePaths(), "prevFilePaths must never be null"); + } + + @Test + public void testWriteCommitCallbackFiresOnClustering() throws Exception { + RecordingCommitCallback.MESSAGES.clear(); + HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder() + .withClusteringMaxNumGroups(10) + .withClusteringSortColumns("_row_key") + .withClusteringTargetPartitions(0) + .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName()) + .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName()) + .build(); + HoodieWriteConfig config = getConfigBuilder(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA, + HoodieIndex.IndexType.INMEMORY) + .withClusteringConfig(clusteringConfig) + .withCallbackConfig(HoodieWriteCommitCallbackConfig.newBuilder() + .writeCommitCallbackOn("true") + .withCallbackClass(RecordingCommitCallback.class.getName()) + .build()) + .build(); + HoodieJavaWriteClient client = getHoodieWriteClient(config); + + // Two inserts create base-file groups that clustering can rewrite. + String commitTime = WriteClientTestUtils.createNewInstantTime(); + insertBatch(config, client, commitTime, "000", 100, HoodieJavaWriteClient::insert, + false, false, 100, 100, 1, Option.empty(), INSTANT_GENERATOR); + commitTime = WriteClientTestUtils.createNewInstantTime(); + insertBatch(config, client, commitTime, "001", 100, HoodieJavaWriteClient::insert, + false, false, 100, 200, 2, Option.empty(), INSTANT_GENERATOR); + + // Schedule and execute clustering inline (shouldComplete = true completes the commit). + Option clusteringTime = client.scheduleClustering(Option.empty()); + assertTrue(clusteringTime.isPresent(), "expected a clustering plan to be scheduled"); + client.cluster(clusteringTime.get(), true); + assertTrue(metaClient.reloadActiveTimeline().filterCompletedInstants().containsInstant(clusteringTime.get())); + + // The callback must fire once for the clustering completion, reporting the action actually on + // the timeline (replacecommit for table version < 8, clustering for 8+). + List clusteringMessages = RecordingCommitCallback.MESSAGES.stream() + .filter(m -> m.getCommitTime().equals(clusteringTime.get())) + .collect(Collectors.toList()); + assertEquals(1, clusteringMessages.size(), "callback must fire once for the clustering commit"); + String action = clusteringMessages.get(0).getCommitActionType().orElse(null); + assertTrue(HoodieTimeline.REPLACE_COMMIT_ACTION.equals(action) || HoodieTimeline.CLUSTERING_ACTION.equals(action), + "clustering callback must report the timeline action, got: " + action); + } + + /** + * A recording {@link HoodieWriteCommitCallback} that captures every fired message so tests can + * assert the callback fires for table-service (compaction/clustering) commits with the expected + * action type. Loaded reflectively from the write config, so it needs a public + * {@code (HoodieWriteConfig)} constructor. + */ + public static class RecordingCommitCallback implements HoodieWriteCommitCallback { + + static final List MESSAGES = new CopyOnWriteArrayList<>(); + + public RecordingCommitCallback(HoodieWriteConfig config) { + // config arg required for reflective instantiation + } + + @Override + public void call(HoodieWriteCommitCallbackMessage callbackMessage) { + MESSAGES.add(callbackMessage); + } + } + } From abea033d67a7df78c861e27986536f768a31703b Mon Sep 17 00:00:00 2001 From: Shuo Cheng Date: Fri, 31 Jul 2026 16:58:36 +0800 Subject: [PATCH 222/255] perf(common): avoid UTF-8 allocations in string comparator (#19414) * perf(common): avoid UTF-8 allocations in string comparator * fix(common): address UTF-8 comparator review feedback Preserve the null-rejection contract, document malformed surrogate behavior, add Firebase attribution, and strengthen the HFile-ordering tests. (cherry picked from commit 377fc0419ae81db92ee5ab2dd862c2554902a383) --- LICENSE | 12 ++++ .../apache/hudi/common/util/StringUtils.java | 37 ++++++---- .../hudi/common/util/TestStringUtils.java | 72 +++++++++++++++++++ 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index 301ea869628ba..f0d0759bd1012 100644 --- a/LICENSE +++ b/LICENSE @@ -349,6 +349,18 @@ Copyright (c) 2005, European Commission project OneLab under contract 034819 (ht ------------------------------------------------------------------------------- + This product includes code from the Google Firebase Android SDK + + * org.apache.hudi.common.util.StringUtils#compareUtf8Bytes ported from + com.google.firebase.firestore.util.Util#compareUtf8Strings + + Copyright 2018 Google LLC + + Home page: https://github.com/firebase/firebase-android-sdk + License: https://www.apache.org/licenses/LICENSE-2.0 + + ------------------------------------------------------------------------------- + This product includes code from StreamSets Data Collector * com.streamsets.pipeline.lib.util.avroorc.AvroToOrcRecordConverter copied and modified to org.apache.hudi.common.util.AvroOrcUtils diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java index 2fdefeadc7931..8d72a20d71993 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java @@ -136,23 +136,36 @@ public static byte[] getUTF8Bytes(String str) { *

    Neither argument may be {@code null}; like {@link String#compareTo(String)}, a {@code null} * argument throws {@link NullPointerException}. * - *

    Assumes well-formed UTF-16 input: {@code String#getBytes(UTF_8)} replaces unpaired surrogates - * with {@code '?'}, so strings differing only in unpaired surrogates compare equal. + *

    This comparison does not materialize the UTF-8 byte arrays. It compares UTF-16 code units + * directly and handles supplementary characters specially to preserve UTF-8 byte order. * - *

    Note: encodes both strings to UTF-8 on every call; for very large sorts consider - * pre-encoding keys to byte arrays once and comparing those. + *

    Assumes well-formed UTF-16 input. For strings containing unpaired surrogates the result no + * longer matches {@code String#getBytes(UTF_8)} byte order: the encoder replaces an unpaired + * surrogate with {@code '?'} while this method sorts it after every BMP character. Production + * callers derive keys by decoding UTF-8, which cannot produce unpaired surrogates. + * + *

    Ported from Google Firebase Firestore's {@code compareUtf8Strings}. */ public static int compareUtf8Bytes(String s1, String s2) { - byte[] b1 = getUTF8Bytes(s1); - byte[] b2 = getUTF8Bytes(s2); - int len = Math.min(b1.length, b2.length); - for (int i = 0; i < len; i++) { - int cmp = (b1[i] & 0xFF) - (b2[i] & 0xFF); - if (cmp != 0) { - return cmp; + // Source: https://github.com/firebase/firebase-android-sdk/blob/f05e4bcb7f86f3b21833b1e0960d793b800d38d1/firebase-firestore/src/main/java/com/google/firebase/firestore/util/Util.java#L76-L132 + // The identity check intentionally avoids scanning when both references point to the same + // non-null String while preserving the method's fail-fast null contract. + if (s1 == s2 && s1 != null) { + return 0; + } + + final int length = Math.min(s1.length(), s2.length()); + for (int i = 0; i < length; i++) { + final char char1 = s1.charAt(i); + final char char2 = s2.charAt(i); + if (char1 != char2) { + return (Character.isSurrogate(char1) == Character.isSurrogate(char2)) + ? Character.compare(char1, char2) + : Character.isSurrogate(char1) ? 1 : -1; } } - return b1.length - b2.length; + + return Integer.compare(s1.length(), s2.length()); } public static String fromUTF8Bytes(byte[] bytes) { diff --git a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java index c5791222f1df2..4d35654b0baf5 100644 --- a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java +++ b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java @@ -19,6 +19,8 @@ package org.apache.hudi.common.util; +import org.apache.hudi.io.hfile.UTF8StringKey; + import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; @@ -306,6 +308,74 @@ public void testCompareUtf8BytesEmptyPrefixAndIdenticalStrings() { assertTrue(StringUtils.compareUtf8Bytes("ab", "abc") < 0); assertTrue(StringUtils.compareUtf8Bytes("abc", "ab") > 0); assertEquals(0, StringUtils.compareUtf8Bytes("abc", "abc")); + assertEquals(0, StringUtils.compareUtf8Bytes(new String("abc"), new String("abc"))); + } + + @Test + public void testCompareUtf8BytesDocumentsUnpairedSurrogateBehavior() { + String unpairedSurrogate = String.valueOf((char) 0xD800); + String replacementCharacter = String.valueOf((char) 0xFFFD); + + // Java's UTF-8 encoder replaces the unpaired surrogate with '?' while the Firestore-derived + // comparator orders all surrogate code units after BMP characters. Production callers decode + // UTF-8 into well-formed UTF-16, so malformed strings are outside this method's contract. + assertTrue(StringUtils.compareUtf8Bytes(unpairedSurrogate, replacementCharacter) > 0); + assertTrue(StringUtils.compareUtf8Bytes(unpairedSurrogate, "?") > 0); + } + + @Test + public void testCompareUtf8BytesMatchesEncodedByteOrder() { + String[] alphabet = { + // One-byte UTF-8 characters, including the upper boundary. + "?", + "a", + String.valueOf((char) 0x007F), + // Two-byte UTF-8 lower and upper boundaries. + String.valueOf((char) 0x0080), + String.valueOf((char) 0x07FF), + // Three-byte UTF-8 boundaries around the surrogate range, plus U+FFFD. + String.valueOf((char) 0x0800), + String.valueOf((char) 0xD7FF), + String.valueOf((char) 0xE000), + String.valueOf((char) 0xFFFD), + // Four-byte UTF-8 supplementary characters, including two sharing a high surrogate. + "😀", // U+1F600 + new String(Character.toChars(0x20000)), + new String(Character.toChars(0x20001)), + new String(Character.toChars(0x10FFFF)) + }; + + // Generate every sequence of zero to three code points from the alphabet. This covers cases + // where strings differ before, within, or after a supplementary character. + List values = new ArrayList<>(); + values.add(""); + for (String first : alphabet) { + values.add(first); + for (String second : alphabet) { + values.add(first + second); + for (String third : alphabet) { + values.add(first + second + third); + } + } + } + + // Pre-encode each value once, then use the production HFile key comparator as the oracle. + UTF8StringKey[] hfileKeys = values.stream() + .map(UTF8StringKey::new) + .toArray(UTF8StringKey[]::new); + + // Compare only the sign because Comparator does not prescribe the magnitude of its result. + for (int leftIndex = 0; leftIndex < values.size(); leftIndex++) { + String left = values.get(leftIndex); + for (int rightIndex = 0; rightIndex < values.size(); rightIndex++) { + String right = values.get(rightIndex); + assertEquals( + Integer.signum(hfileKeys[leftIndex].compareTo(hfileKeys[rightIndex])), + Integer.signum(StringUtils.compareUtf8Bytes(left, right)), + () -> "left=" + Arrays.toString(left.codePoints().toArray()) + + " right=" + Arrays.toString(right.codePoints().toArray())); + } + } } @Test @@ -313,6 +383,8 @@ public void testCompareUtf8BytesEmptyPrefixAndIdenticalStrings() { public void testUtf8LexicographicComparatorSerializableAndRejectsNull() throws Exception { // Like String.compareTo, a null argument is rejected. assertThrows(NullPointerException.class, () -> StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR.compare(null, "a")); + assertThrows(NullPointerException.class, () -> StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR.compare("a", null)); + assertThrows(NullPointerException.class, () -> StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR.compare(null, null)); // The comparator is declared as (Comparator & Serializable) so Spark can capture it inside // serialized closures. Round-trip it through Java serialization and confirm the deserialized From f96c7c1ac2d618a308ac200d72aa6a685875ac3d Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Fri, 31 Jul 2026 02:24:32 -0700 Subject: [PATCH 223/255] fix(hive-sync): make skip_ro_suffix take precedence over sync_snapshot_with_table_name (#19427) * fix(hive-sync): make skip_ro_suffix take precedence over sync_snapshot_with_table_name hoodie.datasource.hive_sync.skip_ro_suffix=true claims the bare table name for the read-optimized (RO) view of a MOR table. Hudi 1.x flipped the default of hoodie.meta.sync.sync_snapshot_with_table_name from false to true (HUDI-7415), so under the default ALL hive-sync-table-strategy, HiveSyncTool.doSync() now syncs the bare table name twice in the same run when both configs are true: once as the RO table, then again as the RT table, with the RT sync silently winning. This makes skip_ro_suffix a no-op and breaks read-optimized queries against the bare table name in case the engine can't read with HoodieParquetRealtimeInputFormat (e.g., Presto). This change skips the redundant bare-name RT sync when skip_ro_suffix is set, and logs a WARN naming the table and stating which config wins. The real-time view remains available at

    _rt. * Address review comments: fix bare-table-name wording, remove now-redundant comment, strengthen regression test to a second sync round * Drop the second-round explanation comment per review feedback (cherry picked from commit 8ee40b4fb54fff3beb924134af2367438913062a) --- .../org/apache/hudi/hive/HiveSyncTool.java | 9 +++- .../apache/hudi/hive/TestHiveSyncTool.java | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 35bced6497e64..4abfa5b2ec8cf 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -211,7 +211,14 @@ protected void doSync() { syncHoodieTable(snapshotTableName, true, false); // sync origin table for MOR if (config.getBoolean(META_SYNC_SNAPSHOT_WITH_TABLE_NAME)) { - syncHoodieTable(tableName, true, false); + if (config.getBoolean(HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE)) { + log.warn("{}=true claims the bare table name '{}' for the read-optimized view; " + + "ignoring {} for this table (the real-time view remains registered as '{}').", + HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key(), tableId(databaseName, tableName), + META_SYNC_SNAPSHOT_WITH_TABLE_NAME.key(), snapshotTableName); + } else { + syncHoodieTable(tableName, true, false); + } } } break; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index fc4b47a4b10d7..988b94a5a56bb 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -99,6 +99,7 @@ import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; +import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_ENABLED; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_THREADS; @@ -119,6 +120,7 @@ import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_SNAPSHOT_WITH_TABLE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TABLE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TOUCH_PARTITIONS_ENABLED; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -1557,6 +1559,51 @@ public void testSyncMergeOnReadWithStrategy(String syncMode, HoodieSyncTableStra } } + @Test + void testSkipRoSuffixTakesPrecedenceOverSnapshotWithTableName() throws Exception { + // skip_ro_suffix explicitly claims the bare table name for the RO view; the now-default-true + // sync_snapshot_with_table_name must not be allowed to flip it to RT. + hiveSyncProps.setProperty(HIVE_SYNC_TABLE_STRATEGY.key(), HoodieSyncTableStrategy.ALL.name()); + hiveSyncProps.setProperty(HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key(), "true"); + hiveSyncProps.setProperty(META_SYNC_SNAPSHOT_WITH_TABLE_NAME.key(), "true"); + hiveSyncProps.setProperty(HIVE_SYNC_AS_DATA_SOURCE_TABLE.key(), "true"); + + String instantTime = "100"; + String deltaCommitTime = "101"; + HiveTestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, true); + + reInitHiveSyncClient(); + reSyncHiveTable(); + + // a second sync round with a new commit reproduces the flip; a single round does not + ZonedDateTime dateTime = ZonedDateTime.now().plusDays(6); + String commitTime2 = "102"; + String deltaCommitTime2 = "103"; + HiveTestUtil.addMORPartitions(1, true, false, true, dateTime, commitTime2, deltaCommitTime2); + reInitHiveSyncClient(); + reSyncHiveTable(); + + String snapshotTableName = HiveTestUtil.TABLE_NAME + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; + String roSuffixTableName = HiveTestUtil.TABLE_NAME + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE; + + // the bare table name must stay registered as the read-optimized view + StorageDescriptor bareTableSd = hiveClient.getMetastoreStorageDescriptor(HiveTestUtil.TABLE_NAME); + assertEquals(HoodieParquetInputFormat.class.getName(), bareTableSd.getInputFormat(), + "Bare table name should remain the RO view (skip_ro_suffix=true) despite sync_snapshot_with_table_name=true"); + assertEquals("true", bareTableSd.getSerdeInfo().getParameters().get(ConfigUtils.IS_QUERY_AS_RO_TABLE), + "Bare table name should still be marked as the RO table"); + + // the real-time/snapshot view remains available, and only there + assertTrue(hiveClient.tableExists(snapshotTableName), "Table " + snapshotTableName + " should exist after sync completes"); + StorageDescriptor snapshotTableSd = hiveClient.getMetastoreStorageDescriptor(snapshotTableName); + assertEquals(HoodieParquetRealtimeInputFormat.class.getName(), snapshotTableSd.getInputFormat(), + "Table " + snapshotTableName + " should use the realtime input format"); + + // no separate "_ro" table should be created when skip_ro_suffix is set + assertFalse(hiveClient.tableExists(roSuffixTableName), + "Table " + roSuffixTableName + " should not exist when skip_ro_suffix is set"); + } + @ParameterizedTest @EnumSource(value = HoodieSyncTableStrategy.class, names = {"RO", "RT"}) public void testSyncMergeOnReadWithStrategyWhenTableExist(HoodieSyncTableStrategy strategy) throws Exception { From 2599498dd364b2cec236d435ed597d86cd291b57 Mon Sep 17 00:00:00 2001 From: Davis-Zhang-Onehouse <169106455+Davis-Zhang-Onehouse@users.noreply.github.com> Date: Fri, 31 Jul 2026 02:25:39 -0700 Subject: [PATCH 224/255] fix(utilities): include metadata table index-init instants in the record index validation snapshot (#19395) HoodieMetadataTableValidator reads the metadata table with a time-travel snapshot anchored to the data table's latest completed commit. On a table version 6 metadata table the partition-initialization deltacommits are that same data instant with a three-digit suffix appended (010 for FILES, 011 for RECORD_INDEX), and Hudi compares instants as strings, so those derived instants sort after the bare data instant and fall outside the snapshot. When the data table has no commit newer than the initialization instant, every record index file slice is filtered out, the index reads back empty, and the validator reports 100% of the data table's keys as missing from it. This is permanent for a table that has stopped receiving writes; cleans and rollbacks do not help because getWriteTimeline() only whitelists commit, deltacommit, compaction, logcompaction and replacecommit. Table version 8 and above are unaffected: generateUniqueInstantTime derives the init instants from SOLO_COMMIT_TIMESTAMP, which sorts below every data instant. Advance the instant used for the metadata table read by one millisecond. That is strictly greater than any , which shares the whole 17-character prefix, while remaining a valid yyyyMMddHHmmssSSS instant - the time travel option runs the value through formatQueryInstant, which rejects anything else. The data table side is unchanged, so both sides stay pinned to the same data instant. Fixes both validateRecordIndexContent and validateRecordIndexCount; the latter carried the same defect, masked only because the content check shadows it. (cherry picked from commit 133a25515676ba31a2a258147dc8d563fb280de1) --- .../HoodieMetadataTableValidator.java | 42 +++++- .../TestHoodieMetadataTableValidator.java | 122 ++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java index d6876bea6e695..8c5558930247b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java @@ -61,6 +61,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.InstantComparison; +import org.apache.hudi.common.table.timeline.TimelineUtils; import org.apache.hudi.common.table.view.FileSystemViewManager; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; @@ -111,11 +112,13 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -209,6 +212,10 @@ public class HoodieMetadataTableValidator implements Serializable { private static final long serialVersionUID = 1L; + // Advance the metadata table query instant by this much so that instants derived from a data table + // instant, which carry a three-digit suffix, fall inside the queried window. See #metadataTableInstantFor. + private static final long METADATA_INSTANT_LOOKAHEAD_MS = 1; + // Spark context private transient JavaSparkContext jsc; // config @@ -1236,7 +1243,7 @@ private void validateRecordIndexCount(HoodieSparkEngineContext sparkEngineContex .select(RECORD_KEY_METADATA_FIELD) .count(); long countKeyFromRecordIndex = sparkEngineContext.getSqlContext().read().format("hudi") - .option(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT().key(),latestCompletedCommit) + .option(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT().key(), metadataTableInstantFor(latestCompletedCommit)) .load(getMetadataTableBasePath(basePath)) .select("key") .filter("type = 5") @@ -1338,6 +1345,37 @@ private void validateRecordIndexContent(HoodieSparkEngineContext sparkEngineCont } } + /** + * Returns the instant to query the metadata table with, so that the snapshot reflects the data + * table as of {@code dataTableInstant}. + *

    + * Metadata table instants derived from a data table instant carry a three-digit numeric suffix: + * partition initialization appends 010 and up (see + * {@code HoodieTableMetadataUtil#createIndexInitTimestamp}), and metadata-table-internal + * compaction, clean, restore, indexing, log compaction and rollback append 001 to 006. Hudi + * compares instants as strings, so every one of those derived instants sorts AFTER the bare data + * instant, and a snapshot taken as of the data instant itself excludes them - leaving, for + * instance, the record index unreadable until the data table receives another commit. + *

    + * The bound is therefore advanced by a single millisecond. That is strictly greater than any + * {@code } (which shares the whole 17-character prefix and so compares + * lower), while still being a valid {@code yyyyMMddHHmmssSSS} instant - the time travel option + * rejects anything else, see {@code HoodieSqlCommonUtils#formatQueryInstant}. Instants that are + * not timestamps (legacy or test instants such as "100") are returned unchanged; they have no + * metadata table counterpart to include. + */ + @VisibleForTesting + static String metadataTableInstantFor(String dataTableInstant) { + try { + Date dataTableInstantDate = TimelineUtils.parseDateFromInstantTime(dataTableInstant); + return TimelineUtils.formatDate(new Date(dataTableInstantDate.getTime() + METADATA_INSTANT_LOOKAHEAD_MS)); + } catch (ParseException e) { + log.warn("Cannot parse instant {} as a timestamp; querying the metadata table as of it verbatim", + dataTableInstant); + return dataTableInstant; + } + } + @VisibleForTesting JavaPairRDD> getRecordLocationsFromFSBasedListing(HoodieSparkEngineContext sparkEngineContext, String basePath, @@ -1358,7 +1396,7 @@ JavaPairRDD> getRecordLocationsFromRLI(HoodieSparkE String basePath, String latestCompletedCommit) { return sparkEngineContext.getSqlContext().read().format("hudi") - .option(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT().key(), latestCompletedCommit) + .option(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT().key(), metadataTableInstantFor(latestCompletedCommit)) .load(getMetadataTableBasePath(basePath)) .filter("type = 5") .select(functions.col("key"), diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java index 24ee11c8ba1ec..49a8f115c9517 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormatWriter; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; @@ -45,6 +46,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.InstantComparison; import org.apache.hudi.common.table.timeline.TimeGenerator; import org.apache.hudi.common.table.timeline.TimeGenerators; import org.apache.hudi.common.table.timeline.TimelineUtils; @@ -57,9 +59,14 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieValidationException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataWriter; +import org.apache.hudi.metadata.MetadataPartitionType; +import org.apache.hudi.metadata.SparkMetadataWriterFactory; import org.apache.hudi.stats.HoodieColumnRangeMetadata; import org.apache.hudi.stats.ValueMetadata; import org.apache.hudi.storage.HoodieStorage; @@ -1677,4 +1684,119 @@ private void mockPartitionWithFiles(List partition1, HoodieStorage stora when(storage.listFiles(new StoragePath(basePath + "/" + partition))).thenReturn(Collections.singletonList(storagePathInfo)); } } + + /** + * On a table version 6 metadata table the partition-initialization deltacommits are the data instant + * they were derived from with a three-digit suffix appended (010 for FILES, 011 for RECORD_INDEX) - + * see {@code HoodieBackedTableMetadataWriterTableVersionSix#createIndexInitTimestamp}. Those instants + * sort AFTER the bare data instant under Hudi's lexicographic instant comparison, so a metadata-table + * snapshot taken as of the data table's latest completed commit must still include them. When the data + * table has no commit after the metadata table was initialized, failing to do so makes the record index + * read back empty and every data-table key is reported as missing from it. + *

    + * Table version 8 and above are unaffected: {@code HoodieBackedTableMetadataWriter#generateUniqueInstantTime} + * derives the init instants from {@code SOLO_COMMIT_TIMESTAMP}, which sorts below every data instant. + */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRecordIndexValidationWhenMdtInitializedAtLatestDataCommit(boolean validateContent) throws Exception { + Map writeOptions = new HashMap<>(); + writeOptions.put(DataSourceWriteOptions.TABLE_NAME().key(), "test_table"); + writeOptions.put("hoodie.table.name", "test_table"); + writeOptions.put(DataSourceWriteOptions.TABLE_TYPE().key(), "COPY_ON_WRITE"); + writeOptions.put(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "_row_key"); + writeOptions.put(DataSourceWriteOptions.PRECOMBINE_FIELD().key(), "timestamp"); + writeOptions.put(DataSourceWriteOptions.OPERATION().key(), WriteOperationType.BULK_INSERT.value()); + // Leave the metadata table off for the write; it is bootstrapped out of band below so that the + // data table's latest completed commit stays the instant the init instants are derived from. + writeOptions.put(HoodieMetadataConfig.ENABLE.key(), "false"); + // Table version 6 is the one whose metadata table initialization instants carry the suffix. + writeOptions.put(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), "6"); + + makeInsertDf("000", 50).write().format("hudi").options(writeOptions) + .mode(SaveMode.Overwrite) + .save(basePath); + + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .forTable("test_table") + .withWriteTableVersion(6) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withEnableGlobalRecordLevelIndex(true) + .withRecordIndexFileGroupCount(1, 1) + .build()) + .build(); + HoodieTableMetaClient metaClientBeforeInit = HoodieTableMetaClient.builder() + .setBasePath(basePath).setConf(HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration())).build(); + assertEquals(HoodieTableVersion.SIX, metaClientBeforeInit.getTableConfig().getTableVersion(), + "the suffixed initialization instants only exist on table version 6"); + + // Creating the writer initializes the FILES and RECORD_INDEX partitions from the filesystem + // without adding a commit to the data table. Go through the factory so the table-version-6 writer + // is selected, exactly as production does. + try (HoodieTableMetadataWriter ignored = SparkMetadataWriterFactory.create( + HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()), writeConfig, context, + Option.empty(), metaClientBeforeInit.getTableConfig())) { + // constructing the writer performs the initialization + } + + HoodieTableMetaClient dataMetaClient = HoodieTableMetaClient.reload(metaClientBeforeInit); + assertTrue(dataMetaClient.getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX), + "record index should be registered on the data table"); + String latestDataCommit = dataMetaClient.getActiveTimeline().getCommitsAndCompactionTimeline() + .filterCompletedInstants().lastInstant().get().requestedTime(); + HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder() + .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(basePath)) + .setConf(HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration())).build(); + List mdtInstants = mdtMetaClient.getActiveTimeline().filterCompletedInstants() + .getInstantsAsStream().map(HoodieInstant::requestedTime).collect(Collectors.toList()); + assertTrue( + mdtInstants.stream().allMatch(instant -> instant.startsWith(latestDataCommit) + && instant.length() > latestDataCommit.length()), + "expected every metadata instant to be a suffixed extension of " + latestDataCommit + + " but got " + mdtInstants); + + HoodieMetadataTableValidator.Config config = new HoodieMetadataTableValidator.Config(); + config.basePath = "file:" + basePath; + config.validateLatestFileSlices = true; + // validateRecordIndexContent shadows validateRecordIndexCount, so toggling it exercises both paths. + config.validateRecordIndexContent = validateContent; + config.validateRecordIndexCount = true; + config.ignoreFailed = true; + + HoodieMetadataTableValidator validator = new HoodieMetadataTableValidator(jsc, config); + // Assert on the record index validation directly: doMetadataTableValidation() reports any + // non-HoodieValidationException as a successful run, so run() alone cannot distinguish a genuine + // pass from the read blowing up. + assertDoesNotThrow(() -> validator.validateRecordIndex(new HoodieSparkEngineContext(jsc), dataMetaClient), + "record index validation should pass against an intact record index"); + assertTrue(validator.run(), "validation should succeed against an intact record index"); + assertFalse(validator.hasValidationFailure(), () -> "unexpected validation failures: " + + validator.getThrowables()); + } + + @Test + public void testMetadataTableInstantForIncludesEveryDerivedInstant() { + String dataTableInstant = "20231012054834279"; + + String queryInstant = HoodieMetadataTableValidator.metadataTableInstantFor(dataTableInstant); + + assertEquals("20231012054834280", queryInstant, "the bound should advance the instant by one millisecond"); + // 010-013 are appended when a metadata table partition is initialized, 001-006 by the + // metadata-table-internal operations; all of them must fall inside the queried window. + for (String suffix : new String[] {"001", "002", "003", "004", "005", "006", "010", "011", "012", "013"}) { + assertTrue( + InstantComparison.compareTimestamps(dataTableInstant + suffix, InstantComparison.LESSER_THAN_OR_EQUALS, queryInstant), + () -> "metadata instant " + dataTableInstant + suffix + " should be included by " + queryInstant); + } + // ... while the next data table instant stays outside it. + assertTrue(InstantComparison.compareTimestamps("20231012054834281", InstantComparison.GREATER_THAN, queryInstant), + "a later data table instant should not be included"); + } + + @Test + public void testMetadataTableInstantForLeavesNonTimestampInstantUnchanged() { + assertEquals("100", HoodieMetadataTableValidator.metadataTableInstantFor("100")); + } } From bc1923558f43a2d99239fce1262c49aead405668 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sat, 1 Aug 2026 14:48:50 +0800 Subject: [PATCH 225/255] fix(trino): read uncompacted MDT HFILE log deltas and guard index pruning (#19298) * fix(trino): read uncompacted MDT HFILE log deltas and guard index pruning Fixes apache/hudi#19279: with the connector default hudi.metadata-enabled=true, any query on a table whose metadata table (MDT) has uncompacted delta commits failed with 'Native HFILE log files are not supported by the Hudi Trino connector'. Since MDT compaction fires only every hoodie.metadata.compact.max.delta.commits (default 10) deltas, actively written tables spend most of their life in that state; the E2E testcontainers pipeline had to set hudi.metadata-enabled=false to pass. - HudiTrinoIOFactory.getFileFormatUtils now returns hudi-common's HFileUtils for HFILE. The native-log read path (HoodieNativeLogFileReader) needs only its readFooter, and HFileUtils is hadoop-free (it decodes via hudi-io's native HFile reader), so it is safe in the Trino runtime -- unlike parquet, whose ParquetUtils lives in the hadoop-excluded hudi-hadoop-common and needs the Trino-native HudiTrinoParquetFileFormatUtils. The reader-factory side (newHFileFileReader) was already implemented; it is how compacted MDT base HFiles are read today. MDT reads never touch HudiTrinoReaderContext. - Guard the partition-stats pruning call in HudiBackgroundSplitLoader.getPartitionInfos: any pruning failure now logs and proceeds unpruned, mirroring the metastore fallback the MDT partition listing directly above it already has. Index pruning is an optimization and must never fail the query. The per-file-slice indexes (column-stats, record-level, secondary) already swallow failures via their future+catch pattern. - Guard the MDT-backed file-system-view load in HudiSnapshotDirectoryLister: on failure, close the partial view and fall back to a direct-listing view built over an MDT-disabled table metadata. This was the second unguarded synchronous MDT read. Tests: UncompactedMetadataHudiTablesInitializer writes a partitioned COW table with an ENABLED metadata table (column-stats + partition-stats indexes) and compact.max.delta.commits=100 -- the inverse of the zip fixtures' compact.max.delta.commits=1, whose always-compacted MDTs are why nothing caught this -- leaving native HFILE log deltas in the files/column_stats/ partition_stats MDT partitions. MDT HFILE writing is native (hudi-io HFileWriterImpl); the 'requires hbase' note in older initializers is stale. TestHudiUncompactedMetadataTable covers MDT-backed listing, MDT-on vs MDT-off result equality, partition-stats pruning (the exact crash path) and column-stats file skipping over the uncompacted stats. The E2E follow-up (separate branch trino-e2e-testcontainers): drop the hudi.metadata-enabled=false workaround from the Trino catalog so the pipeline runs MDT-on. * test(trino): use inherited computeScalar(Session, String) The private computeScalarWith helper duplicated AbstractTestQueryFramework's existing computeScalar(Session, String). Drop it and call the inherited one. * fix(trino): keep shared table metadata open and pin the MDT-failure fallbacks Review follow-ups on the HudiSnapshotDirectoryLister fallback and its tests: - Drop fileSystemView.close() in the catch: closing the failed view would also close the shared HoodieTableMetadata behind lazyTableMetadata, which the split loader and the index supports still read through. Nothing else closes that instance today. - Build the direct-listing fallback via HoodieTableFileSystemView.fileListingBasedFileSystemView with the same completed-commits timeline HudiUtil.getFileSystemView uses, instead of the MDT-disabled NativeTableMetadataFactory detour. - Pin both MDT-failure fallbacks: the initializer writes a corrupted twin table (hudi_corrupted_mdt_pt_cow) whose MDT log files are damaged before upload, and new tests assert the direct-listing fallback returns complete rows and prunePartitionsSafely degrades to a full scan. The damage targets the HFile trailer fields inside the block content and leaves the log block framing intact: framing damage is classified as a HoodieCorruptBlock and skipped without an exception, and bytes near EOF are only trailer padding (hudi-io's 4096-byte trailer keeps its magic/protobuf at the start). - Assert exact split counts in the partition-stats pruning test: equal to a metastore-pruned part_col scan of p1 and strictly below the full scan. Not a literal count, since file groups per partition depend on the write client's small-file packing for this runtime-written table. - Reuse AbstractMergerHudiTablesInitializer.HUDI_META_COLUMNS (now package-private) instead of redeclaring the five _hoodie_* columns. * test(trino): bootstrap the fixture MDT after data so index definitions get source fields The exact split-count assertion added in the previous commit exposed that partition-stats pruning never ran on the fixture table: the CI log shows 'Partition stats index definition is missing or has no source fields defined' and both partitions being listed. Root cause: the MDT bootstrapped during commit 1 on an empty table, so ColumnStatsIndexer.postInitialization registered the column_stats index definition with an empty source-field list, and HoodieJavaWriteClient.updateColumnsToIndexWithColStats is a no-op (unlike the Spark client, which refreshes the definition on each commit), so the definition stayed empty forever and the connector's canApply() silently rejected the col-stats and partition-stats indexes. The pruning and file-skipping tests -- clean and corrupted -- were passing vacuously. Write commit 1 with the MDT disabled and open a second write client with the MDT (and stats indexes) enabled for commits 2 and 3: the bootstrap then sees commit 1's files and registers the definition with real source fields. Commits 2 and 3 still leave uncompacted HFILE log deltas in the files/ column_stats/partition_stats MDT partitions, so the original regression coverage is unchanged, and the corrupted twin still fails every MDT read through its damaged log deltas. * test(trino): verify corrupted-MDT reads fail and pin col-stats skipping by split count Address review comments on #19298: - The initializer now opens a metadata-table reader over the corrupted twin after damaging the log files and asserts getAllPartitionPaths throws, so the fallback tests cannot pass vacuously if the fixed trailer offsets ever miss (checkState(corrupted > 0) only proved bytes were flipped). - testColumnStatsFileSkippingOverUncompactedStats raises the col-stats wait timeout to 10s (shouldSkipFileSlice keeps the file on any failure or after the 1s default timeout, so the old value-only assertion passed without the deltas ever being read) and asserts the filtered scan uses strictly fewer splits than the full scan. * test(trino): correct the zip-fixture rationale and harden the corruption fixture Review follow-ups on the uncompacted-MDT fixture: - The javadoc claimed the zip fixtures escaped this bug because their MDTs are 'always freshly compacted'. Wrong on both counts: only some fixtures compact after every commit, and hudi_trips_cow_v8 (queried MDT-on in TestHudiSmokeTest) ships a 14KB uncompacted delta in the MDT files partition. The real difference is the log FORMAT: those fixtures predate the native-log write path, so their deltas are #HUDI# block-format logs carrying HFILE_DATA_BLOCKs (block type ordinal 4 in the header), which the connector already read through its HFile content reader. Only whole-file native HFILE log files (*.log.hfile, selected by filename in FSUtils.isNativeLogFile) go through HoodieNativeLogFileReader and the previously unimplemented getFileFormatUtils(HFILE) path. Reworded the initializer and test javadocs (and the PR description) accordingly, and corrected the corruption helper's comment, which described block framing these native files do not have. - corruptMetadataLogFiles now requires at least one corrupted log file in EVERY MDT partition directory, not one overall, so small deltas in one partition cannot silently survive; the walk also excludes anything not directly inside a partition directory. - verifyMetadataTableUnreadable catches HoodieMetadataException only -- the wrapper BaseTableMetadata puts around a genuine read failure -- so the bare IllegalArgumentException thrown for a never-initialized MDT fails initialization instead of masquerading as corruption. - The MDT-off-first-commit workaround now cites HUDI-8801 (the Java client's no-op updateColumnsToIndexWithColStats) so it can be dropped when that lands. * fix(trino): declare checked exceptions on verifyMetadataTableUnreadable Narrowing the catch to HoodieMetadataException left the checked exceptions previously swallowed by catch (Exception) unreported: IOException declared by getAllPartitionPaths and Exception from the implicit close() of the try-with-resources. Declare throws Exception so anything unexpected fails table initialization loudly instead of being swallowed; the caller already declares it. * test(trino): drop the stale compact.max.delta.commits=1 parenthetical The createWriteClient comment still repeated the debunked claim that the zip fixtures always compact after every commit; the class javadoc carries the corrected native-log-format rationale, so the file now tells one story. (cherry picked from commit 6894f0c292630b5b125b46277afc9fa5987c01f6) --- .../plugin/hudi/io/HudiTrinoIOFactory.java | 9 + .../query/HudiSnapshotDirectoryLister.java | 35 +- .../hudi/split/HudiBackgroundSplitLoader.java | 24 +- .../TestHudiUncompactedMetadataTable.java | 172 ++++++++ .../AbstractMergerHudiTablesInitializer.java | 2 +- ...ompactedMetadataHudiTablesInitializer.java | 405 ++++++++++++++++++ 6 files changed, 639 insertions(+), 8 deletions(-) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUncompactedMetadataTable.java create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/UncompactedMetadataHudiTablesInitializer.java diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java index ea9c837d47c31..7660f6d51d526 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java @@ -17,6 +17,7 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.FileFormatUtils; +import org.apache.hudi.common.util.HFileUtils; import org.apache.hudi.core.io.storage.HoodieFileReaderFactory; import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; import org.apache.hudi.core.io.storage.HoodieIOFactory; @@ -47,8 +48,16 @@ public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType re public FileFormatUtils getFileFormatUtils(HoodieFileFormat fileFormat) { if (fileFormat == HoodieFileFormat.PARQUET) { + // Parquet needs a Trino-native implementation: hudi's own ParquetUtils lives in + // hudi-hadoop-common, which is excluded from the Trino runtime. return new HudiTrinoParquetFileFormatUtils(); } + if (fileFormat == HoodieFileFormat.HFILE) { + // hudi-common's HFileUtils is hadoop-free (it decodes via hudi-io's native HFile + // reader), so it is safe in the Trino runtime. This is what lets the connector read + // uncompacted metadata-table deltas, which are native HFILE log files. + return new HFileUtils(); + } throw new UnsupportedOperationException( "Native " + fileFormat + " log files are not supported by the Hudi Trino connector"); } diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java index b9a794df871f1..cb90dbd1e3392 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java @@ -21,12 +21,13 @@ import io.trino.plugin.hudi.query.index.IndexSupportFactory; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.SchemaTableName; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.HoodieTimer; -import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.common.util.Lazy; +import org.apache.hudi.metadata.HoodieTableMetadata; import java.util.List; import java.util.Optional; @@ -54,9 +55,22 @@ public HudiSnapshotDirectoryLister( this.lazyFileSystemView = Lazy.lazily(() -> { HoodieTimer timer = HoodieTimer.start(); HoodieTableMetaClient metaClient = tableHandle.getMetaClient(); - HoodieTableFileSystemView fileSystemView = getFileSystemView(lazyTableMetadata.get(), metaClient); - if (enableMetadataTable) { - fileSystemView.loadAllPartitions(); + HoodieTableFileSystemView fileSystemView; + try { + fileSystemView = getFileSystemView(lazyTableMetadata.get(), metaClient); + if (enableMetadataTable) { + fileSystemView.loadAllPartitions(); + } + } + catch (Exception e) { + // A failure here is a metadata-table read failure (the metastore/table itself is + // fine), so fall back to direct file listing instead of failing the query. The + // failed view is deliberately not closed: closing it would also close the shared + // HoodieTableMetadata behind lazyTableMetadata, which the split loader and the + // index supports still read through. + log.error(e, "Failed to load the file system view of table %s via the metadata table, falling back to direct file listing", + schemaTableName); + fileSystemView = createDirectListingFileSystemView(metaClient); } log.info("Created file system view of table %s in %s ms", schemaTableName, timer.endTimer()); return fileSystemView; @@ -67,6 +81,19 @@ public HudiSnapshotDirectoryLister( IndexSupportFactory.createIndexSupport(tableHandle, lazyMetaClient, lazyTableMetadata, tableHandle.getRegularPredicates(), session) : Optional.empty(); } + /** + * Builds a file system view that lists files directly from storage, bypassing the metadata table. + * Used as the fallback when the metadata-table-backed view cannot be loaded (e.g. an MDT read + * failure); it lists lazily per partition, so no {@code loadAllPartitions()} here. + */ + private static HoodieTableFileSystemView createDirectListingFileSystemView(HoodieTableMetaClient metaClient) + { + return HoodieTableFileSystemView.fileListingBasedFileSystemView( + new HoodieLocalEngineContext(metaClient.getStorage().getConf()), + metaClient, + metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); + } + @Override public List listStatus(HudiPartitionInfo partitionInfo, boolean useIndex) { diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java index b48e790f5d012..e33bf772dc1fb 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java @@ -214,9 +214,9 @@ private Deque getPartitionInfos(boolean useIndex) List allPartitions = new ArrayList<>(metadataPartitions.keySet()); - List effectivePartitions = Optional.ofNullable(useIndex && partitionIndexSupportOpt.isPresent() - ? partitionIndexSupportOpt.get().prunePartitions(allPartitions).orElse(null) - : null).orElse(allPartitions); + List effectivePartitions = useIndex && partitionIndexSupportOpt.isPresent() + ? prunePartitionsSafely(allPartitions) + : allPartitions; Map finalMetadataPartitions = metadataPartitions; List hiveHudiPartitionInfos = effectivePartitions.stream() @@ -227,6 +227,24 @@ private Deque getPartitionInfos(boolean useIndex) return new ConcurrentLinkedDeque<>(hiveHudiPartitionInfos); } + /** + * Applies partition-stats index pruning, treating any failure as "no pruning". The pruning read + * goes through the metadata table and can fail for reasons unrelated to the query itself; index + * pruning is an optimization and must never fail the query, mirroring the metastore fallback of + * the metadata-table partition listing above. + */ + private List prunePartitionsSafely(List allPartitions) + { + try { + return partitionIndexSupportOpt.get().prunePartitions(allPartitions).orElse(allPartitions); + } + catch (Exception e) { + log.error(e, "Failed to prune partitions via partition stats index on table %s.%s, proceeding without partition pruning", + tableHandle.getSchemaName(), tableHandle.getTableName()); + return allPartitions; + } + } + private HiveHudiPartitionInfo buildHiveHudiPartitionInfo(HudiTableHandle tableHandle, String partitionName, Partition partition) { return new HiveHudiPartitionInfo( diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUncompactedMetadataTable.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUncompactedMetadataTable.java new file mode 100644 index 0000000000000..cf14fbf3496db --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiUncompactedMetadataTable.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import io.trino.Session; +import io.trino.plugin.hudi.testing.UncompactedMetadataHudiTablesInitializer; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.MaterializedResult; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static io.trino.plugin.hudi.testing.UncompactedMetadataHudiTablesInitializer.CORRUPTED_TABLE_NAME; +import static io.trino.plugin.hudi.testing.UncompactedMetadataHudiTablesInitializer.TABLE_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Regression tests for apache/hudi#19279: queries on tables whose metadata table (MDT) has + * UNCOMPACTED delta commits. Those deltas are native HFILE log files, which the connector previously + * rejected ("Native HFILE log files are not supported..."), failing every query through the unguarded + * partition-stats pruning path. The table written by {@link UncompactedMetadataHudiTablesInitializer} + * keeps its MDT deliberately uncompacted and its deltas are whole-file native HFILE logs (the zip + * fixtures predate that write path; their block-format deltas were always readable), so the queries + * below only succeed if the connector reads native HFILE log files in the MDT's + * {@code files}/{@code column_stats}/{@code partition_stats} partitions. + *

    + * The initializer also writes {@code CORRUPTED_TABLE_NAME}, an identical table whose MDT log files + * are corrupted so every MDT read throws; queries on it pin the fallbacks (direct file listing, + * unpruned split generation) instead of only the clean-read path. + */ +public class TestHudiUncompactedMetadataTable + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return HudiQueryRunner.builder() + .setDataLoader(new UncompactedMetadataHudiTablesInitializer()) + .build(); + } + + @Test + public void testSnapshotReadWithUncompactedMetadataTable() + { + // MDT-backed file listing must read the files partition's HFILE log deltas + assertQuery( + mdtEnabled(), + "SELECT id, name, price FROM " + TABLE_NAME + " ORDER BY id", + "VALUES ('k1', 'k1_c3', CAST(15 AS BIGINT)), ('k2', 'k2_c1', 1000), ('k3', 'k3_c2', 20), ('k4', 'k4_c2', 2000)"); + assertThat(computeScalar(mdtEnabled(), "SELECT count(*) FROM " + TABLE_NAME)).isEqualTo(4L); + } + + @Test + public void testResultsMatchWithMetadataTableDisabled() + { + String query = "SELECT id, name, price, part_col FROM " + TABLE_NAME + " ORDER BY id"; + MaterializedResult withMdt = getQueryRunner().execute(mdtEnabled(), query); + MaterializedResult withoutMdt = getQueryRunner().execute(mdtDisabled(), query); + assertThat(withMdt.getMaterializedRows()).isEqualTo(withoutMdt.getMaterializedRows()); + } + + @Test + public void testPartitionStatsIndexPruningOverUncompactedStats() + { + // The exact crash from the issue: partition-stats pruning reads the partition_stats MDT + // partition, whose deltas are uncompacted HFILE log files. Partition p2 holds prices + // [1000, 2000], so `price < 100` lets the index prune it entirely. + MaterializedResult pruned = getQueryRunner().execute(partitionStatsPruningOnly(), + "SELECT id, price FROM " + TABLE_NAME + " WHERE price < 100"); + assertThat(pruned.getMaterializedRows()).hasSize(2); + + // Index pruning must scan exactly p1's file groups: the same split count as a + // metastore-pruned scan of p1, strictly fewer than the full scan. Split counts are + // compared instead of hardcoded because the number of file groups per partition depends + // on the write client's small-file packing. + int fullScanSplits = totalSplits(mdtEnabled(), "SELECT id, price FROM " + TABLE_NAME); + int p1ScanSplits = totalSplits(mdtEnabled(), "SELECT id, price FROM " + TABLE_NAME + " WHERE part_col = 'p1'"); + assertThat(p1ScanSplits).isLessThan(fullScanSplits); + assertThat(pruned.getStatementStats().get().getTotalSplits()).isEqualTo(p1ScanSplits); + } + + @Test + public void testReadFallsBackToDirectListingOnUnreadableMetadataTable() + { + // The corrupted twin table's MDT log deltas cannot be decoded, so the MDT-backed + // file-system-view load throws; HudiSnapshotDirectoryLister must fall back to listing + // files directly from storage and still return complete, correct rows. + assertQuery( + mdtEnabled(), + "SELECT id, name, price FROM " + CORRUPTED_TABLE_NAME + " ORDER BY id", + "VALUES ('k1', 'k1_c3', CAST(15 AS BIGINT)), ('k2', 'k2_c1', 1000), ('k3', 'k3_c2', 20), ('k4', 'k4_c2', 2000)"); + } + + @Test + public void testPartitionStatsPruningFallsBackUnprunedOnUnreadableMetadataTable() + { + // Same pruning setup as above, but the partition_stats read throws on the corrupted + // table: prunePartitionsSafely must degrade to "no pruning", so the scan covers the + // same splits as a full scan instead of failing the query. + MaterializedResult unpruned = getQueryRunner().execute(partitionStatsPruningOnly(), + "SELECT id, price FROM " + CORRUPTED_TABLE_NAME + " WHERE price < 100"); + assertThat(unpruned.getMaterializedRows()).hasSize(2); + + int fullScanSplits = totalSplits(mdtDisabled(), "SELECT id, price FROM " + CORRUPTED_TABLE_NAME); + assertThat(unpruned.getStatementStats().get().getTotalSplits()).isEqualTo(fullScanSplits); + } + + @Test + public void testColumnStatsFileSkippingOverUncompactedStats() + { + // Column-stats file skipping reads the column_stats MDT partition's HFILE log deltas. + // The wait timeout must be raised above its 1s default (matching the col-stats tests in + // TestHudiSmokeTest): shouldSkipFileSlice keeps the file on any failure or timeout, so + // with the default a broken col-stats read would still return correct rows and a + // value-only assertion would pass without the deltas ever being read. + Session session = SessionBuilder.from(getSession()) + .withMdtEnabled(true) + .withColStatsIndexEnabled(true) + .withRecordLevelIndexEnabled(false) + .withSecondaryIndexEnabled(false) + .withPartitionStatsIndexEnabled(false) + .withColumnStatsTimeout("10s") + .build(); + MaterializedResult skipped = getQueryRunner().execute(session, + "SELECT id, price FROM " + TABLE_NAME + " WHERE price = 15"); + assertThat(skipped.getMaterializedRows()).hasSize(1); + assertThat(skipped.getMaterializedRows().get(0).getFields()).containsExactly("k1", 15L); + + // File skipping must drop at least p2's file group (its prices [1000, 2000] exclude 15), + // so the filtered scan uses strictly fewer splits than the unfiltered full scan + int fullScanSplits = totalSplits(mdtEnabled(), "SELECT id, price FROM " + TABLE_NAME); + assertThat(skipped.getStatementStats().get().getTotalSplits()).isLessThan(fullScanSplits); + } + + private Session mdtEnabled() + { + return SessionBuilder.from(getSession()).withMdtEnabled(true).build(); + } + + private Session mdtDisabled() + { + return SessionBuilder.from(getSession()).withMdtEnabled(false).build(); + } + + /** MDT on with only the partition-stats index enabled, isolating partition pruning. */ + private Session partitionStatsPruningOnly() + { + return SessionBuilder.from(getSession()) + .withMdtEnabled(true) + .withColStatsIndexEnabled(false) + .withRecordLevelIndexEnabled(false) + .withSecondaryIndexEnabled(false) + .withPartitionStatsIndexEnabled(true) + .build(); + } + + private int totalSplits(Session session, String query) + { + return getQueryRunner().execute(session, query).getStatementStats().get().getTotalSplits(); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java index 2aa39bf1cffa8..2eac0a195f87d 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java @@ -89,7 +89,7 @@ public abstract class AbstractMergerHudiTablesInitializer private static final String PARTITION_PATH = ""; /** Hudi metadata columns, prepended to every table's data columns in the metastore definition. */ - private static final List HUDI_META_COLUMNS = ImmutableList.of( + static final List HUDI_META_COLUMNS = ImmutableList.of( new Column("_hoodie_commit_time", HIVE_STRING, Optional.empty(), Map.of()), new Column("_hoodie_commit_seqno", HIVE_STRING, Optional.empty(), Map.of()), new Column("_hoodie_record_key", HIVE_STRING, Optional.empty(), Map.of()), diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/UncompactedMetadataHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/UncompactedMetadataHudiTablesInitializer.java new file mode 100644 index 0000000000000..0c174ca54c3f0 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/UncompactedMetadataHudiTablesInitializer.java @@ -0,0 +1,405 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Column; +import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.Partition; +import io.trino.metastore.PartitionStatistics; +import io.trino.metastore.PartitionWithStatistics; +import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.StorageFormat; +import io.trino.metastore.Table; +import io.trino.plugin.hudi.HudiConnector; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.QueryRunner; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.common.HoodieJavaEngineContext; +import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.util.HoodieStorageUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; +import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.HivePartitionManager.extractPartitionValues; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static java.nio.file.Files.createTempDirectory; + +/** + * Creates a partitioned COW table at test runtime with an ENABLED, UNCOMPACTED metadata table (MDT): + * {@code hoodie.metadata.compact.max.delta.commits} is set high and several commits are written, + * leaving the MDT's {@code files}/{@code column_stats}/{@code partition_stats} partitions with + * NATIVE HFILE log files that the connector must read at query time (issue apache/hudi#19279): + * {@code *.log.hfile} deltas where the whole file is an HFile, as current writers produce. The zip + * fixtures cannot cover this even where their MDTs are uncompacted (e.g. {@code hudi_trips_cow_v8}): + * they predate the native-log write path, so their MDT deltas are {@code #HUDI#} block-format logs + * carrying HFILE_DATA_BLOCKs, which the connector already read through its HFile content reader; + * only whole-file native HFILE logs hit the previously unimplemented + * {@code getFileFormatUtils(HFILE)} path. + *

    + * Note: MDT writing here is fully native -- HFILE base files and log files are written via hudi-io's + * pure-Java {@code HFileWriterImpl}, so no hbase dependency is involved (the "requires hbase" note in + * older initializers is stale). + *

    + * Data layout (partitions {@code part_col=p1} / {@code part_col=p2}, hive-style paths so the MDT + * partition listing and the metastore agree on names): + * 

    + * commit 1 (insert, MDT off): k1(p1, price 10,  ts 100), k2(p2, price 1000, ts 100)
+ * commit 2 (insert, MDT on):  k3(p1, price 20,  ts 200), k4(p2, price 2000, ts 200)
+ * commit 3 (upsert, MDT on):  k1(p1, price 15,  ts 300)
+ *
    + * The MDT is enabled only from commit 2 so its bootstrap sees existing data; a bootstrap over an + * empty table would register the col-stats index definition with no source fields, permanently + * disabling stats-based pruning in the connector (see {@code writeTable}). + * Final rows: k1=15, k2=1000, k3=20, k4=2000. Partition p1 holds prices [15, 20] and p2 holds + * [1000, 2000], so a predicate like {@code price < 100} lets the partition-stats index prune p2. + *

    + * A second, identical table {@link #CORRUPTED_TABLE_NAME} is written whose MDT log files are + * corrupted in place before upload: every MDT read of it throws, pinning the connector's + * fallbacks (direct file listing in {@code HudiSnapshotDirectoryLister}, unpruned split + * generation in {@code HudiBackgroundSplitLoader}) instead of only the clean-read path. + */ +public class UncompactedMetadataHudiTablesInitializer + implements HudiTablesInitializer +{ + public static final String TABLE_NAME = "hudi_uncompacted_mdt_pt_cow"; + public static final String CORRUPTED_TABLE_NAME = "hudi_corrupted_mdt_pt_cow"; + + private static final String RECORD_KEY_FIELD = "id"; + private static final String PARTITION_FIELD = "part_col"; + private static final String ORDERING_FIELD = "ts"; + private static final List PARTITION_PATHS = ImmutableList.of(PARTITION_FIELD + "=p1", PARTITION_FIELD + "=p2"); + + private static final List DATA_COLUMNS = ImmutableList.builder() + .addAll(AbstractMergerHudiTablesInitializer.HUDI_META_COLUMNS) + .add(new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of())) + .add(new Column("name", HIVE_STRING, Optional.empty(), Map.of())) + .add(new Column("price", HIVE_LONG, Optional.empty(), Map.of())) + .add(new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())) + .build(); + + private static final List PARTITION_COLUMNS = + ImmutableList.of(new Column(PARTITION_FIELD, HIVE_STRING, Optional.empty(), Map.of())); + + @Override + public void initializeTables(QueryRunner queryRunner, Location externalLocation, String schemaName) + throws Exception + { + TrinoFileSystem fileSystem = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(TrinoFileSystemFactory.class) + .create(ConnectorIdentity.ofUser("test")); + HiveMetastore metastore = ((HudiConnector) queryRunner.getCoordinator().getConnector("hudi")).getInjector() + .getInstance(HiveMetastoreFactory.class) + .createMetastore(Optional.empty()); + + java.nio.file.Path tempDir = createTempDirectory("uncompacted-mdt"); + try { + for (String tableName : ImmutableList.of(TABLE_NAME, CORRUPTED_TABLE_NAME)) { + java.nio.file.Path tempTableDir = tempDir.resolve(tableName); + writeTable(new Path(tempTableDir.toUri()), tableName); + if (tableName.equals(CORRUPTED_TABLE_NAME)) { + corruptMetadataLogFiles(tempTableDir); + verifyMetadataTableUnreadable(new Path(tempTableDir.toUri())); + } + Location tableLocation = externalLocation.appendPath(tableName); + ResourceHudiTablesInitializer.copyDir(tempTableDir, fileSystem, tableLocation); + + metastore.createTable(createTableDefinition(schemaName, tableName, tableLocation), PrincipalPrivileges.NO_PRIVILEGES); + metastore.addPartitions(schemaName, tableName, createPartitions(schemaName, tableName, tableLocation)); + } + } + finally { + deleteRecursively(tempDir, ALLOW_INSECURE); + } + } + + /** + * Corrupts the MDT log files so that any metadata-table read of the table throws. The deltas + * here are NATIVE HFILE log files (the whole file is an HFile): hudi-io HFiles end with a + * fixed 4096-byte trailer whose magic and protobuf fields sit at the trailer's START, followed + * by padding, so the corrupted window is placed ~4KB before EOF to land on those fields -- + * flipping bytes near EOF would only hit padding and reads would still succeed. + * {@link #verifyMetadataTableUnreadable} then proves the corruption took. Every MDT partition + * directory must end up with at least one corrupted log file, so that none of the read paths + * (files listing, col-stats skipping, partition-stats pruning) can see a clean partition. + */ + private static void corruptMetadataLogFiles(java.nio.file.Path tableDir) + throws IOException + { + java.nio.file.Path metadataDir = tableDir.resolve(".hoodie").resolve("metadata"); + List logFiles; + try (Stream walk = Files.walk(metadataDir)) { + logFiles = walk + .filter(Files::isRegularFile) + .filter(file -> file.getFileName().toString().contains(".log.")) + // Only files directly inside an MDT partition directory, not timeline or + // marker leftovers under the MDT's own .hoodie + .filter(file -> metadataDir.equals(file.getParent().getParent())) + .collect(toImmutableList()); + } + Set partitionsWithLogFiles = new HashSet<>(); + Set partitionsCorrupted = new HashSet<>(); + for (java.nio.file.Path logFile : logFiles) { + partitionsWithLogFiles.add(logFile.getParent()); + byte[] bytes = Files.readAllBytes(logFile); + int start = bytes.length - 4160; + int end = bytes.length - 3648; + if (start < 64) { + // Too small to even hold an HFile trailer: a record-less file-group bootstrap + // marker that no read decodes. The per-partition check below still requires a + // corrupted data-bearing file next to it. + continue; + } + for (int i = start; i < end; i++) { + bytes[i] ^= 0x5A; + } + Files.write(logFile, bytes); + partitionsCorrupted.add(logFile.getParent()); + } + checkState(!partitionsWithLogFiles.isEmpty(), "No MDT log files found under %s", metadataDir); + Set untouched = new HashSet<>(partitionsWithLogFiles); + untouched.removeAll(partitionsCorrupted); + checkState(untouched.isEmpty(), + "No MDT log file corrupted in partition(s) %s; reads of those partitions would still succeed and their fallback tests would pass vacuously", untouched); + } + + /** + * Proves the corruption is effective, not just that bytes were flipped: a metadata-table read + * of the corrupted table must throw. This guards the fallback tests against the fixed trailer + * offsets in {@link #corruptMetadataLogFiles} ever missing (e.g. after an HFile layout change), + * in which case those tests would pass vacuously against a clean MDT read. + */ + private static void verifyMetadataTableUnreadable(Path tablePath) + throws Exception + { + HadoopStorageConfiguration storageConf = new HadoopStorageConfiguration(new Configuration()); + List partitions; + try (HoodieTableMetadata metadata = new HoodieBackedTableMetadata( + new HoodieJavaEngineContext(storageConf), + HoodieStorageUtils.getStorage(tablePath.toString(), storageConf), + HoodieMetadataConfig.newBuilder().enable(true).build(), + tablePath.toString())) { + // The files partition backs getAllPartitionPaths; its log delta is corrupted like all others + partitions = metadata.getAllPartitionPaths(); + } + catch (HoodieMetadataException expected) { + // Only the wrapper BaseTableMetadata puts around a genuine read failure counts: the + // bare IllegalArgumentException it throws for a never-initialized MDT must NOT pass, + // or this guard would go quiet if the fixture's MDT bootstrap ever stopped happening + return; + } + throw new IllegalStateException( + "Metadata table of " + CORRUPTED_TABLE_NAME + " is still readable (partitions: " + partitions + + "); corruptMetadataLogFiles no longer lands on the HFile trailer fields"); + } + + private static void writeTable(Path tablePath, String tableName) + { + Schema schema = createAvroSchema(); + initTable(tablePath, tableName); + + // Commit 1 runs with the MDT off so the MDT bootstraps AFTER data exists. Index + // definitions get their source fields from ColumnStatsIndexer.postInitialization, which + // registers an EMPTY field list when the bootstrap sees no records, and + // HoodieJavaWriteClient.updateColumnsToIndexWithColStats is a no-op (HUDI-8801; the + // Spark client refreshes the definition on each commit), so a definition registered over + // an empty table keeps empty source fields forever and the connector's canApply() then + // rejects the col-stats/partition-stats indexes, silently disabling pruning. Once + // HUDI-8801 fixes the Java client, this two-client split can collapse back to one. + try (HoodieJavaWriteClient writeClient = createWriteClient(schema, tablePath, tableName, false)) { + String firstCommit = writeClient.startCommit(); + List firstStatuses = writeClient.insert(ImmutableList.of( + record(schema, "k1", "k1_c1", 10L, 100L, PARTITION_PATHS.get(0)), + record(schema, "k2", "k2_c1", 1000L, 100L, PARTITION_PATHS.get(1))), firstCommit); + writeClient.commit(firstCommit, firstStatuses); + } + + try (HoodieJavaWriteClient writeClient = createWriteClient(schema, tablePath, tableName, true)) { + String secondCommit = writeClient.startCommit(); + List secondStatuses = writeClient.insert(ImmutableList.of( + record(schema, "k3", "k3_c2", 20L, 200L, PARTITION_PATHS.get(0)), + record(schema, "k4", "k4_c2", 2000L, 200L, PARTITION_PATHS.get(1))), secondCommit); + writeClient.commit(secondCommit, secondStatuses); + + String thirdCommit = writeClient.startCommit(); + List thirdStatuses = writeClient.upsert(ImmutableList.of( + record(schema, "k1", "k1_c3", 15L, 300L, PARTITION_PATHS.get(0))), thirdCommit); + writeClient.commit(thirdCommit, thirdStatuses); + } + } + + private static void initTable(Path tablePath, String tableName) + { + try { + HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE) + .setTableName(tableName) + .setTimelineLayoutVersion(1) + .setBootstrapIndexClass(NoOpBootstrapIndex.class.getName()) + .setPayloadClassName(HoodieAvroPayload.class.getName()) + .setRecordKeyFields(RECORD_KEY_FIELD) + .setPartitionFields(PARTITION_FIELD) + .setHiveStylePartitioningEnable(true) + .setOrderingFields(ORDERING_FIELD) + .initTable(new HadoopStorageConfiguration(new Configuration()), tablePath.toString()); + } + catch (IOException e) { + throw new RuntimeException("Could not init table " + tableName, e); + } + } + + private static HoodieJavaWriteClient createWriteClient(Schema schema, Path tablePath, String tableName, boolean metadataEnabled) + { + Configuration conf = new Configuration(); + HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder() + .withPath(tablePath.toString()) + .withSchema(schema.toString()) + .withParallelism(2, 2) + .withDeleteParallelism(2) + .forTable(tableName) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .withMaxNumDeltaCommitsBeforeCompaction(100) + .build()) + .withEmbeddedTimelineServerEnabled(false) + .withMarkersType(MarkerType.DIRECT.name()) + // The whole point of this initializer: an ENABLED metadata table with stats + // indexes whose compaction never fires within this test, so its partitions keep + // native HFILE log deltas. MDT HFILE writing is native (hudi-io HFileWriterImpl) + // -- no hbase involved. metadataEnabled is false for the first commit only; see + // writeTable. + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(metadataEnabled) + .withMetadataIndexColumnStats(metadataEnabled) + .withMetadataIndexPartitionStats(metadataEnabled) + .withMaxNumDeltaCommitsBeforeCompaction(100) + .build()) + .build(); + return new HoodieJavaWriteClient<>(new HoodieJavaEngineContext(new HadoopStorageConfiguration(conf)), cfg); + } + + private static HoodieRecord record(Schema schema, String key, String name, long price, long ts, String partitionPath) + { + GenericRecord record = new GenericData.Record(schema); + record.put(RECORD_KEY_FIELD, key); + record.put("name", name); + record.put("price", price); + record.put(ORDERING_FIELD, ts); + // Keep the partition column in the data files like Spark-written tables do; Trino reads its + // value from the metastore partition, not from parquet + record.put(PARTITION_FIELD, partitionPath.substring(partitionPath.indexOf('=') + 1)); + HoodieKey hoodieKey = new HoodieKey(key, partitionPath); + return new HoodieAvroRecord<>(hoodieKey, new HoodieAvroPayload(Option.of(record)), null); + } + + private static Schema createAvroSchema() + { + List fields = ImmutableList.of( + new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), + new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("price", Schema.create(Schema.Type.LONG)), + new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG)), + new Schema.Field(PARTITION_FIELD, Schema.create(Schema.Type.STRING))); + return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); + } + + private static Table createTableDefinition(String schemaName, String tableName, Location location) + { + return Table.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setTableType(EXTERNAL_TABLE.name()) + .setOwner(Optional.of("public")) + .setDataColumns(DATA_COLUMNS) + .setPartitionColumns(PARTITION_COLUMNS) + .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) + .withStorage(storageBuilder -> storageBuilder + .setStorageFormat(storageFormat()) + .setLocation(location.toString())) + .build(); + } + + private static List createPartitions(String schemaName, String tableName, Location tableLocation) + { + List partitions = new ArrayList<>(); + for (String partitionName : PARTITION_PATHS) { + // Hive-style partition paths, so the partition NAME and the relative PATH coincide + Partition partition = Partition.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setValues(extractPartitionValues(partitionName)) + .withStorage(storageBuilder -> storageBuilder + .setStorageFormat(storageFormat()) + .setLocation(tableLocation.appendPath(partitionName).toString())) + .setColumns(DATA_COLUMNS) + .build(); + partitions.add(new PartitionWithStatistics(partition, partitionName, PartitionStatistics.empty())); + } + return partitions; + } + + private static StorageFormat storageFormat() + { + return StorageFormat.create( + PARQUET_HIVE_SERDE_CLASS, + HUDI_PARQUET_INPUT_FORMAT, + MAPRED_PARQUET_OUTPUT_FORMAT_CLASS); + } +} From c31aa56e6a76146210b487f1d034255c36546b00 Mon Sep 17 00:00:00 2001 From: Joy <33287603+Joy-2000@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:33:07 +0800 Subject: [PATCH 226/255] fix(flink): rethrow StreamWriteOperatorCoordinator start() failures (#19432) * fix(flink): rethrow coordinator start failures Avoid using context.failJob when StreamWriteOperatorCoordinator startup fails, because global failover can keep the half-initialized coordinator instance alive without invoking start() again. --------- Co-authored-by: jiangyu84 (cherry picked from commit 07a6635e499038467016ec9b830a224769551930) --- .../hudi/sink/StreamWriteOperatorCoordinator.java | 10 +++++++--- .../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java index 686aaaa9509ca..c0cc62bc66409 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java @@ -253,9 +253,13 @@ public void start() throws Exception { initClientIds(conf); } restoreEvents(Long.MAX_VALUE); - } catch (Throwable throwable) { - log.error("Failed to start operator coordinator.", throwable); - context.failJob(throwable); + } catch (Exception exception) { + // Rethrow instead of context.failJob(): failJob triggers an in-graph global failover that + // keeps this same coordinator instance alive without calling start() again, leaving the + // half-initialized null fields (executor, writeClient, metaClient ...) to be reused and NPE + // later. Rethrowing surfaces the failure as a JobMaster start failure so the partially + // initialized instance is discarded rather than kept serving. + throw new HoodieException("Failed to start operator coordinator.", exception); } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java index 69087122a2e81..e7cffbab75af5 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java @@ -752,7 +752,7 @@ public void testWriteMultiWriterInvolved(WriteConcurrencyMode writeConcurrencyMo protected void validateNonBlockingConcurrencyControlConditions() { assertThrows( - IllegalArgumentException.class, + HoodieException.class, () -> preparePipeline(conf), "Non-blocking concurrency control requires the MOR table with simple bucket index"); } From 793f22c628194aaff4ee2724ee1c9e0681ed437c Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Sun, 2 Aug 2026 21:03:12 +0700 Subject: [PATCH 227/255] fix(trino): remap pushed-down predicate columns to physical file ordinals (#19456) * fix(trino): remap pushed-down predicate columns to physical file ordinals * fix(trino): dedupe pushed-down predicate domains per projected field * test(trino): cover stale metastore ordinals and merge-path pushdown --------- Co-authored-by: Vova Kolmakov (cherry picked from commit 64afac03908bba1f32df05982aee388d9011b714) --- .../plugin/hudi/HudiPageSourceProvider.java | 189 +++++- ...stHudiConnectorParquetColumnNamesTest.java | 41 +- .../hudi/TestHudiMorMergeModeSemantics.java | 21 + .../hudi/TestHudiPageSourceProviderTest.java | 558 ++++++++++++++++++ .../AbstractMergerHudiTablesInitializer.java | 12 +- ...ittedMetaColumnsHudiTablesInitializer.java | 157 +++++ 6 files changed, 949 insertions(+), 29 deletions(-) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedMetaColumnsHudiTablesInitializer.java diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index 7ffca56717eff..2803f8ae231a7 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -34,6 +34,7 @@ import io.trino.parquet.reader.RowGroupInfo; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hive.HiveColumnProjectionInfo; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hudi.file.HudiBaseFile; import io.trino.plugin.hudi.reader.HudiTrinoReaderContext; @@ -48,6 +49,7 @@ import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.EmptyPageSource; +import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.TupleDomain; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; @@ -70,11 +72,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.stream.Collectors; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; @@ -383,9 +388,15 @@ static ConnectorPageSource createPageSource( // When not using columnNames, physical indexes are used and there could be cases when the physical index in HiveColumnHandle is different from the fileSchema of the // parquet files. This could happen when schema evolution happened. In such a case, we will need to remap the column indices in the HiveColumnHandles. + // The projection and the predicate resolve the same names against the same file, so the name-to-position + // map is built once per split and shared: one lookup table means the two can never disagree about which + // physical column a name denotes, and a wide table pays for the lower-casing pass only once. + // HiveColumnHandle names are in lower case, case-insensitive + Optional> physicalIndexMap = Optional.empty(); if (!useColumnNames) { - // HiveColumnHandle names are in lower case, case-insensitive - columns = remapColumnIndicesToPhysical(fileSchema, columns, false); + Map indexMap = buildPhysicalIndexMap(fileSchema, false); + columns = remapColumnIndicesToPhysical(fileSchema, columns, indexMap, false); + physicalIndexMap = Optional.of(indexMap); } Optional message = getParquetMessageType(columns, useColumnNames, fileSchema); @@ -397,7 +408,7 @@ static ConnectorPageSource createPageSource( TupleDomain parquetTupleDomain = options.isIgnoreStatistics() || !enablePredicatePushDown ? TupleDomain.all() - : getParquetTupleDomain(descriptorsByPath, getCombinedPredicate(hudiSplit, dynamicFilter), fileSchema, useColumnNames); + : getParquetTupleDomain(descriptorsByPath, getPushdownPredicate(hudiSplit, dynamicFilter, physicalIndexMap), fileSchema, useColumnNames); TupleDomainParquetPredicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath, timeZone); @@ -482,41 +493,165 @@ public static List remapColumnIndicesToPhysical( boolean caseSensitive) { // Create a map from column name to its physical index in the fileSchema. - Map physicalIndexMap = new HashMap<>(); - List fileFields = fileSchema.getFields(); - for (int i = 0; i < fileFields.size(); i++) { - Type field = fileFields.get(i); - String fieldName = field.getName(); - String mapKey = caseSensitive ? fieldName : fieldName.toLowerCase(Locale.ROOT); - physicalIndexMap.put(mapKey, i); - } + return remapColumnIndicesToPhysical(fileSchema, requestedColumns, buildPhysicalIndexMap(fileSchema, caseSensitive), caseSensitive); + } + /** + * {@link #remapColumnIndicesToPhysical(MessageType, List, boolean)} against a {@code physicalIndexMap} the caller + * already built, so a split that remaps both its projection and its predicate builds the map once. + * {@code caseSensitive} must be the one the map was built with, or the lookups miss. + */ + private static List remapColumnIndicesToPhysical( + MessageType fileSchema, + List requestedColumns, + Map physicalIndexMap, + boolean caseSensitive) + { // Iterate through the columns requested by Trino IN ORDER. List remappedHandles = new ArrayList<>(requestedColumns.size()); for (HiveColumnHandle originalHandle : requestedColumns) { - String requestedName = originalHandle.getBaseColumnName(); - - // Determine the key to use for looking up the physical index - String lookupKey = caseSensitive ? requestedName : requestedName.toLowerCase(Locale.ROOT); - // Find the physical index from the file schema map constructed from fileSchema. A column the file // does not carry keeps an index one past the last field, which the parquet reader null-fills. - Integer physicalIndex = physicalIndexMap.get(lookupKey); - - HiveColumnHandle remappedHandle = new HiveColumnHandle( - requestedName, - physicalIndex == null ? fileFields.size() : physicalIndex, - originalHandle.getBaseHiveType(), - originalHandle.getType(), - originalHandle.getHiveColumnProjectionInfo(), - originalHandle.getColumnType(), - originalHandle.getComment()); - remappedHandles.add(remappedHandle); + Integer physicalIndex = physicalIndexMap.get(normalizeColumnName(originalHandle.getBaseColumnName(), caseSensitive)); + remappedHandles.add(withPhysicalIndex(originalHandle, physicalIndex == null ? fileSchema.getFieldCount() : physicalIndex)); } return remappedHandles; } + /** + * Rebuilds a predicate's column handles on physical file ordinals, the predicate-side counterpart of + * {@link #remapColumnIndicesToPhysical}. With {@code hudi.parquet.use-column-names=false}, + * {@code ParquetPageSourceFactory.getParquetTupleDomain} resolves a predicate column positionally, as + * {@code fileSchema.getType(handle.getBaseHiveColumnIndex())}, but the handles reaching it carry METASTORE + * ordinals: a metastore that omits the Hudi meta fields (hive sync with {@code omit_metadata_fields=true}) + * shifts every data column, and so does reordering or dropping one. Left unremapped, the domain attaches to + * whichever column happens to sit at the stale ordinal and row groups are pruned on that column's statistics, + * silently dropping rows. + *

    + * Resolution is by name, so the predicate ends up bound to exactly the column the projection reads - which is + * the property that matters, since the two are compared against each other. It is not a defence against a + * column being dropped and re-added under full schema evolution: name-based binding will match the new column + * to the old one, exactly as the projection remap and the whole {@code use-column-names=true} mode already do. + *

    + * A column the file does not carry is dropped from the predicate rather than mapped to the + * {@link #remapColumnIndicesToPhysical} sentinel, which every absent column would share. Dropping loses row + * group pruning but never a row: the static half of the predicate is handed back to the engine in full as + * {@code HudiMetadata.applyFilter}'s remaining filter, and the dynamic half is by construction redundant with + * the join above the scan. It is also what already happens today for a predicate column the query does not + * read, since {@code descriptorsByPath} is derived from the projection and + * {@code getParquetTupleDomain} skips any column it cannot resolve. + * + * @param fileSchema The MessageType representing the physical schema of the Parquet file. + * @param predicate The predicate to push down, keyed on handles carrying metastore ordinals. + * @param caseSensitive Whether the lookup between Trino column names (from handles) and Parquet field names (from fileSchema) should be case-sensitive. + * @return The same domains, keyed on handles carrying physical ordinals, minus the columns the file lacks. + */ + @VisibleForTesting + public static TupleDomain remapPredicateColumnIndicesToPhysical( + MessageType fileSchema, + TupleDomain predicate, + boolean caseSensitive) + { + return remapPredicateColumnIndicesToPhysical(predicate, buildPhysicalIndexMap(fileSchema, caseSensitive), caseSensitive); + } + + /** + * {@link #remapPredicateColumnIndicesToPhysical(MessageType, TupleDomain, boolean)} against a + * {@code physicalIndexMap} the caller already built, so a split that remaps both its projection and its predicate + * builds the map once. {@code caseSensitive} must be the one the map was built with, or the lookups miss. + */ + private static TupleDomain remapPredicateColumnIndicesToPhysical( + TupleDomain predicate, + Map physicalIndexMap, + boolean caseSensitive) + { + if (predicate.isAll() || predicate.isNone()) { + return predicate; + } + + Set>> pushedFields = new HashSet<>(); + Map remappedDomains = new LinkedHashMap<>(); + for (Map.Entry entry : predicate.getDomains().orElseThrow().entrySet()) { + Integer physicalIndex = physicalIndexMap.get(normalizeColumnName(entry.getKey().getBaseColumnName(), caseSensitive)); + if (physicalIndex == null) { + continue; + } + // Deduplicate on what getParquetTupleDomain resolves the handle to rather than on the handle itself: two + // handles whose names differ only by case resolve to one file column while staying unequal to each other, + // and pushing both down would hand it the same ColumnDescriptor twice, which it rejects by failing the + // split. The base column alone is too coarse a key, because a dereference handle carries its subfield + // path into the descriptor, so the projection is part of the key and two projections of one base column + // both survive. Neither collision is reachable today - the case one needs a metastore holding two such + // columns, which Hive's name normalisation rules out, and trino-parquet lower-cases every field name when + // it builds the MessageType from the footer anyway - but keeping only the first domain is a cheap + // guarantee that the read can never be made worse than pushing nothing down. + if (pushedFields.add(Map.entry(physicalIndex, entry.getKey().getHiveColumnProjectionInfo()))) { + remappedDomains.put(withPhysicalIndex(entry.getKey(), physicalIndex), entry.getValue()); + } + } + return TupleDomain.withColumnDomains(remappedDomains); + } + + /** + * Maps each of {@code fileSchema}'s top-level field names to its physical position. + */ + private static Map buildPhysicalIndexMap(MessageType fileSchema, boolean caseSensitive) + { + Map physicalIndexMap = new HashMap<>(); + List fileFields = fileSchema.getFields(); + for (int i = 0; i < fileFields.size(); i++) { + physicalIndexMap.put(normalizeColumnName(fileFields.get(i).getName(), caseSensitive), i); + } + return physicalIndexMap; + } + + private static String normalizeColumnName(String columnName, boolean caseSensitive) + { + return caseSensitive ? columnName : columnName.toLowerCase(Locale.ROOT); + } + + /** + * Copies {@code handle} with its base column index replaced by a physical one, every other attribute carried + * over unchanged. Note that the constructor's fourth argument is the BASE type: it differs from + * {@code getType()} only for a dereference handle, whose {@code getType()} is the projected subfield's type + * rather than the column's, and {@code createParquetPageSource} reads the base type throughout. + *

    + * Copying a dereference handle's projection across matters: {@code createParquetPageSource} branches on + * {@code isBaseColumn()} and dereferences through {@code getHiveColumnProjectionInfo}, and it reads the base + * column's stored {@code baseType} on the way. The connector never produces such a handle today, because + * {@code HudiMetadata} does not implement {@code applyProjection}. + */ + private static HiveColumnHandle withPhysicalIndex(HiveColumnHandle handle, int physicalIndex) + { + return new HiveColumnHandle( + handle.getBaseColumnName(), + physicalIndex, + handle.getBaseHiveType(), + handle.getBaseType(), + handle.getHiveColumnProjectionInfo(), + handle.getColumnType(), + handle.getComment()); + } + + /** + * Resolves the predicate handed to {@code ParquetPageSourceFactory.getParquetTupleDomain}. Only the + * positional mode needs the handles rebuilt; when columns are resolved by name the metastore ordinals are + * never read, which is exactly when {@code physicalIndexMap} is empty. Being handed the very map the projection + * was remapped with is what makes it structural, rather than a convention, that the two agree about which + * physical column a name denotes. + */ + private static TupleDomain getPushdownPredicate( + HudiSplit hudiSplit, + DynamicFilter dynamicFilter, + Optional> physicalIndexMap) + { + TupleDomain combinedPredicate = getCombinedPredicate(hudiSplit, dynamicFilter); + return physicalIndexMap + .map(indexMap -> remapPredicateColumnIndicesToPhysical(combinedPredicate, indexMap, false)) + .orElse(combinedPredicate); + } + private static TupleDomain getCombinedPredicate(HudiSplit hudiSplit, DynamicFilter dynamicFilter) { // Combine static and dynamic predicates diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java index 7e938ba3aed1e..fda63c8b47140 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConnectorParquetColumnNamesTest.java @@ -13,8 +13,16 @@ */ package io.trino.plugin.hudi; +import io.trino.plugin.hudi.testing.CompositeHudiTablesInitializer; +import io.trino.plugin.hudi.testing.OmittedMetaColumnsHudiTablesInitializer; import io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer; import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; + +import static io.trino.plugin.hudi.testing.OmittedMetaColumnsHudiTablesInitializer.LATE_COLUMN; +import static io.trino.plugin.hudi.testing.OmittedMetaColumnsHudiTablesInitializer.SHADOWED_COLUMN; +import static io.trino.plugin.hudi.testing.OmittedMetaColumnsHudiTablesInitializer.THRESHOLD; +import static io.trino.plugin.hudi.testing.OmittedMetaColumnsHudiTablesInitializer.expectedRowsAboveThreshold; public class TestHudiConnectorParquetColumnNamesTest extends TestHudiSmokeTest @@ -25,7 +33,38 @@ protected QueryRunner createQueryRunner() { return HudiQueryRunner.builder() .addConnectorProperty("hudi.parquet.use-column-names", "false") - .setDataLoader(new ResourceHudiTablesInitializer()) + // The resource tables all register the Hudi meta columns in the metastore, so their metastore + // ordinals already equal their physical ones and nothing here resolves a stale ordinal. The + // second fixture is the one whose metastore omits them. + .setDataLoader(new CompositeHudiTablesInitializer( + new ResourceHudiTablesInitializer(), + new OmittedMetaColumnsHudiTablesInitializer())) .build(); } + + /** + * apache/hudi#19387: with columns resolved positionally, a predicate handle carrying a metastore ordinal has to + * be rebuilt on the file's physical ordinal before it is pushed into the parquet reader. Left unremapped, the + * domain lands on whichever column physically sits at that ordinal -- here {@code shadowed_value}, whose values + * are far below the threshold -- and the only row group is pruned, so the query returns nothing at all. + *

    + * {@code shadowed_value} has to stay in the SELECT list: {@code descriptorsByPath} is derived from the + * projection, so a domain resolving to a column the query does not read finds no descriptor and is dropped + * instead of being misapplied. Narrowing this projection turns the test green against the unfixed code. + *

    + * Only the {@code hudi.parquet.use-column-names=false} suite runs this; the name-based parent resolves the + * predicate by name and was never affected. + */ + @Test + public void testPredicateOnColumnWithStaleMetastoreOrdinal() + { + assertQuery( + "SELECT key, %s, %s FROM %s WHERE %s > %s ORDER BY key".formatted( + SHADOWED_COLUMN, + LATE_COLUMN, + OmittedMetaColumnsHudiTablesInitializer.TABLE_NAME, + LATE_COLUMN, + THRESHOLD), + expectedRowsAboveThreshold()); + } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java index ba21f2b34a679..a3e62db9de0d8 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java @@ -132,4 +132,25 @@ public void testNarrowProjectionMergesCorrectly() "SELECT key, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " ORDER BY key", "VALUES ('k1', CAST(11 AS BIGINT)), ('k4', 40), ('k5', 50), ('k6', 60)"); } + + @Test + public void testPredicateIsNotPushedIntoTheBaseReadOfAMergedSplit() + { + // HudiPageSourceProvider enables parquet predicate pushdown for base-file-only splits ONLY; the merge + // path builds its base page source with it off. That invariant is load-bearing and nothing else pins it: + // pruning happens on the BASE row group's statistics, before the log records the merge needs are seen. + // + // 65 is above every base value (10..60) and below the obsolete k6 update (66), which loses on event time + // and must not surface. With pushdown enabled on this path the whole row group is pruned, the base side + // comes back empty, and every log record is then emitted as an insert -- so k6 appears with 66 and the + // merge is silently skipped. Note the naive shape does NOT discriminate: for a row whose log record wins + // and carries the full record, dropping the base row still yields the right answer. + assertQueryReturnsEmptyResult( + "SELECT key, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " WHERE value > 65"); + // Anchor: the same predicate one step lower does return the rows it should, so the query above is empty + // because of the merge, not because nothing ever matches. + assertQuery( + "SELECT key, value FROM " + EventTimeDeletesHudiTablesInitializer.RT_TABLE_NAME + " WHERE value > 45 ORDER BY key", + "VALUES ('k5', CAST(50 AS BIGINT)), ('k6', 60)"); + } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java index 4bce0ba0cfc55..a224f2ffb6c2f 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiPageSourceProviderTest.java @@ -13,28 +13,137 @@ */ package io.trino.plugin.hudi; +import io.trino.filesystem.local.LocalInputFile; import io.trino.metastore.HiveType; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hive.HiveColumnProjectionInfo; +import io.trino.plugin.hive.parquet.ParquetReaderConfig; +import io.trino.plugin.hudi.file.HudiBaseFile; +import io.trino.spi.SplitWeight; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.predicate.ValueSet; import io.trino.spi.type.BigintType; +import io.trino.spi.type.RowType; import io.trino.spi.type.Type; +import io.trino.testing.MaterializedResult; +import io.trino.testing.TestingConnectorSession; +import org.apache.parquet.conf.PlainParquetConfiguration; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.io.LocalOutputFile; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Types; +import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; +import static io.trino.plugin.hudi.HudiPageSourceProvider.createPageSource; import static io.trino.plugin.hudi.HudiPageSourceProvider.remapColumnIndicesToPhysical; +import static io.trino.plugin.hudi.HudiPageSourceProvider.remapPredicateColumnIndicesToPhysical; import static io.trino.spi.type.DoubleType.DOUBLE; import static io.trino.spi.type.IntegerType.INTEGER; import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.testing.MaterializedResult.materializeSourceDataStream; +import static java.lang.Integer.parseInt; +import static org.apache.hudi.common.model.HoodieRecord.HOODIE_META_COLUMNS; import static org.apache.parquet.schema.Type.Repetition.OPTIONAL; import static org.assertj.core.api.Assertions.assertThat; +/** + * Covers {@link HudiPageSourceProvider}'s column remapping from both sides: the index arithmetic of + * {@code remapColumnIndicesToPhysical} and {@code remapPredicateColumnIndicesToPhysical} on their own, and reads of + * a real base file through {@code createPageSource}, which are what prove a remapped predicate actually reaches the + * parquet reader and prunes the column it was written for. + *

    + * The base file the reading tests share has a physical column order the metastore does not: it carries the five + * {@code _hoodie_*} meta columns, which hive sync with + * {@code hoodie.datasource.hive_sync.omit_metadata_fields=true} leaves out, so every data column's metastore ordinal + * is five below its physical position. With {@code hudi.parquet.use-column-names=false} the parquet page source + * resolves columns positionally, so a predicate whose handle still carries the metastore ordinal lands on whichever + * column physically sits there and row groups get pruned on that column's statistics. The fixture makes that + * observable: {@link #PREDICATE_COLUMN} grows with the row index while every other data column stays in 0..9, so a + * domain meant for it but applied to any other column excludes every row group and the read returns nothing. + *

    + * Note that the shadowed column has to be part of the PROJECTION for the damage to appear: {@code + * descriptorsByPath} is derived from the projection, so a domain resolving to a column the query does not read + * finds no descriptor and is discarded instead. Do not "simplify" the projections below to the predicate column + * alone - that turns those tests green against the unfixed code. + */ class TestHudiPageSourceProviderTest { + private static final int DATA_COLUMN_COUNT = 10; + /** The column the predicate is on: physically at 12, but numbered 7 by a metastore without the meta columns. */ + private static final String PREDICATE_COLUMN = "c7"; + /** The column physically sitting at {@code c7}'s stale ordinal, and therefore the one that shadows it. */ + private static final String SHADOWED_COLUMN = "c2"; + private static final int ROW_COUNT = 1000; + private static final long THRESHOLD = 900; + private static final int MATCHING_ROW_COUNT = (int) (ROW_COUNT - THRESHOLD - 1); + + @TempDir + static Path tempDir; + + private static Path baseFile; + + @BeforeAll + static void writeBaseFile() + throws IOException + { + MessageType schema = hudiFileSchema(DATA_COLUMN_COUNT); + baseFile = tempDir.resolve("base_file.parquet"); + SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(baseFile)) + .withType(schema) + .withConf(new PlainParquetConfiguration()) + .withRowGroupSize(1024L) + .withPageSize(512) + .build()) { + for (int row = 0; row < ROW_COUNT; row++) { + Group group = groupFactory.newGroup(); + for (String metaColumn : HOODIE_META_COLUMNS) { + group.append(metaColumn, metaColumn + "_" + row); + } + for (int column = 0; column < DATA_COLUMN_COUNT; column++) { + String columnName = "c" + column; + group.append(columnName, columnName.equals(PREDICATE_COLUMN) ? row : row % 10); + } + writer.write(group); + } + } + // The writer flushes a row group whenever the buffered size is over withRowGroupSize, checked every + // parquet.page.size.row.check.min records (100 by default), which is what actually splits this file. + // Assert the outcome rather than the knobs: with a single row group there would be nothing to prune, + // and every test below would pass without proving anything. + assertThat(rowGroupCount(baseFile)).as("row groups written").isGreaterThan(1); + } + @Test public void testRemapSimpleMatchCaseInsensitive() { @@ -193,6 +302,455 @@ public void testRemapColumnNotFound() assertHandle(remapped.get(1), "col_x", fileSchema.getFieldCount(), HiveType.HIVE_STRING, VARCHAR); } + @Test + public void testRemapPredicateStaleMetastoreOrdinals() + { + // Physical Schema: the five Hudi meta columns, then [c0, c1, c2] + MessageType fileSchema = hudiFileSchema(3); + + // A metastore synced with omit_metadata_fields=true carries no meta columns, so "c2" is numbered 2 + // while it physically sits at 7, and "c0" is numbered 0 while it physically sits at 5. + HiveColumnHandle staleC2 = createDummyHandle("c2", 2, HiveType.HIVE_INT, INTEGER); + HiveColumnHandle staleC0 = createDummyHandle("c0", 0, HiveType.HIVE_INT, INTEGER); + Domain c2Domain = Domain.create(ValueSet.ofRanges(Range.greaterThan(INTEGER, 900L)), false); + Domain c0Domain = Domain.singleValue(INTEGER, 7L); + + TupleDomain remapped = remapPredicateColumnIndicesToPhysical( + fileSchema, + TupleDomain.withColumnDomains(Map.of(staleC2, c2Domain, staleC0, c0Domain)), + false); + + Map domains = remapped.getDomains().orElseThrow(); + assertThat(domains).hasSize(2); + // Each domain now keys off the column's physical position, so it is matched against that column's statistics + assertThat(handleOf(domains, "c2").getBaseHiveColumnIndex()).isEqualTo(7); + assertThat(domains.get(handleOf(domains, "c2"))).isEqualTo(c2Domain); + assertThat(handleOf(domains, "c0").getBaseHiveColumnIndex()).isEqualTo(5); + assertThat(domains.get(handleOf(domains, "c0"))).isEqualTo(c0Domain); + } + + @Test + public void testRemapPredicateDropsColumnsAbsentFromFile() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + HiveColumnHandle present = createDummyHandle("c0", 0, HiveType.HIVE_INT, INTEGER); + // Added after this base file was written, so the file does not carry them. Two of them, because the + // projection remap's out-of-range sentinel is one value that every absent column would share. + HiveColumnHandle firstAbsent = createDummyHandle("c1", 1, HiveType.HIVE_INT, INTEGER); + HiveColumnHandle secondAbsent = createDummyHandle("c2", 2, HiveType.HIVE_INT, INTEGER); + Domain presentDomain = Domain.singleValue(INTEGER, 1L); + + TupleDomain remapped = remapPredicateColumnIndicesToPhysical( + fileSchema, + TupleDomain.withColumnDomains(Map.of( + present, presentDomain, + firstAbsent, Domain.singleValue(INTEGER, 2L), + secondAbsent, Domain.singleValue(INTEGER, 3L))), + false); + + // Both absent columns are dropped rather than mapped to that shared sentinel, which would have collided. + // Dropping them costs row group pruning only; the engine still applies the filter itself. + Map domains = remapped.getDomains().orElseThrow(); + assertThat(domains).hasSize(1); + assertThat(handleOf(domains, "c0").getBaseHiveColumnIndex()).isEqualTo(5); + assertThat(domains.get(handleOf(domains, "c0"))).isEqualTo(presentDomain); + } + + @Test + public void testRemapPredicateKeepsOneDomainPerPhysicalColumn() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + // Two handles whose names differ only by case resolve to the same file field, so both land on physical + // index 5 while remaining unequal to each other. The connector cannot produce this - Hive normalises + // column names to lower case - but pushing both down would hand getParquetTupleDomain one + // ColumnDescriptor twice, which it rejects by failing the whole split. + HiveColumnHandle upperCase = createDummyHandle("C0", 0, HiveType.HIVE_INT, INTEGER); + HiveColumnHandle lowerCase = createDummyHandle("c0", 3, HiveType.HIVE_INT, INTEGER); + Domain firstDomain = Domain.create(ValueSet.ofRanges(Range.greaterThan(INTEGER, 10L)), false); + // Insertion-ordered so that "first wins" is a deterministic assertion + Map predicate = new LinkedHashMap<>(); + predicate.put(upperCase, firstDomain); + predicate.put(lowerCase, Domain.create(ValueSet.ofRanges(Range.lessThan(INTEGER, 20L)), false)); + + TupleDomain remapped = remapPredicateColumnIndicesToPhysical( + fileSchema, TupleDomain.withColumnDomains(predicate), false); + + // Only the first is pushed down, and no IllegalArgumentException escapes + Map domains = remapped.getDomains().orElseThrow(); + assertThat(domains).hasSize(1); + HiveColumnHandle survivor = handleOf(domains, "C0"); + assertThat(survivor.getBaseHiveColumnIndex()).isEqualTo(5); + assertThat(domains.get(survivor)).isEqualTo(firstDomain); + } + + @Test + public void testRemapPredicateKeepsBothProjectionsOfOneBaseColumn() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + // Two dereference handles projecting DIFFERENT subfields of the same struct column. Both resolve to base + // physical index 5, but getParquetTupleDomain builds a descriptor per subfield path, so it would accept + // both; deduplicating on the base index alone would silently discard one of the two domains. + HiveType structType = HiveType.valueOf("struct"); + RowType baseType = RowType.rowType(RowType.field("f", INTEGER), RowType.field("g", INTEGER)); + HiveColumnHandle onF = dereferenceHandle(structType, baseType, 0, "f"); + HiveColumnHandle onG = dereferenceHandle(structType, baseType, 1, "g"); + Domain fDomain = Domain.singleValue(INTEGER, 5L); + Domain gDomain = Domain.singleValue(INTEGER, 3L); + + TupleDomain remapped = remapPredicateColumnIndicesToPhysical( + fileSchema, + TupleDomain.withColumnDomains(Map.of(onF, fDomain, onG, gDomain)), + false); + + assertThat(remapped.getDomains().orElseThrow()) + .as("both subfield domains survive, each on the base column's physical index") + .isEqualTo(Map.of( + withBaseIndex(onF, 5), fDomain, + withBaseIndex(onG, 5), gDomain)); + } + + @Test + public void testRemapPreservesTheBaseTypeOfADereferenceHandle() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + // A handle projecting one field out of a struct column. HudiMetadata does not implement applyProjection, + // so the connector never builds one today, but the remap has to rebuild it without corrupting it: the + // constructor's type argument is the BASE column's type, while getType() is the projected field's. + RowType baseType = RowType.rowType(RowType.field("f", INTEGER)); + HiveColumnHandle dereference = dereferenceHandle(HiveType.valueOf("struct"), baseType, 0, "f"); + + HiveColumnHandle remapped = remapColumnIndicesToPhysical(fileSchema, List.of(dereference), false).get(0); + + assertThat(remapped.getBaseHiveColumnIndex()) + .as("physical index") + .isEqualTo(5); + assertThat(remapped.getBaseType()) + .as("base type, which is what the parquet page source reads") + .isEqualTo(baseType); + assertThat(remapped.getType()) + .as("projected field type") + .isEqualTo(INTEGER); + assertThat(remapped.getHiveColumnProjectionInfo()) + .as("projection info") + .isEqualTo(dereference.getHiveColumnProjectionInfo()); + } + + @Test + public void testRemapPredicateAllAndNonePassThrough() + { + MessageType fileSchema = hudiFileSchema(1); + + assertThat(remapPredicateColumnIndicesToPhysical(fileSchema, TupleDomain.all(), false)) + .isEqualTo(TupleDomain.all()); + assertThat(remapPredicateColumnIndicesToPhysical(fileSchema, TupleDomain.none(), false)) + .isEqualTo(TupleDomain.none()); + } + + @Test + public void testRemapPredicateCaseSensitivity() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + HiveColumnHandle upperCase = createDummyHandle("C0", 0, HiveType.HIVE_INT, INTEGER); + TupleDomain predicate = TupleDomain.withColumnDomains(Map.of(upperCase, Domain.singleValue(INTEGER, 1L))); + + // Case-insensitive: "C0" resolves to the file's "c0" at physical index 5 + Map insensitive = remapPredicateColumnIndicesToPhysical(fileSchema, predicate, false) + .getDomains().orElseThrow(); + assertThat(handleOf(insensitive, "C0").getBaseHiveColumnIndex()).isEqualTo(5); + + // Case-sensitive: no match, so the domain is dropped instead of being left on a stale ordinal + assertThat(remapPredicateColumnIndicesToPhysical(fileSchema, predicate, true).isAll()).isTrue(); + } + + @Test + public void testRemapPredicatePreservesEveryOtherHandleAttribute() + { + // Physical Schema: the five Hudi meta columns, then [c0] + MessageType fileSchema = hudiFileSchema(1); + + HiveColumnHandle original = new HiveColumnHandle( + "c0", + 0, + HiveType.HIVE_INT, + INTEGER, + Optional.empty(), + HiveColumnHandle.ColumnType.REGULAR, + Optional.of("a comment")); + Domain domain = Domain.create(ValueSet.ofRanges(Range.greaterThan(INTEGER, 900L)), true); + + Map domains = remapPredicateColumnIndicesToPhysical( + fileSchema, + TupleDomain.withColumnDomains(Map.of(original, domain)), + false) + .getDomains().orElseThrow(); + + HiveColumnHandle remapped = handleOf(domains, "c0"); + assertHandle(remapped, "c0", 5, HiveType.HIVE_INT, INTEGER); + assertThat(remapped.getComment()) + .as("Comment mismatch for c0") + .isEqualTo(Optional.of("a comment")); + assertThat(domains.get(remapped)) + .as("Domain mismatch for c0") + .isEqualTo(domain); + } + + @Test + public void testPredicateOnStaleOrdinalStillPrunesRowGroups() + throws Exception + { + List projection = List.of(dataColumn(SHADOWED_COLUMN), dataColumn(PREDICATE_COLUMN)); + + MaterializedResult result = read(projection, greaterThanThreshold(PREDICATE_COLUMN), false, DynamicFilter.EMPTY); + + // Correct results alone would also be produced by pushing nothing down; reading fewer rows than the file + // holds is only possible if the domain reached the column it was written for, and the matching rows must + // survive that pruning. The shadowed column never leaves 0..9, so a domain of "> 900" applied to it would + // prune every row group instead. + assertThat(result.getRowCount()) + .as("rows read out of %s", ROW_COUNT) + .isLessThan(ROW_COUNT); + assertThat(matchingRowCount(result, projection, PREDICATE_COLUMN)) + .as("rows matching %s > %s after pruning", PREDICATE_COLUMN, THRESHOLD) + .isEqualTo(MATCHING_ROW_COUNT); + } + + @Test + public void testStaleOrdinalArrivingThroughADynamicFilter() + throws Exception + { + List projection = List.of(dataColumn(SHADOWED_COLUMN), dataColumn(PREDICATE_COLUMN)); + + // A dynamic filter reaches getCombinedPredicate by its own route, and its handles carry the same stale + // metastore ordinals the split's predicate does + MaterializedResult result = read(projection, TupleDomain.all(), false, + dynamicFilterOn(greaterThanThreshold(PREDICATE_COLUMN))); + + assertThat(matchingRowCount(result, projection, PREDICATE_COLUMN)) + .as("rows matching a dynamic filter of %s > %s", PREDICATE_COLUMN, THRESHOLD) + .isEqualTo(MATCHING_ROW_COUNT); + } + + @Test + public void testPredicateOnColumnAddedAfterBaseFileWasWritten() + throws Exception + { + // The metastore carries one column more than this base file does, numbered 10 - an ordinal that is still + // in range physically, where it picks out "c5" + String addedColumn = "c" + DATA_COLUMN_COUNT; + List projection = List.of(dataColumn("c5"), dataColumn(PREDICATE_COLUMN), dataColumn(addedColumn)); + + // IS NULL, not a range: the added column is null in every row of this base file, so this predicate is + // satisfied by all of them. A range predicate would be unsatisfiable here and the buggy read's empty + // result would be the right answer by accident. + MaterializedResult result = read(projection, + TupleDomain.withColumnDomains(Map.of(dataColumn(addedColumn), Domain.onlyNull(INTEGER))), + false, DynamicFilter.EMPTY); + + // The added column must not stay on its stale metastore ordinal: pushed positionally it would land on + // "c5", which has no nulls at all, and every row group would be pruned. + assertThat(result.getRowCount()).as("rows read").isEqualTo(ROW_COUNT); + assertThat(result.getMaterializedRows().getFirst().getField(2)).as("value of %s", addedColumn).isNull(); + } + + @Test + public void testPositionalAndNameBasedResolutionAgree() + throws Exception + { + List projection = List.of(dataColumn(SHADOWED_COLUMN), dataColumn(PREDICATE_COLUMN)); + TupleDomain predicate = greaterThanThreshold(PREDICATE_COLUMN); + + MaterializedResult positional = read(projection, predicate, false, DynamicFilter.EMPTY); + MaterializedResult byName = read(projection, predicate, true, DynamicFilter.EMPTY); + + // Anchor the comparison: both modes regressing to no pushdown at all would otherwise agree happily + assertThat(byName.getRowCount()).as("rows read with use-column-names=true").isLessThan(ROW_COUNT); + assertThat(positional.getMaterializedRows()) + .as("hudi.parquet.use-column-names=false must read what use-column-names=true reads") + .isEqualTo(byName.getMaterializedRows()); + } + + /** + * Reads the whole base file through the page source the connector builds for a split with no log files, which + * is the only path on which it enables predicate pushdown. + */ + private static MaterializedResult read( + List projection, + TupleDomain predicate, + boolean useParquetColumnNames, + DynamicFilter dynamicFilter) + throws Exception + { + long fileSize = Files.size(baseFile); + HudiSplit split = new HudiSplit( + new HudiBaseFile(baseFile.toString(), baseFile.getFileName().toString(), fileSize, 0, 0, fileSize), + List.of(), + "000", + predicate, + List.of(), + SplitWeight.standard()); + HudiSessionProperties sessionProperties = new HudiSessionProperties( + new HudiConfig().setUseParquetColumnNames(useParquetColumnNames), + new ParquetReaderConfig()); + ConnectorSession session = TestingConnectorSession.builder() + .setPropertyMetadata(sessionProperties.getSessionProperties()) + .build(); + + List types = projection.stream().map(HiveColumnHandle::getType).toList(); + try (ConnectorPageSource pageSource = createPageSource( + session, + projection, + split, + new LocalInputFile(baseFile.toFile()), + baseFile.toString(), + 0L, + fileSize, + OptionalLong.of(fileSize), + new FileFormatDataSourceStats(), + ParquetReaderOptions.builder().build(), + DateTimeZone.UTC, + dynamicFilter, + true)) { + return materializeSourceDataStream(session, pageSource, types).toTestTypes(); + } + } + + /** + * Builds a file schema laid out like a Hudi base file: the five {@code _hoodie_*} meta columns followed by + * {@code dataColumnCount} int columns named {@code c0..cN}. A metastore synced with + * {@code hoodie.datasource.hive_sync.omit_metadata_fields=true} omits the meta columns, so a data column's + * metastore ordinal is its physical ordinal minus five. + */ + private static MessageType hudiFileSchema(int dataColumnCount) + { + List fields = new ArrayList<>(); + for (String metaColumn : HOODIE_META_COLUMNS) { + fields.add(Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, OPTIONAL).as(LogicalTypeAnnotation.stringType()).named(metaColumn)); + } + for (int i = 0; i < dataColumnCount; i++) { + fields.add(Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, OPTIONAL).named("c" + i)); + } + return new MessageType("hudi_base_file", fields); + } + + /** + * Builds the handle a metastore without the Hudi meta columns produces: numbered by its position among the + * data columns alone, which is {@code HOODIE_META_COLUMNS.size()} short of its physical position. Only + * {@code c0..cN} data column names are accepted - the numeric suffix IS the metastore ordinal - so a meta + * column name passed here would fail to parse rather than produce a meaningful handle. + */ + private static HiveColumnHandle dataColumn(String columnName) + { + return createBaseColumn(columnName, parseInt(columnName.substring(1)), HiveType.HIVE_INT, INTEGER, + HiveColumnHandle.ColumnType.REGULAR, Optional.empty()); + } + + /** A handle projecting the {@code fieldIndex}-th field, named {@code fieldName}, out of the struct column {@code c0}. */ + private static HiveColumnHandle dereferenceHandle(HiveType structType, RowType baseType, int fieldIndex, String fieldName) + { + return new HiveColumnHandle( + "c0", + 0, + structType, + baseType, + Optional.of(new HiveColumnProjectionInfo(List.of(fieldIndex), List.of(fieldName), HiveType.HIVE_INT, INTEGER)), + HiveColumnHandle.ColumnType.REGULAR, + Optional.empty()); + } + + /** The handle {@code remapColumnIndicesToPhysical} is expected to rebuild from {@code handle}. */ + private static HiveColumnHandle withBaseIndex(HiveColumnHandle handle, int baseHiveColumnIndex) + { + return new HiveColumnHandle( + handle.getBaseColumnName(), + baseHiveColumnIndex, + handle.getBaseHiveType(), + handle.getBaseType(), + handle.getHiveColumnProjectionInfo(), + handle.getColumnType(), + handle.getComment()); + } + + private static TupleDomain greaterThanThreshold(String columnName) + { + return TupleDomain.withColumnDomains(Map.of( + dataColumn(columnName), + Domain.create(ValueSet.ofRanges(Range.greaterThan(INTEGER, THRESHOLD)), false))); + } + + private static DynamicFilter dynamicFilterOn(TupleDomain predicate) + { + return new DynamicFilter() + { + @Override + public Set getColumnsCovered() + { + return Set.copyOf(predicate.getDomains().orElseThrow().keySet()); + } + + @Override + public CompletableFuture isBlocked() + { + return CompletableFuture.completedFuture(null); + } + + @Override + public boolean isComplete() + { + return true; + } + + @Override + public boolean isAwaitable() + { + return false; + } + + @Override + public TupleDomain getCurrentPredicate() + { + return predicate.transformKeys(ColumnHandle.class::cast); + } + }; + } + + private static long matchingRowCount(MaterializedResult result, List projection, String columnName) + { + int fieldIndex = projection.indexOf(dataColumn(columnName)); + return result.getMaterializedRows().stream() + .map(row -> row.getField(fieldIndex)) + .filter(value -> value != null && ((Number) value).longValue() > THRESHOLD) + .count(); + } + + private static int rowGroupCount(Path path) + throws IOException + { + try (ParquetFileReader reader = ParquetFileReader.open(new org.apache.parquet.io.LocalInputFile(path))) { + return reader.getRowGroups().size(); + } + } + + /** + * Returns the single remapped handle carrying the given base column name. + */ + private static HiveColumnHandle handleOf(Map domains, String baseColumnName) + { + return domains.keySet().stream() + .filter(handle -> handle.getBaseColumnName().equals(baseColumnName)) + .findFirst() + .orElseThrow(() -> new AssertionError("No domain was kept for column " + baseColumnName)); + } + /** * Creates a basic HiveColumnHandle for testing. * Assumes REGULAR column type and no projection info or comments. diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java index 2eac0a195f87d..9bcf0d6bcac4e 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/AbstractMergerHudiTablesInitializer.java @@ -148,6 +148,16 @@ public final void initializeTables(QueryRunner queryRunner, Location externalLoc /** The table's data columns, in schema order; the Hudi metadata columns are prepended by this class. */ protected abstract List dataColumns(); + /** + * Whether the metastore definition prepends {@link #HUDI_META_COLUMNS}. The base file always carries them, so + * returning {@code false} models hive sync with {@code hoodie.datasource.hive_sync.omit_metadata_fields=true}: + * every data column's metastore ordinal then sits five below its physical position in the file. + */ + protected boolean includeMetaColumnsInMetastore() + { + return true; + } + /** The Avro schema the write client writes, matching {@link #dataColumns()}. */ protected abstract Schema avroSchema(); @@ -285,7 +295,7 @@ private Table createTableDefinition(String schemaName, String metastoreTableName .setTableType(EXTERNAL_TABLE.name()) .setOwner(Optional.of("public")) .setDataColumns(ImmutableList.builder() - .addAll(HUDI_META_COLUMNS) + .addAll(includeMetaColumnsInMetastore() ? HUDI_META_COLUMNS : ImmutableList.of()) .addAll(dataColumns()) .build()) .setParameters(ImmutableMap.of("serialization.format", "1", "EXTERNAL", "TRUE")) diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedMetaColumnsHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedMetaColumnsHudiTablesInitializer.java new file mode 100644 index 0000000000000..1be2c44cf65df --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/OmittedMetaColumnsHudiTablesInitializer.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import io.trino.metastore.Column; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; + +/** + * Creates a non-partitioned table whose METASTORE column list omits the five {@code _hoodie_*} meta columns the + * base file itself carries -- the layout hive sync leaves behind with + * {@code hoodie.datasource.hive_sync.omit_metadata_fields=true}. Every data column's metastore ordinal is therefore + * five below its physical position, which is the shape {@code hudi.parquet.use-column-names=false} resolves + * positionally and the one apache/hudi#19387 was about. Every other fixture in this module registers the meta + * columns ({@code ResourceHudiTablesInitializer.TestingTable.getDataColumns} prepends them unconditionally), so + * metastore ordinal equals physical ordinal there and the bug cannot appear. + *

    + * The column list looks padded on purpose. Physical fields 0..4 are always the meta columns, so a data column at + * metastore ordinal {@code i} is read positionally at physical field {@code i}, which is a meta column for any + * {@code i < 5}. A meta column can never be part of a projection here -- the metastore does not expose it -- so + * {@code descriptorsByPath} has no entry for it and a stray domain is silently discarded rather than misapplied. + * Only from the SIXTH data column on does the stale ordinal land on a real, projectable column. Hence + * {@code late_value} at metastore ordinal 5, shadowed by {@code shadowed_value} at physical field 5, and the four + * fillers in between. + *

    + * The values are disjoint so the damage is unambiguous: {@code shadowed_value} stays in 1..5 while + * {@code late_value} is above 1000, so a domain meant for {@code late_value} but matched against + * {@code shadowed_value}'s statistics prunes the only row group and the query returns nothing. + *

    + * A single bulk-insert commit, so the file slice has no log files: predicate pushdown is only enabled for + * base-file-only splits. See {@code TestHudiConnectorParquetColumnNamesTest}. + */ +public class OmittedMetaColumnsHudiTablesInitializer + extends AbstractMergerHudiTablesInitializer +{ + public static final String TABLE_NAME = "omitted_meta_columns_mor"; + + /** The column the predicate goes on: metastore ordinal 5, physical field 10. */ + public static final String LATE_COLUMN = "late_value"; + /** The column physically sitting at {@link #LATE_COLUMN}'s stale ordinal, and therefore the one that shadows it. */ + public static final String SHADOWED_COLUMN = "shadowed_value"; + /** Below every {@link #LATE_COLUMN} value and above every {@link #SHADOWED_COLUMN} one. */ + public static final long THRESHOLD = 900; + + private static final int ROW_COUNT = 5; + private static final List FILLER_COLUMNS = ImmutableList.of("filler_1", "filler_2", "filler_3", "filler_4"); + + public OmittedMetaColumnsHudiTablesInitializer() + { + super(TABLE_NAME); + } + + @Override + protected boolean includeMetaColumnsInMetastore() + { + return false; + } + + @Override + protected List dataColumns() + { + ImmutableList.Builder columns = ImmutableList.builder(); + columns.add(new Column(SHADOWED_COLUMN, HIVE_LONG, Optional.empty(), Map.of())); + FILLER_COLUMNS.forEach(name -> columns.add(new Column(name, HIVE_LONG, Optional.empty(), Map.of()))); + columns.add(new Column(LATE_COLUMN, HIVE_LONG, Optional.empty(), Map.of())); + columns.add(new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of())); + columns.add(new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); + return columns.build(); + } + + @Override + protected Schema avroSchema() + { + List fields = new ArrayList<>(); + fields.add(new Schema.Field(SHADOWED_COLUMN, Schema.create(Schema.Type.LONG))); + FILLER_COLUMNS.forEach(name -> fields.add(new Schema.Field(name, Schema.create(Schema.Type.LONG)))); + fields.add(new Schema.Field(LATE_COLUMN, Schema.create(Schema.Type.LONG))); + fields.add(new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING))); + fields.add(new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); + return Schema.createRecord(TABLE_NAME, null, null, false, fields); + } + + @Override + protected void configureTableConfig(HoodieTableMetaClient.TableBuilder tableBuilder) + { + tableBuilder.setRecordMergeMode(RecordMergeMode.COMMIT_TIME_ORDERING); + } + + @Override + protected void configureWriteConfig(HoodieWriteConfig.Builder writeConfigBuilder) + { + writeConfigBuilder.withRecordMergeMode(RecordMergeMode.COMMIT_TIME_ORDERING); + } + + @Override + protected void writeInitialCommits(HoodieJavaWriteClient client) + { + Schema schema = avroSchema(); + List> records = new ArrayList<>(); + for (int row = 1; row <= ROW_COUNT; row++) { + records.add(record(schema, "k" + row, row, 1000L + row)); + } + // One commit only: a file slice with log files would take the merge path, which disables pushdown. + String commit = client.startCommit(); + List statuses = client.bulkInsert(records, commit); + client.commit(commit, statuses); + } + + /** The expected rows of {@code SELECT key, shadowed_value, late_value ... WHERE late_value > THRESHOLD}. */ + public static String expectedRowsAboveThreshold() + { + List rows = new ArrayList<>(); + for (int row = 1; row <= ROW_COUNT; row++) { + rows.add("('k%s', CAST(%s AS BIGINT), CAST(%s AS BIGINT))".formatted(row, row, 1000 + row)); + } + return "VALUES " + String.join(", ", rows); + } + + private static HoodieRecord record(Schema schema, String key, long shadowedValue, long lateValue) + { + GenericRecord record = new GenericData.Record(schema); + record.put(SHADOWED_COLUMN, shadowedValue); + FILLER_COLUMNS.forEach(name -> record.put(name, 0L)); + record.put(LATE_COLUMN, lateValue); + record.put(RECORD_KEY_FIELD, key); + record.put(ORDERING_FIELD, 100L); + return avroRecord(record, key); + } +} From 99c77c7382f84f755c98f6607cbdbb81686c629f Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Mon, 3 Aug 2026 12:59:28 +0530 Subject: [PATCH 228/255] test(java-client): cover both marker types with the embedded timeline server (#19444) (cherry picked from commit 636e6fab0da87ddbaf48a2e318f7c9fbdaf44761) --- .../apache/hudi/config/HoodieWriteConfig.java | 4 +- .../TestHoodieJavaWriteClientInsert.java | 85 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 317f548faaf80..f5cc2bf67a3e3 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -3903,7 +3903,9 @@ private String getDefaultMarkersType(EngineType engineType) { } case FLINK: case JAVA: - // Timeline-server-based marker is not supported for Flink and Java engines + // Timeline-server-based markers are not the default for Flink and Java, but they are not + // unsupported either: setting hoodie.write.markers.type explicitly selects them, subject to the + // same gates WriteMarkersFactory applies to every engine. return MarkerType.DIRECT.toString(); default: throw new HoodieNotSupportedException("Unsupported engine " + engineType); diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java index 83d51a604aef9..5883c4a558502 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java @@ -27,9 +27,12 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileFormatUtils; +import org.apache.hudi.common.util.MarkerUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; @@ -47,12 +50,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.AVRO_SCHEMA; import static org.apache.hudi.common.testutils.HoodieTestTable.makeNewCommitTime; @@ -140,6 +146,85 @@ public void testWriteClientAndTableServiceClientWithTimelineServer( writeClient.close(); } + /** + * HUDI-5011: exercises a Java-engine write config against both marker types with the embedded timeline + * server. {@code HoodieWriteConfig.Builder} defaults {@link MarkerType#DIRECT} for + * {@link EngineType#JAVA}, so the timeline-server-based path is only reached when + * {@code hoodie.write.markers.type} is set explicitly. Note the default applies to the config's engine + * type, not to the client: {@code HoodieJavaWriteClient} never inspects it, so a config left on the + * builder's SPARK default would resolve to TIMELINE_SERVER_BASED on its own. + * + *

    Backup for the remote file system view is disabled so that a timeline-server failure fails the test + * rather than silently falling back to a local view, matching + * {@code HoodieJavaClientTestHarness#getConfigBuilder}. + */ + @ParameterizedTest + @EnumSource(MarkerType.class) + public void testInsertWithEmbeddedTimelineServerAndMarkerType(MarkerType markerType) throws Exception { + HoodieWriteConfig config = makeHoodieClientConfigBuilder(basePath) + .withMarkersType(markerType.name()) + .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder() + .withEnableBackupForRemoteFileSystemView(false).build()) + .build(); + + HoodieJavaWriteClient writeClient = getHoodieWriteClient(config); + assertTrue(writeClient.getTimelineServer().isPresent(), + "The embedded timeline server should be running for marker type " + markerType); + + List records = new ArrayList<>(); + records.add(createSimpleRecord("1", "2021-09-11T16:16:41.415Z", 1)); + records.add(createSimpleRecord("2", "2021-09-11T16:16:41.415Z", 2)); + + String commitTime = makeNewCommitTime(1, "%09d"); + WriteClientTestUtils.startCommitWithTime(writeClient, commitTime); + List statuses = writeClient.insert(records, commitTime); + + // Inspect the markers before commit removes them. Only the timeline-server path writes MARKERS.type, + // so this is what would notice a silent fallback to DirectWriteMarkers. + metaClient = HoodieTableMetaClient.reload(metaClient); + StoragePath markerDir = new StoragePath(metaClient.getMarkerFolderPath(commitTime)); + boolean markerTypeFileExists = MarkerUtils.doesMarkerTypeFileExist(metaClient.getStorage(), markerDir); + List markerFileNames = listMarkerFileNames(markerDir); + if (markerType == MarkerType.TIMELINE_SERVER_BASED) { + assertTrue(markerTypeFileExists, + "Timeline-server-based markers should have written " + MarkerUtils.MARKER_TYPE_FILENAME); + assertTrue(markerFileNames.stream().anyMatch( + name -> name.startsWith(MarkerUtils.MARKERS_FILENAME_PREFIX) + && !name.equals(MarkerUtils.MARKER_TYPE_FILENAME)), + () -> "Timeline-server-based markers should have written a " + + MarkerUtils.MARKERS_FILENAME_PREFIX + " file. Found: " + markerFileNames); + } else { + assertFalse(markerTypeFileExists, + "Direct markers should not have written " + MarkerUtils.MARKER_TYPE_FILENAME); + // Absence of MARKERS.type alone would also hold if the write produced no markers at all, so + // require the direct markers themselves. + assertTrue(markerFileNames.stream().anyMatch(name -> name.contains(HoodieTableMetaClient.MARKER_EXTN)), + () -> "Direct markers should have written a " + HoodieTableMetaClient.MARKER_EXTN + + " file. Found: " + markerFileNames); + } + + writeClient.commit(commitTime, statuses); + + metaClient = HoodieTableMetaClient.reload(metaClient); + assertTrue(metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().isPresent(), + "The commit should have completed for marker type " + markerType); + assertEquals(1, getIncrementalFiles("2021/09/11", "0", -1).length, + "One base file should have been written for marker type " + markerType); + } + + /** + * The marker file names written under {@code markerDir}, read recursively off storage rather than + * through the timeline server, so the assertion does not lean on the path it is checking. + */ + private List listMarkerFileNames(StoragePath markerDir) throws IOException { + if (!metaClient.getStorage().exists(markerDir)) { + return Collections.emptyList(); + } + return metaClient.getStorage().listFiles(markerDir).stream() + .map(pathInfo -> pathInfo.getPath().getName()) + .collect(Collectors.toList()); + } + @Test public void testInsert() throws Exception { HoodieWriteConfig config = makeHoodieClientConfigBuilder(basePath).withMergeAllowDuplicateOnInserts(true).build(); From ad9682ca914d29f9d7105d8a37eabaca67ff386c Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Mon, 3 Aug 2026 14:06:50 +0530 Subject: [PATCH 229/255] fix(metrics): explain how to enable the CloudWatch reporter when hudi-aws is absent (#19418) The CloudWatch reporter lives in the optional hudi-aws module and is loaded reflectively, but not every engine bundle shades that module. Selecting hoodie.metrics.reporter.type=CLOUDWATCH without it on the classpath failed with "Unable to load class", naming neither the missing class nor a remedy. Report the missing class, point at hudi-aws-bundle, and name the config to change instead. Other reflection failures are rethrown unchanged. ReflectionUtils.getClass now names the class in the generic failure too, which improves the message for every reflective load in Hudi. Closes #15293 (cherry picked from commit 637996c5ab2078dfd858a2a425da8f28af1ce789) Cherry-pick adaptation: - Import conflict only. This branch has not taken the hudi-common package reorganization (#19195), so HoodieMetricsConfig is still imported from org.apache.hudi.config.metrics rather than org.apache.hudi.common.config.metrics. Kept the release import and added the new VisibleForTesting import alongside it. org.apache.hudi.common.util.VisibleForTesting resolves unchanged here; the class sits in hudi-common rather than hudi-io but has the same fully-qualified name. All three files otherwise apply byte-identically to the upstream delta. --- .../hudi/metrics/MetricsReporterFactory.java | 30 ++++++++- .../metrics/TestMetricsReporterFactory.java | 64 ++++++++++++++++++- .../hudi/common/util/ReflectionUtils.java | 2 +- 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index e2b850e8732f3..5a71cae75ad53 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metrics.custom.CustomizableMetricsReporter; @@ -40,6 +41,10 @@ @Slf4j public class MetricsReporterFactory { + @VisibleForTesting + static final String CLOUDWATCH_REPORTER_CLASS = + "org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter"; + public static Option createReporter(HoodieMetricsConfig metricsConfig, MetricRegistry registry) { String reporterClassName = metricsConfig.getMetricReporterClassName(); @@ -84,8 +89,7 @@ public static Option createReporter(HoodieMetricsConfig metrics reporter = new ConsoleMetricsReporter(registry); break; case CLOUDWATCH: - reporter = (MetricsReporter) ReflectionUtils.loadClass("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter", - new Class[]{HoodieMetricsConfig.class, MetricRegistry.class}, metricsConfig, registry); + reporter = createCloudWatchReporter(metricsConfig, registry); break; case M3: reporter = new M3MetricsReporter(metricsConfig, registry); @@ -99,4 +103,26 @@ public static Option createReporter(HoodieMetricsConfig metrics } return Option.ofNullable(reporter); } + + /** + * The CloudWatch reporter ships in the optional {@code hudi-aws} module and so is loaded reflectively. + * Not every engine bundle shades that module, in which case class loading fails without pointing at a + * remedy. Translate that into an actionable error. + */ + private static MetricsReporter createCloudWatchReporter(HoodieMetricsConfig metricsConfig, MetricRegistry registry) { + try { + return (MetricsReporter) ReflectionUtils.loadClass(CLOUDWATCH_REPORTER_CLASS, + new Class[] {HoodieMetricsConfig.class, MetricRegistry.class}, metricsConfig, registry); + } catch (HoodieException e) { + if (e.getCause() instanceof ClassNotFoundException) { + throw new HoodieException(String.format( + "Cannot report metrics to CloudWatch: %s was not found on the classpath. It ships in the " + + "optional hudi-aws module, which not every engine bundle includes. Add the " + + "hudi-aws-bundle jar matching your Hudi version to the classpath, or set %s to a " + + "different reporter type.", + CLOUDWATCH_REPORTER_CLASS, HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), e); + } + throw e; + } + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java index d456748d97692..2845f9bf9ff47 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java +++ b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java @@ -82,7 +82,7 @@ void metricsReporterFactoryShouldReturnCloudWatchReporter() { try (MockedStatic mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { mockedStatic.when(() -> ReflectionUtils.loadClass( - eq("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter"), + eq(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), any(Class[].class), eq(metricsConfig), eq(registry) @@ -94,6 +94,68 @@ void metricsReporterFactoryShouldReturnCloudWatchReporter() { } } + /** + * {@code hudi-aws} is deliberately absent from this module's test classpath, which is exactly the + * situation a user hits on an engine bundle that does not shade that module: the reflectively loaded + * CloudWatch reporter cannot be found. The failure must name the missing class and how to fix it, + * not just report that some class could not be loaded. + */ + @Test + void metricsReporterFactoryShouldExplainHowToEnableCloudWatchWhenHudiAwsIsMissing() { + when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); + + HoodieException exception = assertThrows(HoodieException.class, + () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); + + String message = exception.getMessage(); + assertTrue(message.contains(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), + () -> "The failure should name the missing reporter class, but was: " + message); + assertTrue(message.contains("hudi-aws-bundle"), + () -> "The failure should name the bundle that provides the reporter, but was: " + message); + assertTrue(message.contains(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), + () -> "The failure should name the config to change, but was: " + message); + } + + /** + * The classpath-based test above exercises the real failure but passes for any resolution failure. + * This pins the mapping itself: a ClassNotFoundException cause is what triggers the rewrite. + */ + @Test + void metricsReporterFactoryRewritesClassNotFoundIntoAnActionableMessage() { + when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); + try (MockedStatic mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { + mockedStatic.when(() -> ReflectionUtils.loadClass( + eq(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), any(Class[].class), eq(metricsConfig), eq(registry))) + .thenThrow(new HoodieException("Unable to load class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, + new ClassNotFoundException(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS))); + + HoodieException exception = assertThrows(HoodieException.class, + () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); + assertTrue(exception.getMessage().contains("hudi-aws-bundle"), + () -> "Expected the remedy to be named, but was: " + exception.getMessage()); + } + } + + /** + * The other direction, and the branch most likely to regress: a failure that is not a missing class must + * pass through untouched, so an unrelated instantiation error is never rewritten into "add hudi-aws-bundle". + */ + @Test + void metricsReporterFactoryLeavesNonClassNotFoundFailuresUntouched() { + when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); + try (MockedStatic mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { + mockedStatic.when(() -> ReflectionUtils.loadClass( + eq(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), any(Class[].class), eq(metricsConfig), eq(registry))) + .thenThrow(new HoodieException("Unable to instantiate class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, + new NoSuchMethodException(""))); + + HoodieException exception = assertThrows(HoodieException.class, + () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); + assertEquals("Unable to instantiate class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, + exception.getMessage(), "A non-ClassNotFound failure must not be rewritten"); + } + } + @Test void metricsReporterFactoryShouldReturnUserDefinedReporter() { when(metricsConfig.getMetricReporterClassName()).thenReturn(DummyMetricsReporter.class.getName()); diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java index cb4cbc4290f89..6a6bd436ab419 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java @@ -50,7 +50,7 @@ public static Class getClass(String clazzName) { try { return Class.forName(c); } catch (ClassNotFoundException e) { - throw new HoodieException("Unable to load class", e); + throw new HoodieException("Unable to load class " + c, e); } }); } From 05591536405e719ea8011cf86a69cd73728ce4bb Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Mon, 3 Aug 2026 18:45:29 +0530 Subject: [PATCH 230/255] fix(timeline): do not NPE on archived instants without a completion time (#19452) * fix(timeline): do not NPE on archived instants without a completion time Upgrading a table written by 0.x fails while polling the archived timeline: java.lang.NullPointerException: Cannot invoke "Object.toString()" because the return value of "org.apache.avro.generic.GenericRecord.get(String)" is null at CompletionTimeQueryViewV2.readCompletionTime completionTime is declared ["null","string"] with a null default in HoodieLSMTimelineInstant, and instants archived before that field existed carry no value for it. setCompletionTime already handles the null case by falling back to the instant time, with a comment saying so, but readCompletionTime called toString() on the raw field before reaching it. Read the field as an Object and let the existing fallback apply. Adds unit tests for both the missing and present cases; readCompletionTime is widened to package-private with @VisibleForTesting, matching the annotation already used in this class. The same unguarded toString() on this field also appears in ArchivedTimelineV2#readCommit and MetadataConversionUtils, where the right behaviour for a null value is less obvious. Left alone here and called out in the PR instead. Closes #17095 * fix(timeline): use StringUtils.objToString and tidy the regression test Review feedback. - readCompletionTime now uses StringUtils.objToString, the existing null-safe toString that HoodieAvroUtils.getNullableValAsString is built on, instead of a local variable and a ternary. - Test: added the missing class javadoc, renamed to match the convention in this area (testReadCompletionTime / testReadCompletionTimeWithoutCompletionTime), dropped the instantTime and action fields that readCompletionTime never reads and which implied a coupling that is not there, and made the assertion messages consistent across both cases. On moving the test onto the real archiving harness in hudi-client-common: tried it, and it does not reproduce this bug. Details in the review thread. * test(timeline): cover the null completion time on the real archived read path Adds testReadCompletionTimeWithoutCompletionTime to TestCompletionTimeQueryView. It archives an instant carrying no completion time through LSMTimelineWriter and reads it back through the archived timeline, so the fallback in readCompletionTime is exercised on the path that actually broke. Reverting the fix makes it fail with the HUDI-9655 NPE. The test asserts LSMTimelineWriter's exception handler collected nothing. That handler is optional and the write loop swallows per-instant failures, so without the assertion a failed archive write would leave the test passing against an empty archive. With real-path coverage the mocked TestCompletionTimeQueryViewV2 is redundant, so it goes, and readCompletionTime returns to private. * fix(timeline): null-safe the other two reads of the archived completionTime Review question: the same raw record.get(COMPLETION_TIME_ARCHIVED_META_FIELD) .toString() also lives in ArchivedTimelineV2#readCommit and MetadataConversionUtils#createMetaWrapper. Checked, and both do NPE on the same record shape - createMetaWrapper demonstrably, at line 174, on a record with the field left unset. Both read the same LSM records as the query view, so the trigger is identical: a table archived before completionTime existed. Add ArchivedTimelineV2#completionTimeOrInstantTime so the two sites cannot drift, and route both through it. Falling back to the instant time is the behaviour CompletionTimeQueryViewV2#setCompletionTime already documents for these records, so this follows existing precedent rather than inventing a rule. Both sites build a COMPLETED HoodieInstant, and leaving the completion time null there would only move the failure to whatever compares it. CompletionTimeQueryViewV2#readCompletionTime is left as is: it hands a possibly-null value to setCompletionTime, which owns the fallback, so it needs nothing further. Note the two sites already null-check the neighbouring nullable fields, metadata and plan, so completionTime was the odd one out rather than a deliberate choice. --------- Co-authored-by: voon (cherry picked from commit 55c7a308094c2330189584d9df4cf03f85edd56a) --- .../timeline/TestCompletionTimeQueryView.java | 53 ++++++++ .../timeline/MetadataConversionUtils.java | 2 +- .../versioning/v2/ArchivedTimelineV2.java | 21 +++- .../v2/CompletionTimeQueryViewV2.java | 6 +- .../TestArchivedInstantCompletionTime.java | 116 ++++++++++++++++++ 5 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedInstantCompletionTime.java diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java index 5046b1bf7028d..048ef727dc80c 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java @@ -127,6 +127,59 @@ void testReadCompletionTimeWithCornerCase() throws Exception { } } + /** + * The {@code completionTime} field of {@code HoodieLSMTimelineInstant} is declared + * {@code ["null","string"]} with a null default, and instants archived before the field existed carry + * no value for it. Reading such an instant must fall back to the instant time rather than throwing. + * + *

    See HUDI-9655: upgrading a table written by 0.x produced + * {@code NullPointerException: Cannot invoke "Object.toString()" because the return value of + * "org.apache.avro.generic.GenericRecord.get(String)" is null} while loading the archived timeline. + */ + @Test + void testReadCompletionTimeWithoutCompletionTime() throws Exception { + String tableName = "testTable"; + String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName; + HoodieTableMetaClient metaClient = HoodieTestUtils.init( + HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName); + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + .withMarkersType("DIRECT") + .build(); + HoodieTestTable testTable = HoodieTestTable.of(metaClient); + + // instant 1 only ever exists on the LSM timeline, as an instant archived by an older writer would. + String archivedInstantTime = String.format("%08d", 1); + HoodieCommitMetadata archivedMetadata = testTable.createCommitMetadata( + archivedInstantTime, WriteOperationType.INSERT, Arrays.asList("par1", "par2"), 10, false); + // instants 2..4 stay active, so that the query for instant 1 falls through to the archive. + for (int i = 2; i < 5; i++) { + String instantTime = String.format("%08d", i); + HoodieCommitMetadata metadata = testTable.createCommitMetadata( + instantTime, WriteOperationType.INSERT, Arrays.asList("par1", "par2"), 10, false); + testTable.addCommit(instantTime, Option.of(String.format("%08d", i + 1000)), Option.of(metadata)); + } + + // archive instant 1 with no completion time at all + ActiveAction activeAction = new DummyActiveAction( + INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED, "commit", archivedInstantTime, null), + convertMetadataToByteArray(archivedMetadata)); + List archiveFailures = new ArrayList<>(); + // LSMTimelineWriter#write swallows per-instant failures, so surface them rather than + // silently archiving nothing and leaving the assertion below to pass vacuously. + LSMTimelineWriter.getInstance(writeConfig, getMockHoodieTable(metaClient)) + .write(Collections.singletonList(activeAction), Option.empty(), Option.of(archiveFailures::add)); + assertTrue(archiveFailures.isEmpty(), + "Archiving an instant without a completion time should not fail: " + archiveFailures); + + metaClient.reloadActiveTimeline(); + try (CompletionTimeQueryView view = + metaClient.getTableFormat().getTimelineFactory().createCompletionTimeQueryView(metaClient)) { + assertThat("An archived instant without a completion time should fall back to its instant time", + view.getCompletionTime(archivedInstantTime).orElse(""), is(archivedInstantTime)); + } + } + @Test void testReadStartTime() throws Exception { String tableName = "testTable"; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java index 8ea9ccc03fb9a..b7f4019dfacd4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/MetadataConversionUtils.java @@ -171,7 +171,7 @@ public static HoodieArchivedMetaEntry createMetaWrapper( Option planBytes = planBuffer != null ? Option.of(planBuffer.array()) : Option.empty(); String instantTime = lsmTimelineRecord.get(ArchivedTimelineV2.INSTANT_TIME_ARCHIVED_META_FIELD).toString(); - String completionTime = lsmTimelineRecord.get(ArchivedTimelineV2.COMPLETION_TIME_ARCHIVED_META_FIELD).toString(); + String completionTime = ArchivedTimelineV2.completionTimeOrInstantTime(lsmTimelineRecord, instantTime); HoodieArchivedMetaEntry archivedMetaWrapper = new HoodieArchivedMetaEntry(); archivedMetaWrapper.setCommitTime(instantTime); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java index f88168ea985c4..2f8cdfe7d4e93 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.table.timeline.InstantComparison; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.avro.generic.GenericRecord; import org.slf4j.Logger; @@ -217,9 +218,27 @@ public HoodieArchivedTimeline reload(String startTs) { } } + /** + * The completion time of an archived instant, falling back to its instant time when the record carries + * none. + * + *

    {@code completionTime} is declared {@code ["null","string"]} with a null default in + * {@code HoodieLSMTimelineInstant} and has no value for instants archived before the field existed, so it + * must not be dereferenced. Defaulting to the instant time is the fallback + * {@code CompletionTimeQueryViewV2#setCompletionTime} already documents for the same records. + * + * @param record an LSM timeline record. + * @param instantTime the instant time to fall back to. + * @return the completion time, never null as long as {@code instantTime} is not. + */ + public static String completionTimeOrInstantTime(GenericRecord record, String instantTime) { + String completionTime = StringUtils.objToString(record.get(COMPLETION_TIME_ARCHIVED_META_FIELD)); + return completionTime != null ? completionTime : instantTime; + } + private HoodieInstant readCommit(String instantTime, GenericRecord record, Option> instantDetailsConsumer) { final String action = record.get(ACTION_ARCHIVED_META_FIELD).toString(); - final String completionTime = record.get(COMPLETION_TIME_ARCHIVED_META_FIELD).toString(); + final String completionTime = completionTimeOrInstantTime(record, instantTime); instantDetailsConsumer.ifPresent(consumer -> consumer.accept(instantTime, record)); return instantGenerator.createNewInstant(HoodieInstant.State.COMPLETED, action, instantTime, completionTime); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java index 516b4ad4cedd8..be8dbd0462b15 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/CompletionTimeQueryViewV2.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.InstantComparison; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.VisibleForTesting; import lombok.Getter; @@ -303,8 +304,9 @@ private void load() { } private void readCompletionTime(String instantTime, GenericRecord record) { - final String completionTime = record.get(COMPLETION_TIME_ARCHIVED_META_FIELD).toString(); - setCompletionTime(instantTime, completionTime); + // The field is nullable in HoodieLSMTimelineInstant and is absent for instants archived before it + // existed, so leave the fallback to setCompletionTime rather than dereferencing here. + setCompletionTime(instantTime, StringUtils.objToString(record.get(COMPLETION_TIME_ARCHIVED_META_FIELD))); } private void setCompletionTime(String beginInstantTime, String completionTime) { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedInstantCompletionTime.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedInstantCompletionTime.java new file mode 100644 index 0000000000000..cb6356aa807f1 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedInstantCompletionTime.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.table.timeline; + +import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; +import org.apache.hudi.avro.model.HoodieLSMTimelineInstant; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.versioning.v2.ArchivedTimelineV2; +import org.apache.hudi.common.table.timeline.versioning.v2.InstantGeneratorV2; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.function.BooleanSupplier; + +import static org.apache.hudi.common.table.timeline.versioning.v2.ArchivedTimelineV2.ACTION_ARCHIVED_META_FIELD; +import static org.apache.hudi.common.table.timeline.versioning.v2.ArchivedTimelineV2.COMPLETION_TIME_ARCHIVED_META_FIELD; +import static org.apache.hudi.common.table.timeline.versioning.v2.ArchivedTimelineV2.INSTANT_TIME_ARCHIVED_META_FIELD; +import static org.apache.hudi.common.table.timeline.versioning.v2.ArchivedTimelineV2.METADATA_ARCHIVED_META_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The {@code completionTime} field of {@code HoodieLSMTimelineInstant} is declared + * {@code ["null","string"]} with a null default, and carries no value for instants archived before the + * field existed. Every read of it therefore has to be null-safe; dereferencing it produced + * {@code NullPointerException: Cannot invoke "Object.toString()" because the return value of + * "GenericRecord.get(String)" is null} on a table upgraded from 0.x (HUDI-9655). + * + *

    This covers the two readers that build a completed {@link HoodieInstant} from such a record, both of + * which fall back to the instant time. + */ +class TestArchivedInstantCompletionTime { + + private static final String INSTANT_TIME = "00000001"; + + @Test + void completionTimeFallsBackToTheInstantTimeWhenAbsent() { + GenericRecord record = new GenericData.Record(HoodieLSMTimelineInstant.getClassSchema()); + // completionTime deliberately left unset, as it is for an instant archived before the field existed + + assertEquals(INSTANT_TIME, ArchivedTimelineV2.completionTimeOrInstantTime(record, INSTANT_TIME), + "An archived instant without a completion time should fall back to its instant time"); + } + + @Test + void completionTimeIsUsedWhenPresent() { + GenericRecord record = new GenericData.Record(HoodieLSMTimelineInstant.getClassSchema()); + record.put(COMPLETION_TIME_ARCHIVED_META_FIELD, "00001001"); + + assertEquals("00001001", ArchivedTimelineV2.completionTimeOrInstantTime(record, INSTANT_TIME), + "A present completion time should be used as-is"); + } + + /** + * The same field read on the way to a {@code HoodieArchivedMetaEntry}, which is the path a CLI or + * metadata-conversion caller takes rather than the query view. + */ + @Test + void createMetaWrapperFallsBackToTheInstantTimeWhenCompletionTimeIsAbsent() throws IOException { + GenericRecord record = new GenericData.Record(HoodieLSMTimelineInstant.getClassSchema()); + record.put(INSTANT_TIME_ARCHIVED_META_FIELD, INSTANT_TIME); + record.put(ACTION_ARCHIVED_META_FIELD, HoodieTimeline.COMMIT_ACTION); + record.put(METADATA_ARCHIVED_META_FIELD, ByteBuffer.wrap(new byte[0])); + // completionTime deliberately left unset + + HoodieArchivedMetaEntry entry = + MetadataConversionUtils.createMetaWrapper(mockMetaClientReturningEmptyCommitMetadata(), record); + + assertEquals(INSTANT_TIME, entry.getStateTransitionTime(), + "The archived entry should carry the instant time when the record has no completion time"); + assertEquals(INSTANT_TIME, entry.getCommitTime()); + } + + private static HoodieTableMetaClient mockMetaClientReturningEmptyCommitMetadata() throws IOException { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.EIGHT); + when(metaClient.getInstantGenerator()).thenReturn(new InstantGeneratorV2()); + + CommitMetadataSerDe serDe = mock(CommitMetadataSerDe.class); + when(serDe.deserialize(any(HoodieInstant.class), any(InputStream.class), + any(BooleanSupplier.class), eq(HoodieCommitMetadata.class))) + .thenReturn(new HoodieCommitMetadata()); + when(metaClient.getCommitMetadataSerDe()).thenReturn(serDe); + return metaClient; + } +} From 544d5f85793e19d6c78fade04ac169b9368409d3 Mon Sep 17 00:00:00 2001 From: voonhous Date: Mon, 3 Aug 2026 23:23:58 +0800 Subject: [PATCH 231/255] =?UTF-8?q?perf(trino):=20cache=20decimal=20Avro?= =?UTF-8?q?=20schema=20in=20HudiAvroSerializer=20instead=20=E2=80=A6=20=20?= =?UTF-8?q?(#19483)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(trino): cache decimal Avro schema in HudiAvroSerializer instead of parsing per value AvroDecimalConverter built and JSON-parsed an Avro schema for every decimal cell on the record read path. Cache the schemas by (precision, scale) and build them with LogicalTypes on miss. Also cache buildRecordInPage field positions per record schema instead of doing a name lookup per record per column, and make writeRow's anonymous-field name fallback lazy. Fixes #19361 * address review: replace bit-packed decimal cache key with precision * 100 + scale (cherry picked from commit 2f8a7252c38379f50b8b970f2d841975ddf090a9) --- .../plugin/hudi/util/HudiAvroSerializer.java | 53 +++++-- .../hudi/util/TestHudiAvroSerializer.java | 148 ++++++++++++++++++ 2 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index 0d9bc9f5978b3..fe7ddfe425159 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -42,6 +42,7 @@ import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -59,6 +60,7 @@ import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; @@ -118,6 +120,11 @@ public class HudiAvroSerializer // constructor, which maps page channel i to record position channelToFieldPosition[i]. private final Schema schema; private final int[] channelToFieldPosition; + // Single-entry cache for buildRecordInPage: all records of a split share one schema instance, + // so an identity check makes the per-record, per-column field-name lookup a one-time cost. + // Prefilled (hidden/synthesized) columns are not fields of the record schema; they get -1. + private Schema positionsCacheSchema; + private int[] positionsCache; public HudiAvroSerializer(List columnHandles, PrefilledColumnValues prefilledColumnValues) { @@ -170,21 +177,36 @@ public Object getValue(Page sourcePage, int channel, int position) public void buildRecordInPage(PageBuilder pageBuilder, IndexedRecord record) { pageBuilder.declarePosition(); - int blockSeq = 0; - for (int channel = 0; channel < columnTypes.size(); channel++, blockSeq++) { - BlockBuilder output = pageBuilder.getBlockBuilder(blockSeq); - HiveColumnHandle columnHandle = columnHandles.get(channel); - if (prefilledColumnValues.isPrefilled(columnHandle)) { - prefilledColumnValues.appendTo(columnHandle, output); + // Record may not be projected, get field positions from its own schema + int[] fieldPositions = fieldPositionsFor(record.getSchema()); + for (int channel = 0; channel < columnTypes.size(); channel++) { + BlockBuilder output = pageBuilder.getBlockBuilder(channel); + int fieldPosition = fieldPositions[channel]; + if (fieldPosition < 0) { + prefilledColumnValues.appendTo(columnHandles.get(channel), output); } else { - // Record may not be projected, get index from it - int fieldPosInSchema = getFieldFromSchema(columnHandle.getName(), record.getSchema()).pos(); - appendTo(columnTypes.get(channel), record.get(fieldPosInSchema), output); + appendTo(columnTypes.get(channel), record.get(fieldPosition), output); } } } + private int[] fieldPositionsFor(Schema recordSchema) + { + if (positionsCacheSchema != recordSchema) { + int[] positions = new int[columnHandles.size()]; + for (int channel = 0; channel < columnHandles.size(); channel++) { + HiveColumnHandle columnHandle = columnHandles.get(channel); + positions[channel] = prefilledColumnValues.isPrefilled(columnHandle) + ? -1 + : getFieldFromSchema(columnHandle.getName(), recordSchema).pos(); + } + positionsCache = positions; + positionsCacheSchema = recordSchema; + } + return positionsCache; + } + public static void appendTo(Type type, Object value, BlockBuilder output) { if (value == null) { @@ -493,7 +515,9 @@ private static void writeRow(RowBlockBuilder output, RowType rowType, GenericRec output.buildEntry(fieldBuilders -> { for (int index = 0; index < fields.size(); index++) { RowType.Field field = fields.get(index); - appendTo(field.getType(), record.get(field.getName().orElse("field" + index)), fieldBuilders.get(index)); + int fieldIndex = index; + String fieldName = field.getName().orElseGet(() -> "field" + fieldIndex); + appendTo(field.getType(), record.get(fieldName), fieldBuilders.get(index)); } }); } @@ -524,10 +548,17 @@ private static void writeMap(MapBlockBuilder output, MapType mapType, Map static class AvroDecimalConverter { private static final Conversions.DecimalConversion AVRO_DECIMAL_CONVERSION = new Conversions.DecimalConversion(); + // convert() runs once per decimal cell on the record read path, and building a Schema costs + // orders of magnitude more than the conversion itself. The (precision, scale) space is tiny + // and fixed per column, so cache the schemas globally. + private static final Map DECIMAL_SCHEMAS = new ConcurrentHashMap<>(); BigDecimal convert(int precision, int scale, byte[] bytes) { - Schema schema = new Schema.Parser().parse(format("{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":%d,\"scale\":%d}", precision, scale)); + // The key is unique because precision and scale are at most 38 + Schema schema = DECIMAL_SCHEMAS.computeIfAbsent( + precision * 100 + scale, + key -> LogicalTypes.decimal(precision, scale).addToSchema(Schema.create(Schema.Type.BYTES))); return AVRO_DECIMAL_CONVERSION.fromBytes(ByteBuffer.wrap(bytes), schema, schema.getLogicalType()); } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java new file mode 100644 index 0000000000000..116d50d115e89 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.util; + +import io.trino.metastore.HiveType; +import io.trino.plugin.hive.HiveColumnHandle; +import io.trino.plugin.hive.HivePartitionKey; +import io.trino.plugin.hudi.HudiSplit; +import io.trino.plugin.hudi.file.HudiBaseFile; +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.SplitWeight; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.DecimalType; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Optional; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; + +class TestHudiAvroSerializer +{ + @Test + public void testDecimalConverter() + { + HudiAvroSerializer.AvroDecimalConverter converter = new HudiAvroSerializer.AvroDecimalConverter(); + + assertThat(converter.convert(10, 2, unscaledBytes("123.45"))).isEqualTo(new BigDecimal("123.45")); + // Same (precision, scale) again: served from the cached schema + assertThat(converter.convert(10, 2, unscaledBytes("-0.07"))).isEqualTo(new BigDecimal("-0.07")); + // Same precision, different scale, and vice versa: must not collide in the cache + assertThat(converter.convert(10, 4, unscaledBytes("123.4567"))).isEqualTo(new BigDecimal("123.4567")); + assertThat(converter.convert(18, 2, unscaledBytes("9999999999999999.99"))).isEqualTo(new BigDecimal("9999999999999999.99")); + assertThat(converter.convert(5, 0, unscaledBytes("42"))).isEqualTo(new BigDecimal("42")); + } + + @Test + public void testAppendShortDecimalFromAvroFixed() + { + DecimalType type = DecimalType.createDecimalType(10, 2); + byte[] bytes = unscaledBytes("123.45"); + GenericData.Fixed fixed = new GenericData.Fixed(Schema.createFixed("fix", null, null, bytes.length), bytes); + + BlockBuilder blockBuilder = type.createBlockBuilder(null, 1); + HudiAvroSerializer.appendTo(type, fixed, blockBuilder); + Block block = blockBuilder.build(); + + assertThat(type.getLong(block, 0)).isEqualTo(12345L); + } + + @Test + public void testBuildRecordInPage() + { + // Schema field order (b, a) deliberately differs from projection order (a, b, pk_int), + // so correct output proves positions are resolved from the record's schema. + Schema schema = recordSchema("rec1"); + HudiAvroSerializer serializer = new HudiAvroSerializer(projectedColumns(), prefilledValues()); + PageBuilder pageBuilder = new PageBuilder(List.of(BIGINT, VARCHAR, INTEGER)); + + serializer.buildRecordInPage(pageBuilder, record(schema, 1L, "one")); + // Second record with the same schema instance exercises the cached field positions + serializer.buildRecordInPage(pageBuilder, record(schema, 2L, "two")); + // A schema instance with the opposite field order must invalidate the cache; reusing the + // stale positions would swap the a and b values + serializer.buildRecordInPage(pageBuilder, record(reversedRecordSchema("rec2"), 3L, "three")); + + Page page = pageBuilder.build(); + assertThat(page.getPositionCount()).isEqualTo(3); + for (int position = 0; position < 3; position++) { + assertThat(BIGINT.getLong(page.getBlock(0), position)).isEqualTo(position + 1); + assertThat(INTEGER.getInt(page.getBlock(2), position)).isEqualTo(42); + } + assertThat(VARCHAR.getSlice(page.getBlock(1), 0).toStringUtf8()).isEqualTo("one"); + assertThat(VARCHAR.getSlice(page.getBlock(1), 1).toStringUtf8()).isEqualTo("two"); + assertThat(VARCHAR.getSlice(page.getBlock(1), 2).toStringUtf8()).isEqualTo("three"); + } + + private static byte[] unscaledBytes(String decimal) + { + return new BigDecimal(decimal).unscaledValue().toByteArray(); + } + + private static Schema recordSchema(String name) + { + return SchemaBuilder.record(name).fields() + .name("b").type().stringType().noDefault() + .name("a").type().longType().noDefault() + .endRecord(); + } + + private static Schema reversedRecordSchema(String name) + { + return SchemaBuilder.record(name).fields() + .name("a").type().longType().noDefault() + .name("b").type().stringType().noDefault() + .endRecord(); + } + + private static GenericData.Record record(Schema schema, long a, String b) + { + GenericData.Record record = new GenericData.Record(schema); + record.put("a", a); + record.put("b", b); + return record; + } + + private static List projectedColumns() + { + return List.of( + HiveColumnHandle.createBaseColumn("a", 0, HiveType.HIVE_LONG, BIGINT, HiveColumnHandle.ColumnType.REGULAR, Optional.empty()), + HiveColumnHandle.createBaseColumn("b", 1, HiveType.HIVE_STRING, VARCHAR, HiveColumnHandle.ColumnType.REGULAR, Optional.empty()), + HiveColumnHandle.createBaseColumn("pk_int", -1, HiveType.HIVE_INT, INTEGER, HiveColumnHandle.ColumnType.PARTITION_KEY, Optional.empty())); + } + + private static PrefilledColumnValues prefilledValues() + { + HudiBaseFile baseFile = new HudiBaseFile("s3://bucket/table/file1.parquet", "file1.parquet", 1234, 1700000000123L, 0, 1234); + HudiSplit split = new HudiSplit( + baseFile, + List.of(), + "001", + TupleDomain.all(), + List.of(new HivePartitionKey("pk_int", "42")), + SplitWeight.standard()); + return PrefilledColumnValues.create(split); + } +} From b55a86a161914925ace650a987433a6fa2a15f5b Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 3 Aug 2026 10:50:04 -0700 Subject: [PATCH 232/255] fix(meta-sync): advance last commit time synced when it trails the active timeline midpoint (#19239) * fix(meta-sync): advance last commit time synced when it trails the active timeline midpoint * docs(meta-sync): clarify the marker advances on catalog-visible change, not on data write * docs(meta-sync): say "last commit time synced" instead of "marker" in the javadoc * test(meta-sync): unit-test the timeline-midpoint helper and take the midpoint over completed commits Compute the midpoint over completed commit instants only so an inflight instant cannot shift it, and add TestHiveSyncToolTimelineMidpoint covering the present/empty guards and the inflight-vs-completed boundary. * test(meta-sync): narrow the timeline midpoint to completed commits Use getCommitsTimeline().filterCompletedInstants() so clean, rollback, and other non-commit instants cannot shift the midpoint, and add a test that pins the narrowing to the commits timeline. * refactor(meta-sync): read completed commits via metaClient.getCommitsTimeline() Keep the helper and its test identical across lines by reading the commits timeline from metaClient.getCommitsTimeline(), which is available everywhere, rather than the HoodieTimeline interface method. * test(meta-sync): rename midpoint test for clarity Rename midpointIsTakenOverCompletedCommitsOnly to midpointIsComputedFromCompletedCommitsOnly. (cherry picked from commit 99ceae1b645967dd39c5aee0ed6e904b363027a1) --- .../org/apache/hudi/hive/HiveSyncTool.java | 28 ++++- .../apache/hudi/hive/TestHiveSyncTool.java | 12 +- .../TestHiveSyncToolTimelineMidpoint.java | 114 ++++++++++++++++++ 3 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolTimelineMidpoint.java diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 4abfa5b2ec8cf..62c349b4ae0b0 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.InvalidTableException; import org.apache.hudi.sync.common.HoodieSyncClient; @@ -51,6 +52,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN; +import static org.apache.hudi.common.table.timeline.InstantComparison.compareTimestamps; import static org.apache.hudi.common.util.StringUtils.nonEmpty; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getInputFormatClassName; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getOutputFormatClassName; @@ -275,7 +278,8 @@ protected void syncHoodieTable(String tableName, boolean useRealtimeInputFormat, boolean partitionsChanged = validateAndSyncPartitions(tableName, tableExists); boolean meetSyncConditions = schemaChanged || propertiesChanged || partitionsChanged; - if (!config.getBoolean(META_SYNC_CONDITIONAL_SYNC) || meetSyncConditions) { + if (!config.getBoolean(META_SYNC_CONDITIONAL_SYNC) || meetSyncConditions + || isLastCommitTimeSyncedBehindTimelineMidpoint(tableName)) { syncClient.updateLastCommitTimeSynced(tableName); } syncClient.updateHoodieWriterVersion(tableName); @@ -290,6 +294,28 @@ protected void syncHoodieTable(String tableName, boolean useRealtimeInputFormat, } } + /** + * Whether last commit time synced trails the midpoint of the completed commit instants. + * Advancing it at the midpoint bounds how far it can fall behind, capping the archived-timeline + * scans a stale value forces on every conditional-sync round. + */ + @VisibleForTesting + boolean isLastCommitTimeSyncedBehindTimelineMidpoint(String tableName) { + Option lastCommitTimeSynced = syncClient.getLastCommitTimeSynced(tableName); + if (!lastCommitTimeSynced.isPresent()) { + return false; + } + // Completed commits only: getCommitsTimeline() excludes non-commit actions (clean, rollback), + // and filterCompletedInstants() excludes inflight instants. + List completedCommits = + syncClient.getMetaClient().getCommitsTimeline().filterCompletedInstants().getInstants(); + if (completedCommits.isEmpty()) { + return false; + } + String midpointInstantTime = completedCommits.get(completedCommits.size() / 2).requestedTime(); + return compareTimestamps(lastCommitTimeSynced.get(), LESSER_THAN, midpointInstantTime); + } + private boolean isAlreadySynced(String tableName) { return syncClient.getLastCommitTimeSynced(tableName) .map(lastCommit -> { diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index 988b94a5a56bb..93ccc45a34a4d 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -2500,12 +2500,16 @@ public void testSyncWithoutDiffs(String syncMode) throws Exception { HiveTestUtil.addMORPartitions(0, true, true, true, ZonedDateTime.now().plusDays(2), commitTime2, commitTime3); + // No sync condition is met, but the sync marker trails the midpoint of the active commits + // timeline ([100, 101, 102, 103] with midpoint 102), so it advances to the last commit. + reInitHiveSyncClient(); reSyncHiveTable(); - assertEquals(commitTime1, hiveClient.getLastCommitTimeSynced(tableName).get()); + assertEquals(commitTime3, hiveClient.getLastCommitTimeSynced(tableName).get()); // Let the last commit time synced to be before the start of the active timeline, - // to trigger the fallback of listing all partitions. There is no partition change - // and the last commit time synced should still be the same. + // to trigger the fallback of listing all partitions. There is no partition change, + // and the sync marker again trails the timeline midpoint, so it advances to the + // last commit instead of aging out further. HiveTestUtil.addMORPartitions(0, true, true, true, ZonedDateTime.now().plusDays(2), commitTime4, commitTime5); HiveTestUtil.removeCommitFromActiveTimeline(commitTime0, COMMIT_ACTION); HiveTestUtil.removeCommitFromActiveTimeline(commitTime1, DELTA_COMMIT_ACTION); @@ -2513,7 +2517,7 @@ public void testSyncWithoutDiffs(String syncMode) throws Exception { HiveTestUtil.removeCommitFromActiveTimeline(commitTime3, DELTA_COMMIT_ACTION); reInitHiveSyncClient(); reSyncHiveTable(); - assertEquals(commitTime1, hiveClient.getLastCommitTimeSynced(tableName).get()); + assertEquals(commitTime5, hiveClient.getLastCommitTimeSynced(tableName).get()); } @ParameterizedTest diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolTimelineMidpoint.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolTimelineMidpoint.java new file mode 100644 index 0000000000000..3d0b5b3dd4311 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolTimelineMidpoint.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive; + +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.MockHoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.sync.common.HoodieSyncClient; + +import org.junit.jupiter.api.Test; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link HiveSyncTool#isLastCommitTimeSyncedBehindTimelineMidpoint}, which decides + * whether a no-change conditional sync should still advance last commit time synced. The helper + * reads the completed commits timeline, so these mock {@code getMetaClient().getCommitsTimeline()}; + * a call to any other timeline would hit an unstubbed mock and fail. + */ +class TestHiveSyncToolTimelineMidpoint { + + private static final String TABLE_NAME = "table"; + + @Test + void midpointIsComputedFromCompletedCommitsOnly() { + // Completed commits [100, 102, 104], midpoint 102; the later inflight 106 must not shift it to 104. + HoodieSyncClient syncClient = mockSyncClient(new MockHoodieTimeline(Stream.of("100", "102", "104"), Stream.of("106"))); + HiveSyncTool tool = toolWith(syncClient); + + // Not synced yet: nothing to advance. + stubLastCommitTimeSynced(syncClient, Option.empty()); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + + // Trails the midpoint. + stubLastCommitTimeSynced(syncClient, Option.of("101")); + assertTrue(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + + // At the midpoint is not behind it. + stubLastCommitTimeSynced(syncClient, Option.of("102")); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + + // Past 102 but below 104: behind only if the inflight 106 is wrongly counted. + stubLastCommitTimeSynced(syncClient, Option.of("103")); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + } + + @Test + void midpointHandlesSmallTimelines() { + // Size 1: the sole commit is the midpoint. + HoodieSyncClient syncClient = mockSyncClient(new MockHoodieTimeline(Stream.of("101"), Stream.empty())); + HiveSyncTool tool = toolWith(syncClient); + stubLastCommitTimeSynced(syncClient, Option.of("100")); + assertTrue(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + stubLastCommitTimeSynced(syncClient, Option.of("101")); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + + // Size 2: the midpoint (index 1) is the newer commit. + syncClient = mockSyncClient(new MockHoodieTimeline(Stream.of("100", "102"), Stream.empty())); + tool = toolWith(syncClient); + stubLastCommitTimeSynced(syncClient, Option.of("101")); + assertTrue(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + stubLastCommitTimeSynced(syncClient, Option.of("102")); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + } + + @Test + void emptyCommitsTimelineIsNotBehind() { + HoodieSyncClient syncClient = mockSyncClient(new MockHoodieTimeline(Stream.empty(), Stream.empty())); + HiveSyncTool tool = toolWith(syncClient); + stubLastCommitTimeSynced(syncClient, Option.of("103")); + assertFalse(tool.isLastCommitTimeSyncedBehindTimelineMidpoint(TABLE_NAME)); + } + + private static HoodieSyncClient mockSyncClient(HoodieTimeline commitsTimeline) { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + when(metaClient.getCommitsTimeline()).thenReturn(commitsTimeline); + HoodieSyncClient syncClient = mock(HoodieSyncClient.class); + when(syncClient.getMetaClient()).thenReturn(metaClient); + return syncClient; + } + + private static HiveSyncTool toolWith(HoodieSyncClient syncClient) { + HiveSyncTool tool = mock(HiveSyncTool.class, CALLS_REAL_METHODS); + tool.syncClient = syncClient; + return tool; + } + + private static void stubLastCommitTimeSynced(HoodieSyncClient syncClient, Option value) { + when(syncClient.getLastCommitTimeSynced(TABLE_NAME)).thenReturn(value); + } +} From 37f7d888b33bad4e00f76bd27fcce779852f59d0 Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Tue, 4 Aug 2026 00:17:36 +0530 Subject: [PATCH 233/255] fix(fs): stop depending on the optional FileSystem#getScheme() (#19470) * fix(fs): stop depending on the optional FileSystem#getScheme() FileSystem#getScheme() is optional in Hadoop: the base implementation throws UnsupportedOperationException, and proxy implementations such as Presto's PrestoS3FileSystem do not override it. Hudi called it unguarded on filesystems it did not implement, so opening a log file on such a filesystem failed with "Not implemented by the PrestoS3FileSystem FileSystem implementation" instead of reading anything (HUDI-4602). Adds HadoopFSUtils#getScheme(FileSystem), which returns fs.getScheme() and falls back to fs.getUri().getScheme() when it is unimplemented. getUri() is abstract, so every implementation supplies it, and its scheme is what getScheme() returns wherever both are present. This is the same conclusion as #793, which stopped HoodieWrapperFileSystem calling getScheme() on the filesystem it wraps. Routes the seven unguarded call sites through it: isGCSFileSystem and isCHDFileSystem (the reported read path), registerFileSystem, HoodieWrapperFileSystem#convertToHoodiePath, HoodieRetryWrapperFileSystem#getScheme, WriteMarkersFactory's HDFS gate, and HoodieHadoopStorage#getScheme, which is what the seven HoodieStorage#getScheme callers reach. isGCSFileSystem's comparison is also flipped to put the constant first, matching isCHDFileSystem, so a filesystem whose URI carries no scheme returns false rather than throwing NullPointerException. * test(fs): say which branch of the helper each assertion covers Review nit: the assertion messages did not make clear what had gone wrong. Each now names the filesystem and the branch of the helper it pins - LocalFileSystem overriding getScheme() so the helper returns what it reports, FilterFileSystem not overriding it so the helper falls back to getUri().getScheme(). * fix(fs): fail loudly on an unresolvable scheme, and cover the sites this reroutes Review feedback, all of it well founded. The fallback no longer returns null. InLineFileSystem is the counter-example in this module: getScheme() is "inlinefs" while getUri() is URI.create("inlinefs"), which has no colon and so no scheme, so the two are not interchangeable and the javadoc claim that they agree was simply wrong. A null surfaced far from the cause as "does not support scheme null" or "Unsupported scheme :null" with the UnsupportedOperationException discarded; it now throws with that exception chained. HoodieException rather than HoodieIOException, since the latter only accepts an IOException cause. HoodieHadoopStorage memoizes the scheme. On a filesystem without getScheme() the fallback costs a thrown-and-caught exception, and this is called once per log block via StorageSchemes.isWriteTransactional and three times per immutable-file write via needCreateTempFile. A lazy field keeps all five constructors untouched. Test coverage for what this actually reroutes, none of which any test reached: - registerFileSystem, HoodieWrapperFileSystem#convertToHoodiePath (the write path) and HoodieHadoopStorage#getScheme, via a LocalFileSystem subclass whose getScheme() throws, registered as fs.file.impl so it is reached through FileSystem.get. - isGCSFileSystem and isCHDFileSystem, which become reachable for proxy filesystems for the first time here and select different stream wrappers: a scheme-less filesystem reporting gs:// now yields SchemeAwareFSDataInputStream and ofs:// yields BoundedFsDataInputStream. - the new unresolvable-scheme failure. TestFSUtilsWithRetryWrapperEnable#testGetSchema has been inert since HUDI-5286 added it: it asserted on HoodieWrapperFileSystem#getScheme, which is uri.getScheme() and never dispatches into the retry wrapper, and FakeRemoteFileSystem overrode getScheme() to delegate to a real LocalFileSystem so it could not throw. Dropping that override gives the fake the PrestoS3FileSystem shape and the assertion now targets the retry wrapper, so it guards both HUDI-5286 and this change. Verified: it fails with the pre-PR helper. Also drops the try/catch in convertToHoodiePath that only rethrew HoodieIOException unchanged, dead since ef70de2bba7b, and the duplicated fixture and redundant nested close in TestHadoopFSUtils. (cherry picked from commit 70a5a4dbd81c0b73b32db577aa16dbadcca8ace6) --- .../table/marker/WriteMarkersFactory.java | 2 +- .../apache/hudi/hadoop/fs/HadoopFSUtils.java | 42 ++++- .../fs/HoodieRetryWrapperFileSystem.java | 2 +- .../hadoop/fs/HoodieWrapperFileSystem.java | 8 +- .../storage/hadoop/HoodieHadoopStorage.java | 14 +- .../fs/TestFSUtilsWithRetryWrapperEnable.java | 16 +- .../hudi/hadoop/fs/TestHadoopFSUtils.java | 163 ++++++++++++++++++ 7 files changed, 226 insertions(+), 21 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java index 2765ffdd6286a..8191e6d04cba4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java @@ -53,7 +53,7 @@ public static WriteMarkers get(MarkerType markerType, HoodieTable table, String } String basePath = table.getMetaClient().getBasePath().toString(); if (StorageSchemes.HDFS.getScheme().equals( - HadoopFSUtils.getFs(basePath, table.getContext().getStorageConf(), true).getScheme())) { + HadoopFSUtils.getScheme(HadoopFSUtils.getFs(basePath, table.getContext().getStorageConf(), true)))) { log.warn("Timeline-server-based markers are not supported for HDFS: " + "base path {}. Falling back to direct markers.", basePath); return getDirectWriteMarkers(table, instantTime); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java index 2700405073078..3eb5f15148ec6 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.InvalidHoodiePathException; import org.apache.hudi.storage.StorageConfiguration; @@ -284,7 +285,7 @@ private static FSDataInputStream getFSDataInputStreamForGCS(FSDataInputStream fs * @return true if the inputstream or the wrapped one is of type GoogleHadoopFSInputStream */ public static boolean isGCSFileSystem(FileSystem fs) { - return fs.getScheme().equals(StorageSchemes.GCS.getScheme()); + return StorageSchemes.GCS.getScheme().equals(getScheme(fs)); } /** @@ -292,7 +293,42 @@ public static boolean isGCSFileSystem(FileSystem fs) { * Wrapped by {@code BoundedFsDataInputStream}, to check whether the desired offset is out of the file size in advance. */ public static boolean isCHDFileSystem(FileSystem fs) { - return StorageSchemes.CHDFS.getScheme().equals(fs.getScheme()); + return StorageSchemes.CHDFS.getScheme().equals(getScheme(fs)); + } + + /** + * Resolves the scheme of {@code fs} without depending on {@link FileSystem#getScheme()}. + * + *

    {@code getScheme()} is optional in Hadoop: {@link FileSystem}'s own implementation throws + * {@link UnsupportedOperationException}, and proxy implementations such as Presto's + * {@code PrestoS3FileSystem} do not override it, so calling it unguarded turns an unrelated read into + * "Not implemented by the PrestoS3FileSystem FileSystem implementation" (HUDI-4602). + * {@link FileSystem#getUri()} is abstract, so every implementation supplies one to fall back on. + * + *

    The two are not interchangeable, which is why {@code getScheme()} is tried first: + * {@code InLineFileSystem} returns {@code "inlinefs"} from {@code getScheme()} while its + * {@code getUri()} is {@code URI.create("inlinefs")}, which has no colon and so carries no scheme at all. + * A URI with no scheme is therefore a resolution failure rather than a value to pass on - returning null + * would surface much later as {@code does not support scheme null} or {@code Unsupported scheme :null}, + * with the original {@code UnsupportedOperationException} discarded. + * + * @param fs instance of {@link FileSystem} in use. + * @return the scheme of {@code fs}, never null. + * @throws HoodieException if {@code getScheme()} is unimplemented and the URI carries no scheme. + */ + public static String getScheme(FileSystem fs) { + try { + return fs.getScheme(); + } catch (UnsupportedOperationException e) { + String scheme = fs.getUri().getScheme(); + if (scheme == null) { + // HoodieException rather than HoodieIOException: the latter only accepts an IOException cause, and + // discarding the UnsupportedOperationException is the thing being fixed here. + throw new HoodieException("Cannot resolve the scheme of " + fs.getClass().getName() + + ": getScheme() is unimplemented and its URI " + fs.getUri() + " carries no scheme", e); + } + return scheme; + } } private static StorageConfiguration getStorageConf(Configuration conf, boolean copy) { @@ -301,7 +337,7 @@ private static StorageConfiguration getStorageConf(Configuration public static Configuration registerFileSystem(StoragePath file, Configuration conf) { Configuration returnConf = new Configuration(conf); - String scheme = HadoopFSUtils.getFs(file.toString(), conf).getScheme(); + String scheme = getScheme(HadoopFSUtils.getFs(file.toString(), conf)); returnConf.set("fs." + HoodieWrapperFileSystem.getHoodieScheme(scheme) + ".impl", HoodieWrapperFileSystem.class.getName()); return returnConf; diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieRetryWrapperFileSystem.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieRetryWrapperFileSystem.java index c9d8fff3fbbd7..d7c1ca5f72fc8 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieRetryWrapperFileSystem.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieRetryWrapperFileSystem.java @@ -277,7 +277,7 @@ public Configuration getConf() { @Override public String getScheme() { - return fileSystem.getScheme(); + return HadoopFSUtils.getScheme(fileSystem); } @Override diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieWrapperFileSystem.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieWrapperFileSystem.java index 24674ee725a90..f8d731fd0ce21 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieWrapperFileSystem.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieWrapperFileSystem.java @@ -159,12 +159,8 @@ public HoodieWrapperFileSystem(FileSystem fileSystem, ConsistencyGuard consisten } public static Path convertToHoodiePath(StoragePath file, Configuration conf) { - try { - String scheme = HadoopFSUtils.getFs(file.toString(), conf).getScheme(); - return convertPathWithScheme(convertToHadoopPath(file), getHoodieScheme(scheme)); - } catch (HoodieIOException e) { - throw e; - } + String scheme = HadoopFSUtils.getScheme(HadoopFSUtils.getFs(file.toString(), conf)); + return convertPathWithScheme(convertToHadoopPath(file), getHoodieScheme(scheme)); } public static Path convertPathWithScheme(Path oldPath, String newScheme) { diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java index 87a9ad1019f63..067f77d491bd6 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java @@ -20,6 +20,7 @@ package org.apache.hudi.storage.hadoop; import org.apache.hudi.common.fs.ConsistencyGuard; +import org.apache.hudi.common.util.Lazy; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.hadoop.fs.HoodieRetryWrapperFileSystem; @@ -58,6 +59,13 @@ */ public class HoodieHadoopStorage extends HoodieStorage { private final FileSystem fs; + /** + * Resolved once. On a filesystem that does not implement {@code getScheme()} the fallback in + * {@link HadoopFSUtils#getScheme} costs a thrown-and-caught exception, and this is called once per log + * block via {@code StorageSchemes.isWriteTransactional} and three times per immutable-file write via + * {@code needCreateTempFile}. {@code fs} is final, so the answer cannot change. + */ + private final Lazy scheme = Lazy.lazily(this::resolveScheme); public HoodieHadoopStorage(StoragePath path, StorageConfiguration conf) { super(conf); @@ -111,7 +119,11 @@ public HoodieStorage newInstance(StoragePath path, StorageConfiguration stora @Override public String getScheme() { - return fs.getScheme(); + return scheme.get(); + } + + private String resolveScheme() { + return HadoopFSUtils.getScheme(fs); } @Override diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java index bb0b3608c7fc8..aaaf749e85d46 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java @@ -45,7 +45,6 @@ import java.util.Arrays; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -107,9 +106,13 @@ public void testGetSchema() { FileSystem fileSystem = new HoodieRetryWrapperFileSystem(fakeFs, maxRetryIntervalMs, maxRetryNumbers, initialRetryIntervalMs, ""); - HoodieWrapperFileSystem fs = - new HoodieWrapperFileSystem(fileSystem, new NoOpConsistencyGuard()); - assertDoesNotThrow(fs::getScheme, "Method #getSchema does not implement correctly"); + // FakeRemoteFileSystem deliberately does not override getScheme(), so FileSystem's own implementation + // throws - the PrestoS3FileSystem shape (HUDI-4602). Assert on the retry wrapper itself: asserting on + // HoodieWrapperFileSystem instead would only exercise its own uri.getScheme() and never reach here, + // which is why this guard was inert from the day HUDI-5286 added it. + assertThrows(UnsupportedOperationException.class, fakeFs::getScheme); + assertEquals("file", ((HoodieRetryWrapperFileSystem) fileSystem).getScheme(), + "the retry wrapper should resolve the scheme of a filesystem that does not implement getScheme()"); } @Test @@ -254,11 +257,6 @@ public Configuration getConf() { return fs.getConf(); } - @Override - public String getScheme() { - return fs.getScheme(); - } - @Override public short getDefaultReplication(Path path) { return defaultReplication; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/hadoop/fs/TestHadoopFSUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/hadoop/fs/TestHadoopFSUtils.java index 7768ff4feae7f..da4e9a7500f42 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/hadoop/fs/TestHadoopFSUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/hadoop/fs/TestHadoopFSUtils.java @@ -19,25 +19,188 @@ package org.apache.hudi.hadoop.fs; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import org.apache.hudi.storage.hadoop.HoodieHadoopStorage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; + import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToHadoopFileStatus; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToHadoopPath; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePathInfo; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests {@link HadoopFSUtils} */ public class TestHadoopFSUtils { + /** + * HUDI-4602: {@link FileSystem#getScheme()} is optional in Hadoop -- the base implementation throws + * {@link UnsupportedOperationException} -- and proxy implementations such as Presto's + * {@code PrestoS3FileSystem} do not override it. Opening a log file went straight through + * {@code isGCSFileSystem}, so a MOR {@code _rt} query on Presto failed with + * "Not implemented by the PrestoS3FileSystem FileSystem implementation" rather than reading anything. + * + *

    {@link FilterFileSystem} has the same shape: it leaves {@code getScheme()} to the throwing base + * implementation while overriding {@code getUri()}. + */ + @Test + public void testGetFSDataInputStreamWhenGetSchemeIsUnimplemented(@TempDir java.nio.file.Path tempDir) throws IOException { + java.nio.file.Path file = tempDir.resolve("log.file"); + byte[] contents = new byte[] {1, 2, 3, 4}; + Files.write(file, contents); + // newInstanceLocal rather than getLocal, so closing this does not evict a cached FileSystem that + // other tests in the same JVM share. + try (FileSystem fs = newFsWithoutGetScheme(FileSystem.newInstanceLocal(new Configuration()))) { + try (FSDataInputStream stream = + HadoopFSUtils.getFSDataInputStream(fs, new StoragePath(file.toUri()), 1024, true)) { + byte[] read = new byte[contents.length]; + stream.readFully(read); + assertArrayEquals(contents, read, "The read path should not depend on the optional getScheme()"); + } + } + } + + @Test + public void testGetSchemeFallsBackToTheUriWhenUnimplemented() throws IOException { + try (FileSystem localFs = FileSystem.newInstanceLocal(new Configuration())) { + assertEquals("file", HadoopFSUtils.getScheme(localFs), + "LocalFileSystem overrides getScheme(), so the helper should return what it reports " + + "rather than falling back to getUri()"); + + // FilterFileSystem#close closes the delegate, so the wrapper is not given its own block: it owns + // nothing, and closing it here would close localFs a second time. + FileSystem noScheme = newFsWithoutGetScheme(localFs); + assertEquals("file", HadoopFSUtils.getScheme(noScheme), + "FilterFileSystem does not override getScheme(), so the helper should fall back to " + + "getUri().getScheme()"); + } + } + + /** + * A URI with no scheme cannot stand in for an unimplemented {@code getScheme()}. {@code InLineFileSystem} + * is the case in this module: {@code getScheme()} returns "inlinefs" while {@code getUri()} is + * {@code URI.create("inlinefs")}, which has no colon and so no scheme. Returning null there would surface + * far away as "does not support scheme null" with the original failure discarded, so it must fail here. + */ + @Test + public void testGetSchemeFailsLoudlyWhenNeitherSourceHasOne() throws IOException { + try (FileSystem localFs = FileSystem.newInstanceLocal(new Configuration())) { + FileSystem schemeless = new NoSchemeFileSystem(localFs, URI.create("inlinefs")); + + HoodieException thrown = + assertThrows(HoodieException.class, () -> HadoopFSUtils.getScheme(schemeless)); + assertTrue(thrown.getMessage().contains("carries no scheme"), + () -> "the failure should say the URI carries no scheme, but was: " + thrown.getMessage()); + assertInstanceOf(UnsupportedOperationException.class, thrown.getCause(), + "the original getScheme() failure must be chained rather than discarded"); + } + } + + /** + * The three call sites this rerouted that no test in the repo reached: {@code registerFileSystem}, + * {@code HoodieWrapperFileSystem#convertToHoodiePath} - which is on the write path, via + * {@code HoodieBaseParquetWriter} and friends - and {@code HoodieHadoopStorage#getScheme}. All three threw + * {@link UnsupportedOperationException} on a filesystem without {@code getScheme()} before this change. + */ + @Test + public void testCallSitesWorkOnAFileSystemWithoutGetScheme(@TempDir java.nio.file.Path tempDir) { + Configuration conf = new Configuration(); + conf.setClass("fs.file.impl", NoSchemeLocalFileSystem.class, FileSystem.class); + StoragePath path = new StoragePath(tempDir.toUri()); + + assertDoesNotThrow(() -> HadoopFSUtils.registerFileSystem(path, conf), + "registerFileSystem resolves the scheme to build the fs..impl key"); + assertDoesNotThrow(() -> HoodieWrapperFileSystem.convertToHoodiePath(path, conf), + "convertToHoodiePath is on the write path and resolves the scheme to rewrite it"); + assertEquals("file", new HoodieHadoopStorage(path, HadoopFSUtils.getStorageConf(conf)).getScheme(), + "HoodieHadoopStorage#getScheme is what HoodieStorage callers reach"); + } + + /** + * {@code isGCSFileSystem} and {@code isCHDFileSystem} become reachable for a filesystem without + * {@code getScheme()} for the first time with this change, and they select different stream wrappers. + * Neither predicate had a test before. + */ + @ParameterizedTest + @CsvSource({ + "gs://bucket, org.apache.hudi.hadoop.fs.SchemeAwareFSDataInputStream", + "ofs://cluster, org.apache.hudi.hadoop.fs.BoundedFsDataInputStream" + }) + public void testSchemeSpecificStreamIsSelectedWithoutGetScheme(String uri, String expectedStream, + @TempDir java.nio.file.Path tempDir) throws IOException { + java.nio.file.Path file = tempDir.resolve("log.file"); + Files.write(file, new byte[] {1, 2, 3, 4}); + try (FileSystem localFs = FileSystem.newInstanceLocal(new Configuration())) { + // Reports a gs:// or ofs:// URI while leaving getScheme() to the throwing base implementation. + FileSystem fs = new NoSchemeFileSystem(localFs, URI.create(uri)); + assertThrows(UnsupportedOperationException.class, fs::getScheme); + + try (FSDataInputStream stream = + HadoopFSUtils.getFSDataInputStream(fs, new StoragePath(file.toUri()), 1024, true)) { + assertEquals(expectedStream, stream.getClass().getName(), + "the scheme-specific wrapper should be selected from the fallback-resolved scheme"); + } + } + } + + /** A FileSystem with the reported shape: {@code getUri()} works, {@code getScheme()} throws. */ + private static FileSystem newFsWithoutGetScheme(FileSystem delegate) { + FileSystem fs = new FilterFileSystem(delegate); + // The premise of every assertion below: this is the call the read path used to make unguarded. + assertThrows(UnsupportedOperationException.class, fs::getScheme); + return fs; + } + + /** Same shape, but reporting a URI of our choosing so scheme-specific branches can be reached. */ + private static class NoSchemeFileSystem extends FilterFileSystem { + private final URI uri; + + NoSchemeFileSystem(FileSystem delegate, URI uri) { + super(delegate); + this.uri = uri; + } + + @Override + public URI getUri() { + return uri; + } + } + + /** + * A {@link LocalFileSystem} that does not implement {@code getScheme()}, so it can be registered as + * {@code fs.file.impl} and reached through the normal {@code FileSystem.get} path. + */ + public static class NoSchemeLocalFileSystem extends LocalFileSystem { + @Override + public String getScheme() { + throw new UnsupportedOperationException( + "Not implemented by the NoSchemeLocalFileSystem FileSystem implementation"); + } + } + @ParameterizedTest @ValueSource(strings = { "/a/b/c", From 2df42dee8b4eac74686ac7a7d3b86cfd8d3e0082 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Mon, 3 Aug 2026 21:38:48 -0700 Subject: [PATCH 234/255] fix(schema): require a per-field override to promote a bare long to a timestamp logical type (#19384) Promoting a bare long column to a timestamp logical type during writer-schema deduction is now uniformly gated behind the per-field override hoodie.write.timestamp.logical.type.overrides: rejected with an actionable error when the field has no override, and applied when it does. This holds for all four target types (timestamp-micros, timestamp-millis, local-timestamp-micros, local-timestamp-millis) and in both reconcile paths (reconcileSchema and reconcileTimestampLogicalType). Previously, long to local-timestamp was already override-gated, but long to UTC-timestamp was silently allowed on the default (non-reconcile) write path, because isGatedTimestampChange did not treat it as a gated change. A bare long carries no precision signal (millis vs micros), so silently attaching a UTC timestamp logical type could mislabel stored values. This makes the two cases behave identically. Timestamp precision flips are unchanged. timestampPrecisionChangeError is made public so tests assert the exact message without duplicating its format. Tests: TestSchemaChangeUtils and TestAvroSchemaEvolutionUtils cover the gating on both reconcile paths, with and without an override, for all four targets including nested fields; TestHoodieDeltaStreamer.testLongToTimestampPromotionGated exercises the promotion end to end. (cherry picked from commit e4718be307900b5348f26bc113885f224c33896c) --- .../common/config/HoodieCommonConfig.java | 7 +- .../utils/AvroSchemaEvolutionUtils.java | 7 +- .../schema/utils/SchemaChangeUtils.java | 13 +-- .../utils/TestAvroSchemaEvolutionUtils.java | 100 ++++++++++++++++++ .../schema/utils/TestSchemaChangeUtils.java | 29 +++++ .../TestHoodieDeltaStreamer.java | 98 +++++++++++++++++ 6 files changed, 244 insertions(+), 10 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java index 13a3e2e6e480e..0eaf9533a7373 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java @@ -94,9 +94,10 @@ public class HoodieCommonConfig extends HoodieConfig { + "entry is pinned to that logical type: an incoming value of a different precision is coerced to it, and " + "the change from the table's current type is permitted. A timestamp precision change with no entry for " + "the field is rejected with an error, so an unverified micros/millis flip can never happen silently. " - + "An entry also attaches a local-timestamp logical type to a column that 0.x persisted as a bare long " - + "because its converter did not recognize the type. A UTC/local zone change is never authorized by " - + "this config, whatever the entry says, since no rescale can express it. " + + "An entry also attaches a timestamp logical type (UTC or local) to a column persisted as a bare " + + "long, including one that 0.x stored without a logical type because its converter did not recognize " + + "it. A UTC/local zone change is never authorized by this config, whatever the entry says, since no " + + "rescale can express it. " + "Derive the value from the stored longs, never from the incoming schema: for instants after 1990 an " + "epoch-millis value is around 1e12 while epoch-micros is around 1e15, so the two ranges do not " + "overlap. TimestampLogicalTypeClassifier implements that verdict for inspection tooling to reuse. " diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java index fd6084b201a34..19bb29742edc7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java @@ -277,7 +277,12 @@ private static SchemaCompatibilityException crossZoneTimestampChangeError(String col, from, to, HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key())); } - private static SchemaCompatibilityException timestampPrecisionChangeError(String col, Type from, Type to) { + /** + * Builds the actionable error for a gated timestamp logical-type change with no per-field override + * in {@code hoodie.write.timestamp.logical.type.overrides}. Public so tests can assert the exact + * message without duplicating its format. + */ + public static SchemaCompatibilityException timestampPrecisionChangeError(String col, Type from, Type to) { return new SchemaCompatibilityException(String.format( "Refusing to change the timestamp logical type of column '%s' from '%s' to '%s' without an explicit " + "verdict. This precision change is not applied automatically because the correct target depends " diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java index e63165a1a295a..5c14e16bc5535 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java @@ -96,8 +96,8 @@ private static Type timestampTypeFromToken(String token) { /** * Whether a column type change is a timestamp precision change that must be authorized by an * explicit per-field override (see {@code hoodie.write.timestamp.logical.type.overrides}). This - * covers timestamp-micros/millis flips, local-timestamp-micros/millis flips, and the forward-fix - * from a bare {@code long} to a local-timestamp logical type that 0.x dropped. + * covers timestamp-micros/millis flips, local-timestamp-micros/millis flips, and promoting a bare + * {@code long} to a timestamp logical type (UTC or local). */ public static boolean isGatedTimestampChange(Type src, Type dst) { if (src.equals(dst)) { @@ -109,7 +109,7 @@ public static boolean isGatedTimestampChange(Type src, Type dst) { if (isLocalTimestamp(src) && isLocalTimestamp(dst)) { return true; } - return src.typeId() == Type.TypeID.LONG && isLocalTimestamp(dst); + return src.typeId() == Type.TypeID.LONG && (isUtcTimestamp(dst) || isLocalTimestamp(dst)); } /** @@ -170,9 +170,10 @@ private static boolean isTypeUpdateAllowInternal(Type src, Type dst, boolean all || dst == Types.DoubleType.get() || dst == Types.StringType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED; case LONG: if (allowTimestampPrecisionEvolution - && (dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MILLIS || dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MICROS)) { - // Forward-fix path: 0.x stored local-timestamp columns as bare long because its converter - // did not recognize the logical type. Allow attaching the logical type when the gate is open. + && (dst.typeId() == Type.TypeID.TIMESTAMP || dst.typeId() == Type.TypeID.TIMESTAMP_MILLIS + || dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MILLIS || dst.typeId() == Type.TypeID.LOCAL_TIMESTAMP_MICROS)) { + // A bare long carries no precision signal, so promoting it to a timestamp logical type is + // authorized only by an explicit per-field override. return true; } return dst == Types.FloatType.get() || dst == Types.DoubleType.get() || dst == Types.StringType.get() || dst.typeId() == Type.TypeID.DECIMAL || dst.typeId() == Type.TypeID.DECIMAL_FIXED; diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java index b4e476ce1e07e..e33efec9e3fde 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -840,4 +841,103 @@ public void testCrossZoneTimestampChangeIsRejected() { SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-millis")).toAvroSchema(); Assertions.assertEquals("local-timestamp-millis", stillWorks.getField("ts").schema().getLogicalType().getName()); } + + @Test + void testLongToUtcTimestampGatedInBothReconcilePaths() { + // Bare long to a UTC timestamp is override-gated exactly like the local-timestamp case: rejected + // without a per-field override and applied with one, in both reconcile paths. The non-reconcile + // guard previously skipped this and let it through silently on the default write path. + HoodieSchema tableBareLong = HoodieSchema.fromAvroSchema(tripAvro(Schema.create(Schema.Type.LONG))); + HoodieSchema incomingMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + + // No override: rejected in both paths with the exact actionable error. + Map noOverride = SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""); + String expectedError = AvroSchemaEvolutionUtils.timestampPrecisionChangeError( + "ts", Types.LongType.get(), Types.TimestampType.get()).getMessage(); + SchemaCompatibilityException reconcileError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incomingMicros, tableBareLong, false, noOverride)); + assertEquals(expectedError, reconcileError.getMessage()); + SchemaCompatibilityException guardError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incomingMicros, tableBareLong, noOverride)); + assertEquals(expectedError, guardError.getMessage()); + + // With the override: the promotion is applied in both paths. + Schema viaReconcile = AvroSchemaEvolutionUtils.reconcileSchema(incomingMicros, tableBareLong, false, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + assertEquals("timestamp-micros", viaReconcile.getField("ts").schema().getLogicalType().getName()); + Schema viaGuard = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incomingMicros, tableBareLong, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:timestamp-micros")).toAvroSchema(); + assertEquals("timestamp-micros", viaGuard.getField("ts").schema().getLogicalType().getName()); + } + + @Test + void testLongToLocalTimestampGatedInBothReconcilePaths() { + // Bare long to local timestamp is override-gated (not forbidden): rejected without an override + // and applied with one, and the non-reconcile guard must agree with reconcileSchema. + HoodieSchema tableBareLong = HoodieSchema.fromAvroSchema(tripAvro(Schema.create(Schema.Type.LONG))); + HoodieSchema incomingLocalMicros = HoodieSchema.fromAvroSchema(tripAvro(LogicalTypes.localTimestampMicros().addToSchema(Schema.create(Schema.Type.LONG)))); + + // No override: rejected in both paths with the exact actionable error. + Map noOverride = SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""); + String expectedError = AvroSchemaEvolutionUtils.timestampPrecisionChangeError( + "ts", Types.LongType.get(), Types.LocalTimestampMicrosType.get()).getMessage(); + SchemaCompatibilityException reconcileError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incomingLocalMicros, tableBareLong, false, noOverride)); + assertEquals(expectedError, reconcileError.getMessage()); + SchemaCompatibilityException guardError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incomingLocalMicros, tableBareLong, noOverride)); + assertEquals(expectedError, guardError.getMessage()); + Schema repaired = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incomingLocalMicros, tableBareLong, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("ts:local-timestamp-micros")).toAvroSchema(); + assertEquals("local-timestamp-micros", repaired.getField("ts").schema().getLogicalType().getName()); + } + + @Test + void testNestedLongToTimestampGated() { + // The gate resolves fully-qualified column names, so it applies to nested fields too. A nested + // long -> timestamp (UTC or local) is override-gated via the dotted-key override. + for (String token : new String[] {"timestamp-micros", "local-timestamp-millis"}) { + HoodieSchema tableNested = HoodieSchema.fromAvroSchema(nestedTrip(Schema.create(Schema.Type.LONG))); + HoodieSchema incoming = HoodieSchema.fromAvroSchema(nestedTrip(logicalLong(token))); + // No override: rejected in both paths with the exact actionable error. + Map noOverride = SchemaChangeUtils.parseTimestampLogicalTypeOverrides(""); + String expectedError = AvroSchemaEvolutionUtils.timestampPrecisionChangeError("payload.event_ts", Types.LongType.get(), + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("field:" + token).get("field")).getMessage(); + SchemaCompatibilityException reconcileError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileSchema(incoming, tableNested, false, noOverride)); + assertEquals(expectedError, reconcileError.getMessage()); + SchemaCompatibilityException guardError = assertThrows(SchemaCompatibilityException.class, + () -> AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incoming, tableNested, noOverride)); + assertEquals(expectedError, guardError.getMessage()); + // The dotted-key override authorizes the nested promotion. + Schema repairedNested = AvroSchemaEvolutionUtils.reconcileTimestampLogicalType(incoming, tableNested, + SchemaChangeUtils.parseTimestampLogicalTypeOverrides("payload.event_ts:" + token)).toAvroSchema(); + assertEquals(token, + repairedNested.getField("payload").schema().getField("event_ts").schema().getLogicalType().getName()); + } + } + + private static Schema nestedTrip(Schema eventTsType) { + Schema payload = Schema.createRecord("payloadrec", null, null, false, Arrays.asList( + new Schema.Field("event_ts", eventTsType, null, null))); + return Schema.createRecord("trip", null, null, false, Arrays.asList( + new Schema.Field("id", Schema.create(Schema.Type.STRING), null, null), + new Schema.Field("payload", payload, null, null))); + } + + private static Schema logicalLong(String token) { + Schema longSchema = Schema.create(Schema.Type.LONG); + switch (token) { + case "timestamp-micros": + return LogicalTypes.timestampMicros().addToSchema(longSchema); + case "timestamp-millis": + return LogicalTypes.timestampMillis().addToSchema(longSchema); + case "local-timestamp-micros": + return LogicalTypes.localTimestampMicros().addToSchema(longSchema); + case "local-timestamp-millis": + return LogicalTypes.localTimestampMillis().addToSchema(longSchema); + default: + throw new IllegalArgumentException(token); + } + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java index f192e3fe32f67..6d17f359f66eb 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestSchemaChangeUtils.java @@ -26,6 +26,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -120,4 +121,32 @@ public void parseResultIsUnmodifiable() { assertThrows(UnsupportedOperationException.class, () -> overrides.put("other", Types.TimestampMillisType.get())); } + + @Test + void gatedTimestampChangeCoversFlipsAndLongPromotions() { + // Precision flips (either direction) are gated. + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.TimestampType.get(), Types.TimestampMillisType.get())); + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.LocalTimestampMicrosType.get(), Types.LocalTimestampMillisType.get())); + // Promoting a bare long to any timestamp logical type (UTC or local) is gated. + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.LongType.get(), Types.TimestampType.get())); + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.LongType.get(), Types.TimestampMillisType.get())); + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.LongType.get(), Types.LocalTimestampMillisType.get())); + assertTrue(SchemaChangeUtils.isGatedTimestampChange(Types.LongType.get(), Types.LocalTimestampMicrosType.get())); + // Unrelated promotions and identical types are not gated. + assertFalse(SchemaChangeUtils.isGatedTimestampChange(Types.LongType.get(), Types.StringType.get())); + assertFalse(SchemaChangeUtils.isGatedTimestampChange(Types.IntType.get(), Types.TimestampType.get())); + assertFalse(SchemaChangeUtils.isGatedTimestampChange(Types.TimestampType.get(), Types.TimestampType.get())); + } + + @Test + void typeUpdateAllowGatesLongToTimestampBehindTheOverride() { + // Promoting a bare long to any timestamp logical type is allowed only when the gate is open. + for (Type ts : new Type[] {Types.TimestampType.get(), Types.TimestampMillisType.get(), + Types.LocalTimestampMillisType.get(), Types.LocalTimestampMicrosType.get()}) { + assertTrue(SchemaChangeUtils.isTypeUpdateAllow(Types.LongType.get(), ts, true)); + assertFalse(SchemaChangeUtils.isTypeUpdateAllow(Types.LongType.get(), ts, false)); + } + // Existing long widening is unaffected by the gate. + assertTrue(SchemaChangeUtils.isTypeUpdateAllow(Types.LongType.get(), Types.DoubleType.get(), false)); + } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 1bd53c605ae49..d431158139fb5 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -54,6 +54,10 @@ import org.apache.hudi.common.schema.HoodieSchema.TimePrecision; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.schema.internal.Type; +import org.apache.hudi.common.schema.internal.Types; +import org.apache.hudi.common.schema.internal.utils.AvroSchemaEvolutionUtils; +import org.apache.hudi.common.schema.internal.utils.SchemaChangeUtils; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; @@ -132,6 +136,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -201,6 +207,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -760,6 +767,97 @@ public void testTimestampMillis() throws Exception { assertEquals(0, sqlContext.read().options(hudiOpts).format("org.apache.hudi").load(tableBasePath).filter("current_ts < '1980-01-01'").count()); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testLongToTimestampPromotionGated(boolean setNullForMissingColumns) throws Exception { + // Promoting a plain long column to a timestamp logical type is override-gated: rejected without a + // per-field override for every target type, and applied with one. A bare long carries no precision + // signal, so the override is the explicit verdict that authorizes the promotion. One bare-long seed + // is reused: the rejection cases all throw (table stays bare long), and the accepted case runs last. + String tableBasePath = basePath + "/testLongToTs" + setNullForMissingColumns; + defaultSchemaProviderClassName = FilebasedSchemaProvider.class.getName(); + + // Sync 0: seed the table with `seconds_since_epoch` stored as a bare long. + HoodieDeltaStreamer.Config seed = TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT, + Collections.singletonList(TestIdentityTransformer.class.getName()), PROPS_FILENAME_TEST_SOURCE, + false, true, false, null, HoodieTableType.COPY_ON_WRITE.name()); + seed.configs.add("hoodie.streamer.schemaprovider.source.schema.file=" + basePath + "/source-timestamp-millis.avsc"); + seed.configs.add("hoodie.streamer.schemaprovider.target.schema.file=" + basePath + "/source-timestamp-millis.avsc"); + seed.configs.add("hoodie.datasource.write.row.writer.enable=false"); + seed.configs.add(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key() + "=" + setNullForMissingColumns); + new HoodieDeltaStreamer(seed, jsc).sync(); + + Schema tableSchema = new TableSchemaResolver(HoodieTestUtils.createMetaClient(storage, tableBasePath)) + .getTableSchema(false).toAvroSchema(); + assertNull(tableSchema.getField("seconds_since_epoch").schema().getLogicalType(), + "seconds_since_epoch must be seeded as a bare long in the table"); + Schema baseSchema = new Schema.Parser().parse(fs.open(new Path(basePath + "/source-timestamp-millis.avsc"))); + + // Every target type is rejected without a per-field override. + for (LogicalType targetType : new LogicalType[] {LogicalTypes.timestampMillis(), LogicalTypes.timestampMicros(), + LogicalTypes.localTimestampMillis(), LogicalTypes.localTimestampMicros()}) { + String schemaFile = writePromotedSchema(baseSchema, targetType, setNullForMissingColumns); + HoodieDeltaStreamer.Config reject = promoteConfig(tableBasePath, schemaFile, setNullForMissingColumns, null); + HoodieDeltaStreamer streamer = new HoodieDeltaStreamer(reject, jsc); + // sync() wraps the guard's SchemaCompatibilityException in a HoodieIngestionException, so walk + // the cause chain to assert on the underlying exception. + Throwable thrown = assertThrows(Exception.class, streamer::sync, + "long -> " + targetType.getName() + " must be rejected without an override"); + Throwable cause = thrown; + while (cause != null && !(cause instanceof SchemaCompatibilityException)) { + cause = cause.getCause(); + } + assertTrue(cause instanceof SchemaCompatibilityException, + "Expected a SchemaCompatibilityException in the cause chain, got: " + thrown); + Type toType = SchemaChangeUtils.parseTimestampLogicalTypeOverrides("field:" + targetType.getName()).get("field"); + assertEquals(AvroSchemaEvolutionUtils.timestampPrecisionChangeError( + "seconds_since_epoch", Types.LongType.get(), toType).getMessage(), cause.getMessage()); + } + + // With an override the promotion is authorized (local promotions are covered end-to-end by + // testCOWLogicalRepair / testMORLogicalRepair); verify a UTC promotion succeeds and lands on the + // table schema. + String utcSchemaFile = writePromotedSchema(baseSchema, LogicalTypes.timestampMicros(), setNullForMissingColumns); + HoodieDeltaStreamer.Config accept = promoteConfig(tableBasePath, utcSchemaFile, setNullForMissingColumns, + "seconds_since_epoch:timestamp-micros"); + new HoodieDeltaStreamer(accept, jsc).sync(); + Schema evolved = new TableSchemaResolver(HoodieTestUtils.createMetaClient(storage, tableBasePath)) + .getTableSchema(false).toAvroSchema(); + assertEquals("timestamp-micros", evolved.getField("seconds_since_epoch").schema().getLogicalType().getName()); + } + + private String writePromotedSchema(Schema baseSchema, LogicalType targetType, boolean setNull) throws IOException { + Schema incoming = replaceFieldType(baseSchema, "seconds_since_epoch", + targetType.addToSchema(Schema.create(Schema.Type.LONG))); + String schemaFile = basePath + "/promote-" + targetType.getName() + "-nul" + setNull + ".avsc"; + UtilitiesTestBase.Helpers.saveStringsToDFS(new String[] {incoming.toString()}, storage, schemaFile); + return schemaFile; + } + + private HoodieDeltaStreamer.Config promoteConfig(String tableBasePath, String schemaFile, + boolean setNullForMissingColumns, String override) { + HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, + Collections.singletonList(TestIdentityTransformer.class.getName()), PROPS_FILENAME_TEST_SOURCE, + false, true, false, null, HoodieTableType.COPY_ON_WRITE.name()); + cfg.configs.add("hoodie.streamer.schemaprovider.source.schema.file=" + schemaFile); + cfg.configs.add("hoodie.streamer.schemaprovider.target.schema.file=" + schemaFile); + cfg.configs.add("hoodie.datasource.write.row.writer.enable=false"); + cfg.configs.add(HoodieCommonConfig.SET_NULL_FOR_MISSING_COLUMNS.key() + "=" + setNullForMissingColumns); + if (override != null) { + cfg.configs.add(HoodieCommonConfig.TIMESTAMP_LOGICAL_TYPE_OVERRIDES.key() + "=" + override); + } + return cfg; + } + + private static Schema replaceFieldType(Schema recordSchema, String fieldName, Schema newFieldType) { + List fields = new ArrayList<>(); + for (Schema.Field field : recordSchema.getFields()) { + Schema fieldSchema = field.name().equals(fieldName) ? newFieldType : field.schema(); + fields.add(new Schema.Field(field.name(), fieldSchema, field.doc(), field.defaultVal())); + } + return Schema.createRecord(recordSchema.getName(), recordSchema.getDoc(), recordSchema.getNamespace(), false, fields); + } + @Test public void testLogicalTypes() throws Exception { try { From f6832a5a18211d7872c4fd6bbdec4d318044e881 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 4 Aug 2026 19:08:57 +0800 Subject: [PATCH 235/255] fix(trino): report real block size and slice splits solely by target_split_size (#19478) * fix(trino): report real block size in HudiTrinoStorage and slice splits by target_split_size HudiTrinoStorage hardcoded blockSize=0 in convertToPathInfo and getPathInfo, the hudi-trino counterpart of trinodb/trino#29842. Report the file length as the block size instead, matching the upstream fix at the storage layer. HudiSplitFactory used max(target_split_size, blockSize) to size base file splits. With the storage layer now reporting length as block size, that max() would silently disable the target_split_size knob, so make the policy explicit: slicing is governed solely by target_split_size. Also fail fast on a non-positive target, which previously looped forever. Covers apache/hudi#19231. * fix(trino): validate target_split_size at config time and pin the split sizing tests Addresses review feedback on #19478: - Move the non-positive target split size guard from the middle of createSplitsForBaseFile into the HudiSplitFactory constructor, so it covers every split path instead of only base file slicing past the fileSize == 0 early return. createHudiSplits becomes private, leaving the constructor as the single entry point that can be handed a bad target. - Reject a zero target at config time as well: @MinDataSize("1B") on HudiConfig.getTargetSplitSize and validateMinDataSize on the target_split_size session property, matching the shape already used by parquet_small_file_threshold. The constructor check stays as a backstop. - Set the TestHudiSplitFactory fixture block size to the base file length, which is what HudiTrinoStorage now reports. The old fixed 8MB fixture was below the 128MB target, so the default-target tests passed on master unchanged and did not pin the max() removal. With the block size tracking the file length, restoring max(target, blockSize) fails 5 tests, including the 500MB default-target case. (cherry picked from commit 66b9c6eeb60f377b86ebaaa7a0e96d37f007593b) --- .../java/io/trino/plugin/hudi/HudiConfig.java | 2 + .../plugin/hudi/HudiSessionProperties.java | 2 + .../plugin/hudi/split/HudiSplitFactory.java | 8 +- .../plugin/hudi/storage/HudiTrinoStorage.java | 5 +- .../io/trino/plugin/hudi/TestHudiConfig.java | 13 ++ .../hudi/TestHudiSessionProperties.java | 18 +++ .../hudi/split/TestHudiSplitFactory.java | 78 +++++++-- .../hudi/storage/TestHudiTrinoStorage.java | 150 ++++++++++++++++++ 8 files changed, 257 insertions(+), 19 deletions(-) create mode 100644 hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java index 8ecb76eeda9d8..c5ce5ce0e23d9 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java @@ -19,6 +19,7 @@ import io.airlift.configuration.DefunctConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.airlift.units.MinDataSize; import jakarta.validation.constraints.DecimalMax; import jakarta.validation.constraints.DecimalMin; import jakarta.validation.constraints.Min; @@ -220,6 +221,7 @@ public HudiConfig setTargetSplitSize(DataSize targetSplitSize) } @NotNull + @MinDataSize("1B") public DataSize getTargetSplitSize() { return targetSplitSize; diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java index 32ac46a95977f..8a47e3a8b03d8 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java @@ -31,6 +31,7 @@ import static io.trino.plugin.base.session.PropertyMetadataUtil.dataSizeProperty; import static io.trino.plugin.base.session.PropertyMetadataUtil.durationProperty; import static io.trino.plugin.base.session.PropertyMetadataUtil.validateMaxDataSize; +import static io.trino.plugin.base.session.PropertyMetadataUtil.validateMinDataSize; import static io.trino.plugin.hive.parquet.ParquetReaderConfig.PARQUET_READER_MAX_SMALL_FILE_THRESHOLD; import static io.trino.spi.StandardErrorCode.INVALID_SESSION_PROPERTY; import static io.trino.spi.session.PropertyMetadata.booleanProperty; @@ -188,6 +189,7 @@ public HudiSessionProperties(HudiConfig hudiConfig, ParquetReaderConfig parquetR TARGET_SPLIT_SIZE, "The target split size", hudiConfig.getTargetSplitSize(), + value -> validateMinDataSize(TARGET_SPLIT_SIZE, value, DataSize.ofBytes(1)), false), integerProperty( MAX_SPLITS_PER_SECOND, diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java index 6bab996e199bb..ece6472473c0d 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java @@ -51,6 +51,8 @@ public HudiSplitFactory( this.hudiTableHandle = requireNonNull(hudiTableHandle, "hudiTableHandle is null"); this.hudiSplitWeightProvider = requireNonNull(hudiSplitWeightProvider, "hudiSplitWeightProvider is null"); this.targetSplitSize = requireNonNull(targetSplitSize, "targetSplitSize is null"); + // A non-positive target would make split generation loop forever, so reject it here rather than mid-scan + checkArgument(targetSplitSize.toBytes() > 0, "targetSplitSize must be positive: %s", targetSplitSize); } public List createSplits(List partitionKeys, FileSlice fileSlice, String commitTime) @@ -65,7 +67,7 @@ public List createSplits(List partitionKeys, FileSl *

    * For regular MOR tables, a single split is created for the combination of the base file and its log files. */ - public static List createHudiSplits( + private static List createHudiSplits( HudiTableHandle hudiTableHandle, List partitionKeys, FileSlice fileSlice, @@ -127,7 +129,9 @@ private static List createSplitsForBaseFile( } ImmutableList.Builder splits = ImmutableList.builder(); - long targetSplitSizeInBytes = Math.max(targetSplitSize.toBytes(), baseFile.getPathInfo().getBlockSize()); + // Slicing is governed solely by the target split size; the block size reported by + // storage is not meaningful on object stores and must not influence split sizing. + long targetSplitSizeInBytes = targetSplitSize.toBytes(); long bytesRemaining = fileSize; while (((double) bytesRemaining) / targetSplitSizeInBytes > SPLIT_SLOP) { diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java index 48c5409c10d83..1edff63e8a7b5 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java @@ -71,7 +71,7 @@ public static StoragePathInfo convertToPathInfo(FileEntry fileEntry) fileEntry.length(), false, (short) 0, - 0, + fileEntry.length(), fileEntry.lastModified().toEpochMilli()); } @@ -170,7 +170,8 @@ public StoragePathInfo getPathInfo(StoragePath path) if (!inputFile.exists()) { throw new FileNotFoundException("Path " + path + " does not exist"); } - return new StoragePathInfo(path, inputFile.length(), false, (short) 0, 0, inputFile.lastModified().toEpochMilli()); + long length = inputFile.length(); + return new StoragePathInfo(path, length, false, (short) 0, length, inputFile.lastModified().toEpochMilli()); } @Override diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java index d5353b896a1a5..29aeded55ffd0 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.airlift.units.MinDataSize; import org.junit.jupiter.api.Test; import java.util.Map; @@ -24,6 +25,7 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; import static io.airlift.units.DataSize.Unit.MEGABYTE; public class TestHudiConfig @@ -131,4 +133,15 @@ public void testExplicitPropertyMappings() assertFullMapping(properties, expected); } + + @Test + public void testTargetSplitSizeValidation() + { + // A zero target split size would make split generation loop forever, so reject it at config time + assertFailsValidation( + new HudiConfig().setTargetSplitSize(DataSize.ofBytes(0)), + "targetSplitSize", + "must be greater than or equal to 1B", + MinDataSize.class); + } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java index ed17a203089ed..b97bdc47750e6 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java @@ -14,14 +14,18 @@ package io.trino.plugin.hudi; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.trino.plugin.hive.parquet.ParquetReaderConfig; +import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.testing.TestingConnectorSession; import org.junit.jupiter.api.Test; import static io.trino.plugin.hudi.HudiSessionProperties.getColumnsToHide; import static io.trino.plugin.hudi.HudiSessionProperties.getRecordMergerImpls; +import static io.trino.plugin.hudi.HudiSessionProperties.getTargetSplitSize; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestHudiSessionProperties { @@ -50,4 +54,18 @@ public void testSessionPropertyRecordMergerImpls() assertThat(getRecordMergerImpls(session)) .containsExactly("com.example.MergerOne", "com.example.MergerTwo"); } + + @Test + public void testSessionPropertyTargetSplitSizeRejectsZero() + { + // A zero target split size would make split generation loop forever, so reject it when the property is read + HudiSessionProperties sessionProperties = new HudiSessionProperties(new HudiConfig(), new ParquetReaderConfig()); + ConnectorSession session = TestingConnectorSession.builder() + .setPropertyMetadata(sessionProperties.getSessionProperties()) + .setPropertyValues(ImmutableMap.of("target_split_size", "0B")) + .build(); + assertThatThrownBy(() -> getTargetSplitSize(session)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("target_split_size must be at least 1B: 0B"); + } } diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java index d659044d24569..067c0f7b3dbdf 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java @@ -35,6 +35,7 @@ import static io.airlift.units.DataSize.Unit.MEGABYTE; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestHudiSplitFactory { @@ -101,19 +102,63 @@ public void testCreateHudiSplitsWithOversizedFileExceedingSlop() } @Test - public void testCreateHudiSplitsWithLargerBlockSize() + public void testCreateHudiSplitsIgnoresBlockSize() { - // Test with 1MB target split size and 32MB base file - // - should create 4 splits because the block size of 8MB is larger than the target split size + // Test with 2MB target and 8MB base file whose reported block size is 8MB + // - the block size must be ignored, so 4 splits of the 2MB target size are expected + // (previously the 8MB block size beat the target and produced 1 split of 8MB) testSplitCreation( - DataSize.of(1, MEGABYTE), - DataSize.of(32, MEGABYTE), + DataSize.of(2, MEGABYTE), + DataSize.of(8, MEGABYTE), Option.empty(), ImmutableList.of( - Pair.of(0L, DataSize.of(8, MEGABYTE)), - Pair.of(DataSize.of(8, MEGABYTE).toBytes(), DataSize.of(8, MEGABYTE)), - Pair.of(DataSize.of(16, MEGABYTE).toBytes(), DataSize.of(8, MEGABYTE)), - Pair.of(DataSize.of(24, MEGABYTE).toBytes(), DataSize.of(8, MEGABYTE)))); + Pair.of(0L, DataSize.of(2, MEGABYTE)), + Pair.of(DataSize.of(2, MEGABYTE).toBytes(), DataSize.of(2, MEGABYTE)), + Pair.of(DataSize.of(4, MEGABYTE).toBytes(), DataSize.of(2, MEGABYTE)), + Pair.of(DataSize.of(6, MEGABYTE).toBytes(), DataSize.of(2, MEGABYTE)))); + } + + @Test + public void testCreateHudiSplitsWithFileSmallerThanDefaultTarget() + { + // Regression test for the split inflation reported in trinodb/trino#29842 (hudi#19231): + // a ~120MB file with the default 128MB target must produce exactly 1 split + testSplitCreation( + DataSize.of(128, MEGABYTE), + DataSize.of(120, MEGABYTE), + Option.empty(), + ImmutableList.of( + Pair.of(0L, DataSize.of(120, MEGABYTE)))); + } + + @Test + public void testCreateHudiSplitsWithFileLargerThanDefaultTarget() + { + // Test with 128MB target and 500MB base file + // - should be sliced at target boundaries into 3 x 128MB + 116MB remainder, even though the + // reported block size (500MB, the file length) would otherwise force a single split + testSplitCreation( + DataSize.of(128, MEGABYTE), + DataSize.of(500, MEGABYTE), + Option.empty(), + ImmutableList.of( + Pair.of(0L, DataSize.of(128, MEGABYTE)), + Pair.of(DataSize.of(128, MEGABYTE).toBytes(), DataSize.of(128, MEGABYTE)), + Pair.of(DataSize.of(256, MEGABYTE).toBytes(), DataSize.of(128, MEGABYTE)), + Pair.of(DataSize.of(384, MEGABYTE).toBytes(), DataSize.of(116, MEGABYTE)))); + } + + @Test + public void testCreateHudiSplitsWithZeroTargetSplitSize() + { + // A zero target split size must be rejected on construction, before any file slice is seen, + // instead of looping forever once split generation reaches a non-empty base file + assertThatThrownBy(() -> new HudiSplitFactory( + createTableHandle(), + new SizeBasedSplitWeightProvider(0.05, DataSize.of(128, MEGABYTE)), + DataSize.ofBytes(0))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("targetSplitSize"); } @Test @@ -151,8 +196,8 @@ private static void testSplitCreation( FileSlice fileSlice = createFileSlice(baseFileSize, logFileSize); - List splits = HudiSplitFactory.createHudiSplits( - tableHandle, PARTITION_KEYS, fileSlice, COMMIT_TIME, weightProvider, targetSplitSize); + List splits = new HudiSplitFactory(tableHandle, weightProvider, targetSplitSize) + .createSplits(PARTITION_KEYS, fileSlice, COMMIT_TIME); assertThat(splits).hasSize(expectedSplitInfo.size()); @@ -192,14 +237,17 @@ private static FileSlice createFileSlice(DataSize baseFileSize, Option { String fileId = "5a4f6a70-0306-40a8-952b-045b0d8ff0d4-0"; HoodieFileGroupId fileGroupId = new HoodieFileGroupId("partition", fileId); - long blockSize = 8L * 1024 * 1024; + // Block size mirrors the file length, which is what HudiTrinoStorage now reports. Split + // generation must ignore it, so every multi-split expectation below would collapse to a + // single whole-file split if the block size were allowed back into the sizing decision. String baseFilePath = "/test/path/" + fileGroupId + "_4-19-0_" + COMMIT_TIME + ".parquet"; String logFilePath = "/test/path/." + fileId + "_2025062515374131546.log.1_0-53-80"; + long logFileSizeInBytes = logFileSize.isPresent() ? logFileSize.get().toBytes() : 0L; StoragePathInfo baseFileInfo = new StoragePathInfo( - new StoragePath(baseFilePath), baseFileSize.toBytes(), false, (short) 0, blockSize, System.currentTimeMillis()); + new StoragePath(baseFilePath), baseFileSize.toBytes(), false, (short) 0, baseFileSize.toBytes(), System.currentTimeMillis()); StoragePathInfo logFileInfo = new StoragePathInfo( - new StoragePath(logFilePath), logFileSize.isPresent() ? logFileSize.get().toBytes() : 0L, - false, (short) 0, blockSize, System.currentTimeMillis()); + new StoragePath(logFilePath), logFileSizeInBytes, + false, (short) 0, logFileSizeInBytes, System.currentTimeMillis()); HoodieBaseFile baseFile = new HoodieBaseFile(baseFileInfo); return new FileSlice(fileGroupId, COMMIT_TIME, baseFile, logFileSize.isPresent() ? ImmutableList.of(new HoodieLogFile(logFileInfo)) : ImmutableList.of()); diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java new file mode 100644 index 0000000000000..a085a73af5512 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java @@ -0,0 +1,150 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.storage; + +import io.trino.filesystem.FileEntry; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.memory.MemoryFileSystem; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class TestHudiTrinoStorage +{ + @Test + void testConvertToPathInfo() + { + FileEntry fileEntry = new FileEntry( + Location.of("memory:///table/data.parquet"), + 42, + Instant.ofEpochMilli(1234567890123L), + Optional.empty()); + + StoragePathInfo pathInfo = HudiTrinoStorage.convertToPathInfo(fileEntry); + + assertThat(pathInfo.getPath()).isEqualTo(new StoragePath("memory:///table/data.parquet")); + assertThat(pathInfo.getLength()).isEqualTo(42); + assertThat(pathInfo.isFile()).isTrue(); + assertThat(pathInfo.getBlockReplication()).isEqualTo((short) 0); + assertThat(pathInfo.getBlockSize()).isEqualTo(42); + assertThat(pathInfo.getModificationTime()).isEqualTo(1234567890123L); + } + + @Test + void testGetPathInfoForFile() + throws IOException + { + TrinoFileSystem fileSystem = new MemoryFileSystem(); + writeFile(fileSystem, "memory:///table/data.parquet", 42); + HudiTrinoStorage storage = new HudiTrinoStorage(fileSystem, new TrinoStorageConfiguration()); + + StoragePathInfo pathInfo = storage.getPathInfo(new StoragePath("memory:///table/data.parquet")); + + assertThat(pathInfo.getLength()).isEqualTo(42); + assertThat(pathInfo.isFile()).isTrue(); + assertThat(pathInfo.getBlockSize()).isEqualTo(42); + assertThat(pathInfo.getModificationTime()).isGreaterThan(0); + } + + @Test + void testGetPathInfoForDirectory() + throws IOException + { + TrinoFileSystem fileSystem = new MemoryFileSystem(); + writeFile(fileSystem, "memory:///table/data.parquet", 42); + HudiTrinoStorage storage = new HudiTrinoStorage(fileSystem, new TrinoStorageConfiguration()); + + StoragePathInfo pathInfo = storage.getPathInfo(new StoragePath("memory:///table")); + + assertThat(pathInfo.isDirectory()).isTrue(); + assertThat(pathInfo.getLength()).isEqualTo(0); + assertThat(pathInfo.getBlockSize()).isEqualTo(0); + } + + @Test + void testListFiles() + throws IOException + { + HudiTrinoStorage storage = createStorageWithFiles(); + + List entries = storage.listFiles(new StoragePath("memory:///table")); + + assertThat(entries).hasSize(3); + assertThat(entries.get(0).getPath()).isEqualTo(new StoragePath("memory:///table/a.parquet")); + assertThat(entries.get(1).getPath()).isEqualTo(new StoragePath("memory:///table/b.parquet")); + assertThat(entries.get(2).getPath()).isEqualTo(new StoragePath("memory:///table/nested/c.parquet")); + assertThat(entries.get(0).getLength()).isEqualTo(10); + assertThat(entries.get(1).getLength()).isEqualTo(20); + assertThat(entries.get(2).getLength()).isEqualTo(30); + for (StoragePathInfo entry : entries) { + assertThat(entry.getBlockSize()).isEqualTo(entry.getLength()); + } + } + + @Test + void testListDirectEntries() + throws IOException + { + HudiTrinoStorage storage = createStorageWithFiles(); + + List entries = storage.listDirectEntries(new StoragePath("memory:///table")); + + assertThat(entries).hasSize(2); + assertThat(entries.get(0).getPath()).isEqualTo(new StoragePath("memory:///table/a.parquet")); + assertThat(entries.get(1).getPath()).isEqualTo(new StoragePath("memory:///table/b.parquet")); + for (StoragePathInfo entry : entries) { + assertThat(entry.getBlockSize()).isEqualTo(entry.getLength()); + } + } + + @Test + void testListDirectEntriesWithFilter() + throws IOException + { + HudiTrinoStorage storage = createStorageWithFiles(); + + List entries = storage.listDirectEntries( + new StoragePath("memory:///table"), + path -> path.getName().equals("b.parquet")); + + assertThat(entries).hasSize(1); + assertThat(entries.get(0).getPath()).isEqualTo(new StoragePath("memory:///table/b.parquet")); + assertThat(entries.get(0).getLength()).isEqualTo(20); + assertThat(entries.get(0).getBlockSize()).isEqualTo(20); + } + + private static HudiTrinoStorage createStorageWithFiles() + throws IOException + { + TrinoFileSystem fileSystem = new MemoryFileSystem(); + writeFile(fileSystem, "memory:///table/a.parquet", 10); + writeFile(fileSystem, "memory:///table/b.parquet", 20); + writeFile(fileSystem, "memory:///table/nested/c.parquet", 30); + return new HudiTrinoStorage(fileSystem, new TrinoStorageConfiguration()); + } + + private static void writeFile(TrinoFileSystem fileSystem, String location, int length) + throws IOException + { + fileSystem.newOutputFile(Location.of(location)).createOrOverwrite(new byte[length]); + } +} From 37cf01d265d8c4a72044c06d82bb64dc34fadc55 Mon Sep 17 00:00:00 2001 From: voonhous Date: Tue, 4 Aug 2026 19:26:57 +0800 Subject: [PATCH 236/255] perf(trino): drop the decimal schema cache and memoize prefilled values (#19495) * perf(trino): drop the decimal schema cache and memoize prefilled column values Follow-up to #19483, addressing wombatu-kun's review comments. The decimal schema cache was unnecessary rather than merely mis-keyed. Avro's DecimalConversion.fromBytes reads only the scale (it never touches precision, and ignores its schema argument), and Decimals.encodeShortScaledValue then calls setScale to that same scale, which returns the BigDecimal unchanged. The pair reduces to new BigInteger(fixed.bytes()).longValueExact(), so AvroDecimalConverter and its ConcurrentHashMap are deleted instead of re-keyed. PrefilledColumnValues resolved every value through HiveUtil.getPrefilledColumnValue on each call, and appendTo runs once per prefilled column per record. Every input is a split constant, so the resolved value is now memoized per column. Tests: the decimal test now drives the public appendTo path over scales, signs and the widest short decimal rather than the deleted converter; it passes against both the old and new implementations. Added repeated resolution of a hive-null column, the case a naive memo would get wrong. * refactor(trino): rename the uncached prefilled resolver to computeNativeValue * perf(trino): collapse the prefilled memo hit path to a single hash lookup Addresses wombatu-kun's review comment on #19495. containsKey-then-get was two hash lookups per prefilled column per record on the buildRecordInPage path. An UNRESOLVED sentinel with getOrDefault does it in one, while still distinguishing a not-yet-resolved column from one resolved to null (the hive-"\N" convention, and the lenient fallback for a column the split cannot provide). The map still stores real nulls, so it stays a HashMap. * test(trino): build the decimal fixed the way Avro writes it Addresses wombatu-kun's two review comments on #19495. The short-decimal cases built the Fixed from BigDecimal.unscaledValue().toByteArray(), the minimal two's-complement encoding, and sized the schema to those bytes. Avro's DecimalConversion.toFixed instead left-pads to the schema's fixed size with the sign byte, so a real decimal(10,2) is always five bytes. The negative cases were one and two bytes wide, meaning no case exercised sign extension across padding -- the thing the decode is most likely to get wrong. The fixture now sizes the schema from the precision and runs the value through Avro's own conversion, matching how TestHudiUtilColumnHandles builds its decimal fixed schema. Cases are unchanged and still pass; -0.07 now decodes from FF FF FF FF F9 rather than F9. Also drops an overreaching claim on the hive-null repeats: a null-check memo returns null on every call too, so the repeats do not rule it out. They cover what they actually cover, that both read paths keep returning null once the memo is populated. (cherry picked from commit c2e884aa20a48c539750b7b4ac7211f12c41d9fe) --- .../plugin/hudi/util/HudiAvroSerializer.java | 33 ++------- .../hudi/util/PrefilledColumnValues.java | 27 +++++++ .../hudi/TestPrefilledColumnValues.java | 13 +++- .../hudi/util/TestHudiAvroSerializer.java | 74 +++++++++++++------ 4 files changed, 98 insertions(+), 49 deletions(-) diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index fe7ddfe425159..8b035c084af66 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -41,8 +41,6 @@ import io.trino.spi.type.Type; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -50,6 +48,7 @@ import org.apache.avro.util.Utf8; import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.time.DateTimeException; import java.time.Instant; @@ -60,7 +59,6 @@ import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; @@ -70,7 +68,6 @@ import static io.trino.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.Decimals.encodeShortScaledValue; import static io.trino.spi.type.Decimals.writeBigDecimal; import static io.trino.spi.type.Decimals.writeShortDecimal; import static io.trino.spi.type.IntegerType.INTEGER; @@ -108,7 +105,6 @@ public class HudiAvroSerializer 1, // 9 digits after the dot }; - private static final AvroDecimalConverter DECIMAL_CONVERTER = new AvroDecimalConverter(); private final PrefilledColumnValues prefilledColumnValues; private final List columnHandles; @@ -262,8 +258,13 @@ else if (type instanceof DecimalType decimalType) { } else if (value instanceof GenericData.Fixed fixed) { verify(decimalType.isShort(), "The type should be short decimal"); - BigDecimal decimal = DECIMAL_CONVERTER.convert(decimalType.getPrecision(), decimalType.getScale(), fixed.bytes()); - type.writeLong(output, encodeShortScaledValue(decimal, decimalType.getScale())); + // Avro stores a decimal as its unscaled value in big-endian two's complement, which is + // exactly what Trino's short decimal holds. Going through Avro's DecimalConversion and + // Decimals.encodeShortScaledValue is a no-op round trip: DecimalConversion.fromBytes reads + // only the scale (it ignores precision, and its schema argument entirely) to build + // BigDecimal(unscaled, scale), and encodeShortScaledValue then calls setScale to that same + // scale, which returns the BigDecimal unchanged, before taking the unscaled value back out. + type.writeLong(output, new BigInteger(fixed.bytes()).longValueExact()); } else { throw new TrinoException(GENERIC_INTERNAL_ERROR, @@ -544,22 +545,4 @@ private static void writeMap(MapBlockBuilder output, MapType mapType, Map } }); } - - static class AvroDecimalConverter - { - private static final Conversions.DecimalConversion AVRO_DECIMAL_CONVERSION = new Conversions.DecimalConversion(); - // convert() runs once per decimal cell on the record read path, and building a Schema costs - // orders of magnitude more than the conversion itself. The (precision, scale) space is tiny - // and fixed per column, so cache the schemas globally. - private static final Map DECIMAL_SCHEMAS = new ConcurrentHashMap<>(); - - BigDecimal convert(int precision, int scale, byte[] bytes) - { - // The key is unique because precision and scale are at most 38 - Schema schema = DECIMAL_SCHEMAS.computeIfAbsent( - precision * 100 + scale, - key -> LogicalTypes.decimal(precision, scale).addToSchema(Schema.create(Schema.Type.BYTES))); - return AVRO_DECIMAL_CONVERSION.fromBytes(ByteBuffer.wrap(bytes), schema, schema.getLogicalType()); - } - } } diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java index 19c6c9d01322a..c55d46956c279 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/PrefilledColumnValues.java @@ -22,6 +22,7 @@ import io.trino.spi.block.BlockBuilder; import io.trino.spi.block.RunLengthEncodedBlock; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.OptionalInt; @@ -45,11 +46,18 @@ */ public class PrefilledColumnValues { + // Absent-marker for the memo, so a not-yet-resolved column is distinguishable from one resolved to null + private static final Object UNRESOLVED = new Object(); + private final Map partitionKeysByName; private final String partitionName; private final String filePath; private final long fileSize; private final long fileModifiedTime; + // Resolved native value per column name, populated lazily. One instance belongs to one split and a + // split is read by a single driver thread, so a plain HashMap is enough; it also has to hold nulls, + // which a ConcurrentHashMap could not. + private final Map resolvedValues = new HashMap<>(); public static PrefilledColumnValues create(HudiSplit hudiSplit) { @@ -108,6 +116,25 @@ public Block toRleBlock(HiveColumnHandle columnHandle, int positionCount) } private Object nativeValueOf(HiveColumnHandle columnHandle) + { + // Every input to computeNativeValue() is a constant of the split, but appendTo is called once per + // prefilled column per record, and computing re-parses the partition string each time + // ($file_modified_time even formats a timestamp and parses it straight back). Memoize per column so + // each one is resolved once per split. Keyed on the name rather than the handle because + // HiveColumnHandle.hashCode hashes seven fields through a varargs array, whereas a String caches + // its hash. A sentinel rather than a null check, because null is a legitimate resolved value -- both + // for the hive-null convention and for the lenient fallback below -- and getOrDefault keeps the hit + // path, the one taken per record, to a single hash lookup. + String name = columnHandle.getName(); + Object value = resolvedValues.getOrDefault(name, UNRESOLVED); + if (value == UNRESOLVED) { + value = computeNativeValue(columnHandle); + resolvedValues.put(name, value); + } + return value; + } + + private Object computeNativeValue(HiveColumnHandle columnHandle) { if (!isPrefilled(columnHandle)) { // Lenient null fill, e.g. for a hidden column Trino defines but Hudi does not populate. diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java index 4ee508cada0b0..59442f8cdbe6a 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestPrefilledColumnValues.java @@ -81,9 +81,20 @@ public void testHiveNullPartitionValue() { // Trino's HivePartitionKey encodes a null partition value as the literal string "\N" PrefilledColumnValues values = prefilledValues(new HivePartitionKey("pk_string", "\\N")); + HiveColumnHandle handle = partitionKey("pk_string", VARCHAR, HiveType.HIVE_STRING); - Block block = singleValueBlock(values, partitionKey("pk_string", VARCHAR, HiveType.HIVE_STRING)); + Block block = singleValueBlock(values, handle); assertThat(block.isNull(0)).isTrue(); + + // Values are resolved once per column and reused, so both read paths have to keep returning null + // for a hive-null column after the first read has populated the memo. + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2); + values.appendTo(handle, blockBuilder); + values.appendTo(handle, blockBuilder); + Block repeated = blockBuilder.build(); + assertThat(repeated.isNull(0)).isTrue(); + assertThat(repeated.isNull(1)).isTrue(); + assertThat(values.toRleBlock(handle, 1).isNull(0)).isTrue(); } @Test diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java index 116d50d115e89..514218d48adde 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java @@ -25,14 +25,21 @@ import io.trino.spi.block.BlockBuilder; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.type.DecimalType; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.IntegerType.INTEGER; @@ -41,32 +48,37 @@ class TestHudiAvroSerializer { - @Test - public void testDecimalConverter() - { - HudiAvroSerializer.AvroDecimalConverter converter = new HudiAvroSerializer.AvroDecimalConverter(); - - assertThat(converter.convert(10, 2, unscaledBytes("123.45"))).isEqualTo(new BigDecimal("123.45")); - // Same (precision, scale) again: served from the cached schema - assertThat(converter.convert(10, 2, unscaledBytes("-0.07"))).isEqualTo(new BigDecimal("-0.07")); - // Same precision, different scale, and vice versa: must not collide in the cache - assertThat(converter.convert(10, 4, unscaledBytes("123.4567"))).isEqualTo(new BigDecimal("123.4567")); - assertThat(converter.convert(18, 2, unscaledBytes("9999999999999999.99"))).isEqualTo(new BigDecimal("9999999999999999.99")); - assertThat(converter.convert(5, 0, unscaledBytes("42"))).isEqualTo(new BigDecimal("42")); - } - - @Test - public void testAppendShortDecimalFromAvroFixed() + /** + * A short decimal is stored in Trino as the unscaled value, which is exactly what Avro writes into the + * fixed bytes, so the read is a plain big-endian two's complement decode. The cases below pin the parts + * that decode gets wrong if it is ever rewritten: sign extension for negatives, scale 0, and the + * full-width value at the maximum short-decimal precision. + */ + @ParameterizedTest + @MethodSource("shortDecimals") + public void testAppendShortDecimalFromAvroFixed(int precision, int scale, String value, long expectedUnscaled) { - DecimalType type = DecimalType.createDecimalType(10, 2); - byte[] bytes = unscaledBytes("123.45"); - GenericData.Fixed fixed = new GenericData.Fixed(Schema.createFixed("fix", null, null, bytes.length), bytes); + DecimalType type = DecimalType.createDecimalType(precision, scale); BlockBuilder blockBuilder = type.createBlockBuilder(null, 1); - HudiAvroSerializer.appendTo(type, fixed, blockBuilder); + HudiAvroSerializer.appendTo(type, avroDecimalFixed(precision, scale, value), blockBuilder); Block block = blockBuilder.build(); - assertThat(type.getLong(block, 0)).isEqualTo(12345L); + assertThat(type.getLong(block, 0)).isEqualTo(expectedUnscaled); + } + + private static Stream shortDecimals() + { + return Stream.of( + Arguments.of(10, 2, "123.45", 12345L), + Arguments.of(10, 2, "-0.07", -7L), + Arguments.of(10, 2, "0.00", 0L), + Arguments.of(10, 4, "123.4567", 1234567L), + Arguments.of(5, 0, "42", 42L), + Arguments.of(5, 0, "-42", -42L), + // Widest short decimal: 18 digits, both signs + Arguments.of(18, 2, "9999999999999999.99", 999999999999999999L), + Arguments.of(18, 2, "-9999999999999999.99", -999999999999999999L)); } @Test @@ -96,9 +108,25 @@ public void testBuildRecordInPage() assertThat(VARCHAR.getSlice(page.getBlock(1), 2).toStringUtf8()).isEqualTo("three"); } - private static byte[] unscaledBytes(String decimal) + /** + * Encodes the value the way an Avro writer does, via Avro's own conversion: a fixed sized from the + * precision, holding the unscaled value left-padded to that width with the sign byte (0xFF for + * negatives). Building the fixed from the minimal two's-complement encoding instead would leave the + * padding bytes, and so sign extension across them, untested. + */ + private static GenericData.Fixed avroDecimalFixed(int precision, int scale, String value) + { + LogicalTypes.Decimal decimalType = LogicalTypes.decimal(precision, scale); + Schema fixedSchema = decimalType.addToSchema( + Schema.createFixed("fix", null, null, decimalFixedSize(precision))); + return (GenericData.Fixed) new Conversions.DecimalConversion() + .toFixed(new BigDecimal(value), fixedSchema, decimalType); + } + + /** Bytes needed to hold the widest unscaled value at this precision, i.e. the fixed size Avro sizes a decimal to. */ + private static int decimalFixedSize(int precision) { - return new BigDecimal(decimal).unscaledValue().toByteArray(); + return BigInteger.TEN.pow(precision).subtract(BigInteger.ONE).toByteArray().length; } private static Schema recordSchema(String name) From 392d76335c7165cb804793396e994ac554af7c86 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Tue, 4 Aug 2026 22:51:16 +0800 Subject: [PATCH 237/255] fix(flink): close CDC image spillable maps on failures (#19482) CDC iterators leaked ExternalSpillableMap instances when construction or iteration failed: the image manager was never closed on the failure path, leaving spill files behind. Route cleanup through a shared CloseableUtils.closeSuppressing helper, close the image manager via try-with-resources, and retain CDC images across child splits. (cherry picked from commit f41e8e3070d94c7997eed933ab76ceb250d77e53) Cherry-pick adaptations: - HoodieSplitReaderFunction: the private closeSuppressing helper being deleted is typed HoodieFileGroupReader here rather than HoodieRecordReader, since the LSM reader refactor (#18987, #19079, #19307) is not on this branch. Same deletion, different pre-existing signature. The shared replacement takes AutoCloseable and HoodieFileGroupReader implements Closeable, so call sites are unchanged. - CdcIterators: upstream drops the FormatUtils import and keeps HoodieRowDataFileReader / InternalSchemaManager as context. This branch's copy of the file never imported the latter two and does not reference them, so only the FormatUtils import is dropped. Adding the other two would be unused imports and fail checkstyle. The static import of FormatUtils.buildAvroRecordBySchema is unaffected. - Dropped the TestCdcImageManager and TestCdcIterators changes. Both test classes arrive with #19402, which is not on this branch, and both assert behavior from that commit's production half: TestCdcImageManager expects skipBytesToRead to throw EOFException, which without #19402 loops forever rather than failing. Importing them would add a hanging test. The fix therefore lands without its CDC test coverage. TestCloseableUtils is included, so the shared helper itself is covered. --- .../hudi/common/util/CloseableUtils.java | 35 ++++++++++++++++ .../hudi/common/util/TestCloseableUtils.java | 40 +++++++++++++++++++ .../HoodieCdcSplitReaderFunction.java | 15 +++++-- .../function/HoodieSplitReaderFunction.java | 11 +---- .../table/format/cdc/CdcImageManager.java | 31 +++++++++++--- .../hudi/table/format/cdc/CdcInputFormat.java | 17 +++++--- .../hudi/table/format/cdc/CdcIterators.java | 30 ++++++++------ 7 files changed, 144 insertions(+), 35 deletions(-) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/CloseableUtils.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestCloseableUtils.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CloseableUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CloseableUtils.java new file mode 100644 index 0000000000000..50b6e2e6ee80b --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CloseableUtils.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util; + +/** Utility methods for closing resources. */ +public final class CloseableUtils { + + private CloseableUtils() { + } + + /** Closes {@code closeable}, attaching any failure to {@code primary} as a suppressed exception. */ + public static void closeSuppressing(AutoCloseable closeable, Throwable primary) { + try { + closeable.close(); + } catch (Throwable closeError) { + primary.addSuppressed(closeError); + } + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCloseableUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCloseableUtils.java new file mode 100644 index 0000000000000..f18339047905d --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCloseableUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +class TestCloseableUtils { + + @Test + void testCloseSuppressing() { + IOException primary = new IOException("primary"); + IOException closeError = new IOException("close"); + + CloseableUtils.closeSuppressing(() -> { + throw closeError; + }, primary); + + assertArrayEquals(new Throwable[] {closeError}, primary.getSuppressed()); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java index 266c8ababa791..b69bd49570ca5 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieCdcSplitReaderFunction.java @@ -65,6 +65,8 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; + /** * CDC reader function for source V2. Reads CDC splits ({@link HoodieCdcSourceSplit}) and * emits change-log {@link RowData} records tagged with the appropriate {@link org.apache.flink.types.RowKind}. @@ -222,10 +224,15 @@ private ClosableIterator createRecordIterator( String logFilePath = new Path(tablePath, fileSplit.getCdcFiles().get(0)).toString(); MergeOnReadInputSplit split = CdcIterators.singleLogFile2Split(tablePath, logFilePath, maxCompactionMemoryInBytes); ClosableIterator> recordIterator = getFileSliceHoodieRecordIterator(split); - return new CdcIterators.DataLogFileIterator( - maxCompactionMemoryInBytes, imageManager, fileSplit, tableSchema, - tableState.getRequiredRowType(), tableState.getRequiredPositions(), - recordIterator, getMetaClient(), getWriteConfig()); + try { + return new CdcIterators.DataLogFileIterator( + maxCompactionMemoryInBytes, imageManager, fileSplit, tableSchema, + tableState.getRequiredRowType(), tableState.getRequiredPositions(), + recordIterator, getMetaClient(), getWriteConfig()); + } catch (IOException | RuntimeException | Error e) { + closeSuppressing(recordIterator, e); + throw e; + } } case REPLACE_COMMIT: { return new CdcIterators.ReplaceCommitIterator( diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java index 2e88d6163a52d..7c73a8f01de9c 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/function/HoodieSplitReaderFunction.java @@ -44,6 +44,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; + /** * Default reader function implementation for both MOR and COW tables. */ @@ -89,15 +91,6 @@ protected ClosableIterator createRecordIterator(HoodieSourceSplit split } } - /** Closes {@code reader}, attaching any close failure to {@code primary} as a suppressed exception. */ - private static void closeSuppressing(HoodieFileGroupReader reader, Throwable primary) { - try { - reader.close(); - } catch (Exception closeError) { - primary.addSuppressed(closeError); - } - } - @Override protected RowType producedRowType() { return HoodieSchemaConverter.convertToRowType(requiredSchema); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcImageManager.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcImageManager.java index cad323b7fb815..5492d153ebe8d 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcImageManager.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcImageManager.java @@ -44,6 +44,7 @@ import java.util.TreeMap; import java.util.function.Function; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS; /** @@ -104,13 +105,16 @@ private ExternalSpillableMap loadImageRecords( serializer.serialize(row, new BytesArrayOutputView(baos)); imageRecordsMap.put(recordKey, baos.toByteArray()); } + } catch (IOException | RuntimeException | Error e) { + closeSuppressing(imageRecordsMap, e); + throw e; } return imageRecordsMap; } public RowData getImageRecord( String recordKey, - ExternalSpillableMap imageCache, + Map imageCache, RowKind rowKind) { byte[] bytes = imageCache.get(recordKey); ValidationUtils.checkState(bytes != null, @@ -126,7 +130,7 @@ public RowData getImageRecord( public void updateImageRecord( String recordKey, - ExternalSpillableMap imageCache, + Map imageCache, RowData row) { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); try { @@ -139,7 +143,7 @@ public void updateImageRecord( public RowData removeImageRecord( String recordKey, - ExternalSpillableMap imageCache) { + Map imageCache) { byte[] bytes = imageCache.remove(recordKey); if (bytes == null) { return null; @@ -153,8 +157,25 @@ public RowData removeImageRecord( @Override public void close() { - cache.values().forEach(ExternalSpillableMap::close); - cache.clear(); + RuntimeException failure = null; + try { + for (ExternalSpillableMap spillableMap : cache.values()) { + try { + spillableMap.close(); + } catch (RuntimeException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + } finally { + cache.clear(); + } + if (failure != null) { + throw failure; + } } // ------------------------------------------------------------------------- diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java index 27bb2b3a56e8d..cc4de580ae770 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java @@ -48,6 +48,8 @@ import java.util.List; import java.util.function.Function; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; + /** * The base InputFormat class to read Hoodie data set as change logs. */ @@ -157,11 +159,16 @@ private ClosableIterator getRecordIterator( String logFilepath = new Path(tablePath, fileSplit.getCdcFiles().get(0)).toString(); MergeOnReadInputSplit split = CdcIterators.singleLogFile2Split(tablePath, logFilepath, maxCompactionMemoryInBytes); ClosableIterator> recordIterator = getSplitRecordIterator(split); - return new CdcIterators.DataLogFileIterator( - maxCompactionMemoryInBytes, imageManager, fileSplit, - HoodieSchema.parse(tableState.getTableSchema()), - tableState.getRequiredRowType(), tableState.getRequiredPositions(), - recordIterator, metaClient, imageManager.getWriteConfig()); + try { + return new CdcIterators.DataLogFileIterator( + maxCompactionMemoryInBytes, imageManager, fileSplit, + HoodieSchema.parse(tableState.getTableSchema()), + tableState.getRequiredRowType(), tableState.getRequiredPositions(), + recordIterator, metaClient, imageManager.getWriteConfig()); + } catch (IOException | RuntimeException | Error e) { + closeSuppressing(recordIterator, e); + throw e; + } case REPLACE_COMMIT: return new CdcIterators.ReplaceCommitIterator( tablePath, tableState.getRequiredRowType(), tableState.getRequiredPositions(), diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java index 20f6801aec0f9..2a01db2da646e 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java @@ -54,7 +54,6 @@ import org.apache.hudi.storage.HoodieStorageUtils; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.format.FlinkReaderContextFactory; -import org.apache.hudi.table.format.FormatUtils; import org.apache.hudi.table.format.mor.MergeOnReadInputSplit; import org.apache.hudi.util.AvroToRowDataConverters; import org.apache.hudi.util.HoodieSchemaConverter; @@ -71,9 +70,11 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; import static org.apache.hudi.table.format.FormatUtils.buildAvroRecordBySchema; /** @@ -137,11 +138,12 @@ public RowData next() { @Override public void close() { - if (recordIterator != null) { - recordIterator.close(); - } - if (imageManager != null) { - imageManager.close(); + try (CdcImageManager ignored = imageManager) { + if (recordIterator != null) { + recordIterator.close(); + } + } finally { + recordIterator = null; imageManager = null; } } @@ -244,7 +246,7 @@ public static class DataLogFileIterator implements ClosableIterator { private final String[] orderingFields; private final TypedProperties props; - private ExternalSpillableMap beforeImages; + private Map beforeImages; private RowData currentImage; private RowData sideImage; @@ -278,15 +280,15 @@ public DataLogFileIterator( metaClient.getTableConfig().getPartialUpdateMode()); this.logRecordIterator = logRecordIterator; this.deleteContext = new DeleteContext(props, tableSchema).withReaderSchema(tableSchema); - initImages(cdcFileSplit, writeConfig); + initImages(cdcFileSplit); } - private void initImages(HoodieCDCFileSplit fileSplit, HoodieWriteConfig writeConfig) throws IOException { + private void initImages(HoodieCDCFileSplit fileSplit) throws IOException { if (fileSplit.getBeforeFileSlice().isPresent() && !fileSplit.getBeforeFileSlice().get().isEmpty()) { this.beforeImages = imageManager.getOrLoadImages( maxCompactionMemoryInBytes, fileSplit.getBeforeFileSlice().get()); } else { - this.beforeImages = FormatUtils.spillableMap(writeConfig, maxCompactionMemoryInBytes, getClass().getSimpleName()); + this.beforeImages = Collections.emptyMap(); } } @@ -339,7 +341,6 @@ public RowData next() { @Override public void close() { logRecordIterator.close(); - imageManager.close(); } @SuppressWarnings("unchecked") @@ -533,7 +534,12 @@ public BeforeImageIterator( this.maxCompactionMemoryInBytes = maxCompactionMemoryInBytes; this.projection = RowDataProjection.instance(requiredRowType, requiredPositions); this.imageManager = imageManager; - initImages(fileSplit); + try { + initImages(fileSplit); + } catch (IOException | RuntimeException | Error e) { + closeSuppressing(this, e); + throw e; + } } protected void initImages(HoodieCDCFileSplit fileSplit) throws IOException { From 662fd5fe834449be4d22d32e137865ba9646a4d9 Mon Sep 17 00:00:00 2001 From: voon Date: Thu, 6 Aug 2026 17:55:03 +0800 Subject: [PATCH 238/255] fix(build): import Lazy from its release-branch package in HoodieHadoopStorage The cherry-pick of #19470 (b8a09bf2ef9d) brought master's org.apache.hudi.common.util.Lazy import, but the hudi-common core package reorganization (#19195) that moved Lazy there is not on this branch. Lazy is still org.apache.hudi.util.Lazy here, so hudi-hadoop-common failed to compile with "cannot find symbol: class Lazy", breaking every downstream module. Note: 14 files under hudi-trino carry the same master-only Lazy import. That module is profile-gated off by default (-Phudi-trino) so it does not break the default build, and is left alone here. --- .../org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java index 067f77d491bd6..841ea2b40754c 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java @@ -20,7 +20,6 @@ package org.apache.hudi.storage.hadoop; import org.apache.hudi.common.fs.ConsistencyGuard; -import org.apache.hudi.common.util.Lazy; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.hadoop.fs.HoodieRetryWrapperFileSystem; @@ -32,6 +31,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathFilter; import org.apache.hudi.storage.StoragePathInfo; +import org.apache.hudi.util.Lazy; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; From ea9dda08aa0bc3b5fb660ed9270eecdc47103d09 Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Tue, 4 Aug 2026 21:06:57 +0530 Subject: [PATCH 239/255] fix(metrics): do not drop the whole CloudWatch batch on one unmappable metric name (#19476) * fix(metrics): do not drop the whole CloudWatch batch on one unmappable metric name stageMetricDatum derives the CloudWatch Table dimension from the part of the metric name before the first dot, so a name without one cannot be mapped. It threw for that case, and report() stages every metric into one list before calling putMetricData, so throwing part-way through meant the request was never sent: one unmappable name cost every metric in the interval. ScheduledReporter then suppresses the exception, so the user saw a log line and an empty dashboard. Such names still reach the reporter on master. HoodieMetadataMetrics#setMetric registers gauges with no prefix, unlike Metrics#registerGauges, so getStats contributes a bare partitionCount and BaseTableMetadata a bare lookup_meta_index_bloom_filters_file_count. Skip the metric that cannot be mapped and publish the rest, logging the name once rather than every interval. The intent of the previous check is kept - the metric is still not reported under a wrong table, and is now named in a warning - without taking the other metrics down with it. Fail-fast was never reachable here anyway, since ScheduledReporter suppresses whatever report() throws. * fix(metrics): also skip metrics whose table name is empty, and cover the log-once path Review feedback. The headline example in this PR was unreachable and I have replaced it. partitionCount comes from HoodieMetadataMetrics.getStats only when detailed == true, while the gauge-registering path calls getStats(false, ...); the only detailed=true caller is HoodieBackedTableMetadata.stats(), whose sole consumer prints the map in hudi-cli and registers nothing. The fixture and javadoc now use lookup_meta_index_bloom_filters_file_count, which BaseTableMetadata registers on the normal bloom-index read path. An empty first segment was still losing the batch, which is the same bug class this PR exists to fix. hoodie.metrics.reporter.metricsname.prefix defaults to "" and Metrics#registerGauges still joins it with a dot, so ".foo" splits into two parts, passed the length check, and asked CloudWatch for an empty Table dimension value - which it rejects for the entire PutMetricData request. The guard now also rejects an empty first segment. The warning no longer claims a

    . convention that Hudi does not follow: no metadata metric carries a table name, and an operator has no knob that changes the names being skipped. It now names the prefix config and points at #19507 for the producer side. Three tests added: the empty-table-name case, an interval where every metric is unmappable asserting that no empty PutMetricData request is sent, and one that reports twice and asserts a single WARN, so the once-per-name set is no longer uncovered. Both new guards fail the suite when reverted. (cherry picked from commit c63c9bfa79fdb5a606fd7452fdfb3510f2f5739a) --- .../cloudwatch/CloudWatchReporter.java | 26 +++- .../cloudwatch/TestCloudWatchReporter.java | 145 ++++++++++++++++-- 2 files changed, 158 insertions(+), 13 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java index ba9abc55bef79..470fda3dfa1f5 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java @@ -20,7 +20,7 @@ import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.common.util.StringUtils; import com.codahale.metrics.Clock; import com.codahale.metrics.Counter; @@ -45,7 +45,9 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.SortedMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -66,6 +68,8 @@ public class CloudWatchReporter extends ScheduledReporter { private final String prefix; private final String namespace; private final int maxDatumsPerRequest; + /** Metric names already reported as unmappable, so the warning is logged once rather than every interval. */ + private final Set unmappableMetricNames = ConcurrentHashMap.newKeySet(); public static Builder forRegistry(MetricRegistry registry) { return new Builder(registry); @@ -276,8 +280,24 @@ private void stageMetricDatum(String metricName, long timestampMilliSec, List metricData) { String[] metricNameParts = metricName.split("\\.", 2); - ValidationUtils.checkArgument(metricNameParts.length >= 2, - "metricName doesn't follow the naming convention and doesn't contain a dot as splitter! metricName:" + metricName); + if (metricNameParts.length < 2 || StringUtils.isNullOrEmpty(metricNameParts[0])) { + // The table dimension comes from the part before the first dot, so a name without one, or one whose + // first segment is empty, cannot be mapped. An empty first segment is reachable: + // hoodie.metrics.reporter.metricsname.prefix defaults to "" and Metrics#registerGauges still joins it + // with a dot, producing ".foo" - and CloudWatch rejects a whole PutMetricData request whose dimension + // value is empty, which would lose the batch again. + // + // Skip just this metric rather than throwing: ScheduledReporter suppresses whatever report() throws, + // so failing here dropped every metric staged in the same interval and left no metrics in CloudWatch + // at all. + if (unmappableMetricNames.add(metricName)) { + log.warn("Not reporting metric \"{}\" to CloudWatch: no table name can be derived for the Table " + + "dimension. Metric names normally carry hoodie.metrics.reporter.metricsname.prefix, but some " + + "Hudi-internal metadata metrics do not (see HUDI issue #19507). Other metrics in this batch " + + "are unaffected, and this is logged once per metric name.", metricName); + } + return; + } String tableName = metricNameParts[0]; metricData.add(MetricDatum.builder() diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java index 0073f3687db2b..d6fb7a4cfb7ed 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java @@ -27,6 +27,12 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,6 +49,8 @@ import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.SortedMap; import java.util.TreeMap; @@ -54,7 +62,6 @@ import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_METRIC_TYPE_KEY; import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_TABLE_NAME_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; @ExtendWith(MockitoExtension.class) public class TestCloudWatchReporter { @@ -168,21 +175,139 @@ public void testReporter() { Mockito.verify(cloudWatchAsync).close(); } + /** + * A metric name with no dot has no table name to report under, and such names do reach the reporter: + * {@code HoodieMetadataMetrics#setMetric} registers gauges without the metrics-name prefix, so + * {@code BaseTableMetadata#getBloomFilters} contributes a bare + * {@code lookup_meta_index_bloom_filters_file_count} on the normal bloom-index read path. This used to + * throw, and {@link com.codahale.metrics.ScheduledReporter} suppresses whatever {@code report()} throws, + * so no metrics reached CloudWatch at all - which is what #12182 and #13051 report. The unmappable metric + * is now skipped and the rest of the batch is still published. See HUDI issue #19507 for the producer side. + */ @Test - public void testReportOnMetricsWithoutTableName() { + public void testReportSkipsMetricsWithoutTableNameAndPublishesTheRest() { SortedMap gauges = new TreeMap<>(); - Gauge gauge1 = () -> 100L; - Gauge gauge2 = () -> 100.1; - gauges.put("gauge1", gauge1); - gauges.put(TABLE_NAME + ".gauge2", gauge2); + Gauge unmappable = () -> 7L; + Gauge wellFormed = () -> 100.1; + gauges.put("lookup_meta_index_bloom_filters_file_count", unmappable); + gauges.put(TABLE_NAME + ".gauge2", wellFormed); Mockito.when(metricRegistry.getGauges(MetricFilter.ALL)).thenReturn(gauges); - // should fail if metric name doesn't have at least two parts - assertThrows(IllegalArgumentException.class, () -> reporter.report()); + reporter.report(); - reporter.stop(); - Mockito.verify(cloudWatchAsync).close(); + Mockito.verify(cloudWatchAsync, Mockito.times(1)).putMetricData(putMetricDataRequestCaptor.capture()); + List metricData = putMetricDataRequestCaptor.getValue().metricData(); + assertEquals(1, metricData.size(), + "The unmappable metric should be skipped and the well-formed one still published"); + assertEquals(PREFIX + ".gauge2", metricData.get(0).metricName()); + assertEquals(wellFormed.getValue(), metricData.get(0).value()); + assertDimensions(metricData.get(0).dimensions(), DIMENSION_GAUGE_TYPE_VALUE); + } + + /** + * An empty first segment is reachable: {@code hoodie.metrics.reporter.metricsname.prefix} defaults to + * {@code ""} and {@code Metrics#registerGauges} still joins it with a dot, giving {@code ".foo"}. That + * splits into two parts and so passed the length check, then asked CloudWatch for an empty {@code Table} + * dimension value, which it rejects for the whole PutMetricData request - losing the batch again. + */ + @Test + public void testReportSkipsMetricsWithAnEmptyTableName() { + SortedMap gauges = new TreeMap<>(); + gauges.put(".gauge1", (Gauge) () -> 7L); + gauges.put(TABLE_NAME + ".gauge2", (Gauge) () -> 100L); + + Mockito.when(metricRegistry.getGauges(MetricFilter.ALL)).thenReturn(gauges); + + reporter.report(); + + Mockito.verify(cloudWatchAsync, Mockito.times(1)).putMetricData(putMetricDataRequestCaptor.capture()); + List metricData = putMetricDataRequestCaptor.getValue().metricData(); + assertEquals(1, metricData.size(), "a metric whose table name is empty should be skipped"); + assertEquals(PREFIX + ".gauge2", metricData.get(0).metricName()); + } + + /** + * An interval in which every metric is unmappable leaves nothing staged. CloudWatch rejects an empty + * PutMetricData request, so the reporter must not send one. + */ + @Test + public void testReportSendsNothingWhenEveryMetricIsUnmappable() { + SortedMap gauges = new TreeMap<>(); + gauges.put("lookup_meta_index_bloom_filters_file_count", (Gauge) () -> 7L); + gauges.put("bootstrap_error", (Gauge) () -> 1L); + + Mockito.when(metricRegistry.getGauges(MetricFilter.ALL)).thenReturn(gauges); + + reporter.report(); + + Mockito.verify(cloudWatchAsync, Mockito.never()).putMetricData(ArgumentMatchers.any(PutMetricDataRequest.class)); + } + + /** + * The unmappable-name set exists so a persistent offender is logged once rather than every reporting + * interval. Without this, deleting the set and logging unconditionally would pass the suite. + */ + @Test + public void testUnmappableMetricIsLoggedOncePerName() { + SortedMap gauges = new TreeMap<>(); + gauges.put("lookup_meta_index_bloom_filters_file_count", (Gauge) () -> 7L); + Mockito.when(metricRegistry.getGauges(MetricFilter.ALL)).thenReturn(gauges); + + CapturingAppender appender = CapturingAppender.attachTo(CloudWatchReporter.class); + try { + reporter.report(); + reporter.report(); + } finally { + appender.detach(); + } + + assertEquals(1, appender.warningsContaining("lookup_meta_index_bloom_filters_file_count"), + "a persistent unmappable name should be warned about once, not once per interval"); + } + + /** Captures WARN events from a single logger, so "logged once" can be asserted. */ + private static final class CapturingAppender extends AbstractAppender { + private final List warnings = Collections.synchronizedList(new ArrayList<>()); + private final LoggerConfig loggerConfig; + private final Level previousLevel; + + private CapturingAppender(LoggerConfig loggerConfig) { + super("CapturingAppender", null, null, true, null); + this.loggerConfig = loggerConfig; + this.previousLevel = loggerConfig.getLevel(); + } + + static CapturingAppender attachTo(Class loggerFor) { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + LoggerConfig loggerConfig = context.getConfiguration().getLoggerConfig(loggerFor.getName()); + CapturingAppender appender = new CapturingAppender(loggerConfig); + appender.start(); + loggerConfig.addAppender(appender, Level.WARN, null); + loggerConfig.setLevel(Level.WARN); + context.updateLoggers(); + return appender; + } + + void detach() { + loggerConfig.removeAppender(getName()); + loggerConfig.setLevel(previousLevel); + ((LoggerContext) LogManager.getContext(false)).updateLoggers(); + stop(); + } + + long warningsContaining(String needle) { + synchronized (warnings) { + return warnings.stream().filter(m -> m.contains(needle)).count(); + } + } + + @Override + public void append(LogEvent event) { + if (event.getLevel().isMoreSpecificThan(Level.WARN)) { + warnings.add(event.getMessage().getFormattedMessage()); + } + } } private void assertDimensions(List actualDimensions, String metricTypeDimensionVal) { From a85860f880bab11bf35eb00fe7bbdad1009971f1 Mon Sep 17 00:00:00 2001 From: voonhous Date: Thu, 6 Aug 2026 10:03:57 +0800 Subject: [PATCH 240/255] test(trino): add a Trino E2E testcontainers pipeline for the RFC-105 connector (#19217) * test(integ-test): add Trino E2E harness for the RFC-105 connector Revives the testcontainers Trino harness from the stale branch, adapted: - TrinoService + ITTestTrino{Smoke,StockTicks,CustomType}, gated on a new docker-compose 'trino' profile (-Dcompose.profiles=trino) so existing hive-sync CI rows are unaffected - docker/trino: Dockerfile (FROM trinodb/trino:481, bakes the plugin dir built from the Trino repo's thin trino-hudi shim + etc configs), overlay-aware entrypoint (jar-bearing TRINO_PLUGIN_DIR mount overrides the baked plugin for fast iteration), build_image.sh - trinocoordinator service (profile-gated) in the spark402 compose pair * ci(trino): complete the Trino E2E pipeline for the RFC-105 connector The harness from the previous commit could not actually run: nothing in the repo could assemble the Trino plugin directory (the RFC-105 shim is not yet released upstream), the stock-ticks seed fixture was never committed, and no workflow activated the trino compose profile. - docker/trino/shim: standalone maven project mirroring the upstream trinodb/trino plugin/trino-hudi shim (parent io.trino:trino-root:481 from Maven Central, packaging trino-plugin, depends on org.apache.hudi:hudi-trino). Carries its own thin HudiPlugin class because trino-maven-plugin's service descriptor generator only scans the module's own classes; never deployed (maven.deploy.skip). Build with package, never install. - docker/trino/Dockerfile + overlay-entrypoint.sh: preserve the stock image's plugin/hudi/hdfs loader jars at /opt/hudi-hdfs-lib and re-attach them when the baked or overlaid plugin dir lacks them (shim-assembled dirs always do; io.trino:trino-hdfs:zip is not on Maven Central, it only ships in the server tarball / base image). - docker/demo/sparksql-stock-ticks-trino.commands: the COW + MOR seed fixture ITTestTrinoStockTicks references, jdbc-mode hive sync like the existing custom-type fixtures. - .github/workflows/hudi_trino_e2e.yml: JDK 17 reactor build, JDK 25 connector + shim build, local image build via build_image.sh, then ITTestTrino* against the spark402 compose stack with -Dcompose.profiles=trino. No trinodb/trino checkout and no DockerHub publish anywhere in the flow. - docker/trino/.gitignore: anchor plugin/ to /plugin/ so it does not swallow the shim's io/trino/plugin source package. - READMEs: local E2E flow + overlay fast loop; drop stale hudi-trino-plugin path comments. * ci(trino): TEMP: push-trigger the E2E workflow on the dev branch workflow_dispatch cannot target a workflow that does not exist on the default branch yet. Drop this commit before merging upstream. * fix(trino): drop jts-core provided override in the E2E shim pom Trino 481's SpiDependencyChecker rejects provided scope for org.locationtech.jts:jts-core: it is not part of the 481 SPI surface (trino-spi 481 depends only on jackson-annotations, slice, and the opentelemetry api/api-incubator/common/context set). jts-core now flows into the assembled plugin dir at its transitive scope. The upstream 482-SNAPSHOT shim declares it provided because the SPI gains jts there. * test(integ-test): surface Trino boot failures in the E2E pipeline All three ITTestTrino* classes failed with 'execInContainer can only be used while the Container is running': the coordinator dies at startup and nothing captured why (compose teardown discards the container). - ITTestBaseTestcontainers: stream trinocoordinator logs into the test output via Slf4jLogConsumer when the trino profile is active, so a startup crash leaves its root cause in the failsafe report. - TrinoService.waitUntilReady: fail fast with a pointed message when the container is no longer running instead of exhausting 18 retries against a dead container (saves ~3 min per class). - hudi_trino_e2e.yml: assert the assembled plugin dir is populated and carries the services jar; add a standalone smoke-boot of the built image (docker run + SELECT 1 via the CLI, boot log dumped on failure) so image-level boot failures surface ~30 min before the IT step. * fix(trino): drop GCLocker JVM flags the Trino 481 JDK no longer accepts The standalone smoke-boot caught the coordinator crash root cause: 'Unrecognized VM option GCLockerRetryAllocationCount=32'. The flag was carried over from Hudi's JDK 11/17 CI OOM workaround, but the GCLocker was removed in modern JDKs, so the JDK 25 JVM inside trinodb/trino:481 refuses to start and the container dies before any Trino code runs -- which is exactly the 'container is not running' failure all three ITTestTrino* classes hit. * test(integ-test): surface server-side stacks for failing Trino queries The E2E run reached real query execution (ITTestTrinoSmoke fully green, the stock-ticks fixture seeds and syncs correctly) but every table read fails with 'Failed to generate splits for default.
    ' and only the one-line message survives into the report. - TrinoService: run the CLI with --debug so query failures carry the full server-side stack into the assertion message. - hudi_trino_e2e.yml: on failure, print the failsafe *-output.txt tails (where surefire redirects the streamed trinocoordinator logs). * ci(trino): add the ASF license header to the empty-overlay .gitkeep The release licensing check (scripts/release/validate_source_copyright.sh) requires every non-excluded file to contain the ASF header, and an empty .gitkeep cannot. The file only keeps docker/trino/empty-overlay in git as the compose default plugin-overlay mount source; its content is ignored at runtime (the overlay entrypoint only applies an overlay that contains jars), so the header plus a purpose note satisfies the check without touching the shared release script. * test(integ-test): address Trino E2E review on teardown, naming and MOR signal - Guard @AfterAll teardown in ITTestTrinoStockTicks/ITTestTrinoCustomType: JUnit runs @AfterAll even when the @BeforeAll assumption aborts setupOnce() before initializeServices(), so a non-trino stack NPE'd on the null spark service instead of skipping cleanly. - Rename the BothPartitionsVisible tests: the emptied dt=2024-01-02 partition is invisible by design, so the old names claimed the opposite of the check; pin the invisibility with an absence assertion. - Give stock_ticks_mor a log-only UPDATE delta so the _ro and _rt views diverge: _ro must keep serving the base row (now absence-pinned) and two new _rt tests verify the merged read, so the MOR rows exercise a genuinely MOR read path instead of the base-file-only path COW already covers. * ci(trino): harden the Trino E2E workflow and shim per review - Drop the TEMP dev-branch push trigger: only squash merge is enabled in .asf.yaml, so there is no separate commit to drop and the line would ship to master with the branch head. - Widen the demo paths filter to docker/demo/**: the ITs drive several demo fixtures beyond the stock-ticks commands file, and edits to those must re-trigger this pipeline. - Carry bot.yml's maven.wagon retry flags in MVN_ARGS: the JDK 17 step is the cold-cache full-reactor build those flags were added for. - Redirect failsafe test output to files so the on-failure dump step has the *-output.txt files it tails; failsafe defaults the redirect off, so the loop never matched anything before. - Derive dep.hudi.version from the reactor pom when building the shim: cut_release_branch.sh's versions:set cannot bump the out-of-reactor shim pom, and a stale literal could resolve silently from the actions maven cache instead of failing loudly. - Hard-disable install for the shim pom (maven.install.skip) so a stray install cannot publish a stub at the real io.trino:trino-hudi coordinates, and dockerignore the shim tree so shim/target stays out of the docker build context. * test(integ-test): retire the legacy trino-coordinator demo path The trino-coordinator-1 service is gone from every compose file, the ITTestHoodieDemo trino steps have been commented out since the HUDI-8269/HUDI-8270 breakage, and the integ2 Trino E2E suite now covers the same stock-ticks queries against the native RFC-105 connector. Remove the dead ITTestBase helpers and the ITTestHoodieDemo constants, copy steps and private test methods; the three trino-*.commands demo files they consumed were dropped in the preceding fixture commit. * ci(trino): add the ASF license header to the trino .dockerignore The release licensing check (scripts/release/validate_source_copyright.sh) greps every file in the source release tree for the ASF header. The release rsync (create_source_directory.sh) strips .gitignore files but has no exclude for .dockerignore, so the file ships in the source release and must carry the header, same as the empty-overlay .gitkeep. Docker treats leading # lines as comments, so the ignore behavior is unchanged. * ci(trino): use bot.yml's aether resolver retry flags and pin the GOOG row count - MVN_ARGS carried maven.wagon.* names, which Maven 3.9's native resolver transport ignores; swap in bot.yml's five aether.connector.http.* flags verbatim so the 429/5xx retry actually applies to the cold-cache build. - The _ro/_rt projected-column asserts matched one row substring, which still passes when an unmerged base row is returned next to it; pin the symbol count to one. * test(integ-test): correct the Trino custom-type javadoc and drop the dead Spark 3.5 guards Three doc/scope corrections from review on ITTestTrinoCustomType: - the class javadoc credited ITTestCustomTypeHiveSync with a re-seeding @BeforeAll and a cleaning @AfterAll, but it seeds inside each @Test and cleans in @AfterEach. What actually keeps the two classes from contaminating each other is tearDownDockerCompose dropping the whole stack between classes; say that instead. - trinocoordinator exists only in the spark402 compose pair, so this class either skips entirely (no trino profile) or runs on a Spark 4.x stack -- isSpark4Compose() is always true when it runs. Drop the conditional VARIANT seed and the seven assumeSpark4Compose() guards, and replace the "on Spark 3.5 the BLOB and VECTOR coverage still runs" claim with the reason VARIANT is unconditional here. - the two EmptiedPartitionInvisible tests claimed to exercise partition pruning, but GROUP BY dt only emits groups for partitions that still hold rows, so it cannot distinguish a connector that prunes from one that lists -- and dt=2024-01-02 stays registered in the metastore either way. Reword both comments to state what is actually asserted. Also trims the now-stale usage guidance on isSpark4Compose(), which no longer has a @BeforeAll-seeding caller. * ci(trino): widen the E2E path filter to hadoop.env and fix the shim/catalog docs Review round 3 on the Trino E2E pipeline: - ITTestBaseTestcontainers copies docker/compose/hadoop.env into the generated compose dir and all nine services in the spark402 pair load it, but the workflow path filter matched only the spark402 YAMLs. Add hadoop.env to both the push and pull_request lists. - hudi.properties dated the native connector at Trino 472. plugin/trino-hudi first appears upstream at tag 398 (404 at 397), so correct the number. - The shim build steps in hudi-trino/README.md and docker/README.md used the pom's literal dep.hudi.version, which the workflow itself documents as going stale because cut_release_branch.sh cannot bump a pom outside the reactor. Derive it from help:evaluate the same way the workflow does. - Rename Containers.TRINOCOORDINATOR to TRINO_COORDINATOR for consistency with SPARK_MASTER / ADHOC_1. The value stays "trinocoordinator" to match the compose service name. * ci(trino): trigger the E2E pipeline on root pom.xml and all of hudi-integ-test Two gaps in the path filter: - root pom.xml owns trino.version (481), which the shim pom's parent coordinate, the Dockerfile TRINO_VERSION arg, build_image.sh's default and four hardcoded trino-hudi-481 paths in this workflow all track by hand. A bump there needs this pipeline to run; hudi_trino_ci.yml already lists pom.xml for the same reason. - the test entry was scoped to integ2/**, but the IT step runs `mvn verify -pl hudi-integ-test`, which compiles the whole module. This PR's own edits to integ/ITTestBase and integ/ITTestHoodieDemo sit outside integ2/, so a break there would fail this pipeline without triggering it. Widen to hudi-integ-test/**. Also folds the per-entry comments into one rationale block above the push list, with the pull_request list pointing at it, so the two stay easy to diff against each other. (cherry picked from commit 0db52460cfa6a5ce74115e43f75709a99821fee2) --- .github/workflows/hudi_trino_e2e.yml | 186 +++++++++ docker/README.md | 27 ++ ...pose_hadoop340_hive2310_spark402_amd64.yml | 23 ++ ...pose_hadoop340_hive2310_spark402_arm64.yml | 25 +- .../demo/sparksql-stock-ticks-trino.commands | 98 +++++ docker/demo/trino-batch1.commands | 23 -- .../trino-batch2-after-compaction.commands | 21 - docker/demo/trino-table-check.commands | 20 - docker/trino/.dockerignore | 21 + docker/trino/.gitignore | 3 + docker/trino/Dockerfile | 51 +++ docker/trino/build_image.sh | 77 ++++ docker/trino/empty-overlay/.gitkeep | 21 + docker/trino/etc/catalog/hudi.properties | 31 ++ docker/trino/etc/config.properties | 26 ++ docker/trino/etc/hadoop-conf/core-site.xml | 23 ++ docker/trino/etc/hadoop-conf/hdfs-site.xml | 27 ++ docker/trino/etc/jvm.config | 38 ++ docker/trino/etc/node.properties | 22 ++ docker/trino/overlay-entrypoint.sh | 52 +++ docker/trino/shim/pom.xml | 161 ++++++++ .../java/io/trino/plugin/hudi/HudiPlugin.java | 39 ++ .../org/apache/hudi/integ/ITTestBase.java | 22 -- .../apache/hudi/integ/ITTestHoodieDemo.java | 61 +-- .../ITTestBaseTestcontainers.java | 55 ++- .../testcontainers/TestcontainersConfig.java | 21 + .../testcontainers/service/TrinoService.java | 117 ++++++ .../trino/ITTestTrinoCustomType.java | 369 ++++++++++++++++++ .../trino/ITTestTrinoSmoke.java | 71 ++++ .../trino/ITTestTrinoStockTicks.java | 134 +++++++ hudi-trino/README.md | 42 ++ 31 files changed, 1757 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/hudi_trino_e2e.yml create mode 100644 docker/demo/sparksql-stock-ticks-trino.commands delete mode 100644 docker/demo/trino-batch1.commands delete mode 100644 docker/demo/trino-batch2-after-compaction.commands delete mode 100644 docker/demo/trino-table-check.commands create mode 100644 docker/trino/.dockerignore create mode 100644 docker/trino/.gitignore create mode 100644 docker/trino/Dockerfile create mode 100755 docker/trino/build_image.sh create mode 100644 docker/trino/empty-overlay/.gitkeep create mode 100644 docker/trino/etc/catalog/hudi.properties create mode 100644 docker/trino/etc/config.properties create mode 100644 docker/trino/etc/hadoop-conf/core-site.xml create mode 100644 docker/trino/etc/hadoop-conf/hdfs-site.xml create mode 100644 docker/trino/etc/jvm.config create mode 100644 docker/trino/etc/node.properties create mode 100755 docker/trino/overlay-entrypoint.sh create mode 100644 docker/trino/shim/pom.xml create mode 100644 docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java create mode 100644 hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java diff --git a/.github/workflows/hudi_trino_e2e.yml b/.github/workflows/hudi_trino_e2e.yml new file mode 100644 index 0000000000000..df2f945e36d34 --- /dev/null +++ b/.github/workflows/hudi_trino_e2e.yml @@ -0,0 +1,186 @@ +name: Hudi Trino E2E + +on: + push: + branches: + - master + - 'release-*' + paths: + # Several entries are deliberately broad -- keep them that way: + # docker/demo/** the ITs drive demo fixture scripts + # (sparksql-*.commands, setup_demo_container.sh). + # docker/compose/hadoop.env + # copied into the generated compose dir and loaded + # by every service in the stack. + # hudi-integ-test/** the IT step runs `mvn verify -pl hudi-integ-test`, + # which compiles the whole module, so a break + # anywhere in it fails this pipeline -- not just + # under integ2/. + # pom.xml owns trino.version, which the shim pom's parent, + # the Dockerfile TRINO_VERSION arg and the + # hardcoded 481 paths below all track by hand. + - 'hudi-trino/**' + - 'docker/trino/**' + - 'docker/compose/docker-compose_hadoop340_hive2310_spark402*' + - 'docker/compose/hadoop.env' + - 'docker/demo/**' + - 'hudi-integ-test/**' + - 'pom.xml' + - '.github/workflows/hudi_trino_e2e.yml' + pull_request: + branches: + - master + - 'release-*' + # Keep in sync with the push paths above; see the rationale there. + paths: + - 'hudi-trino/**' + - 'docker/trino/**' + - 'docker/compose/docker-compose_hadoop340_hive2310_spark402*' + - 'docker/compose/hadoop.env' + - 'docker/demo/**' + - 'hudi-integ-test/**' + - 'pom.xml' + - '.github/workflows/hudi_trino_e2e.yml' + workflow_dispatch: + +concurrency: + group: hudi-trino-e2e-${{ github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master') && !contains(github.ref, 'release-') }} + +env: + # The aether.connector.http.* retry/timeout knobs are copied verbatim from bot.yml's + # MVN_ARGS: the JDK 17 step below is a cold-cache full-reactor build, exactly what + # those were added for. They are maven-resolver properties -- Maven 3.9 resolves through + # the native resolver HTTP transport, which ignores the older maven.wagon.* names. + MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.interval=3000 -Daether.connector.http.retryHandler.intervalMax=30000 -Daether.connector.http.retryHandler.serviceUnavailable=429,500,502,503,504 -Daether.connector.http.connectionMaxTtl=25 + SCALA_PROFILE: -Dscala-2.13 -Dscala.binary.version=2.13 + COMPOSE_PREFIX: docker-compose_hadoop340_hive2310_spark402 + +jobs: + trino-e2e: + # Testcontainers E2E for the RFC-105 native trino-hudi connector: builds + # hudi-trino at HEAD, assembles the plugin dir via the in-repo shim + # (docker/trino/shim, standing in for the not-yet-released upstream + # trinodb/trino plugin/trino-hudi shim), bakes it into a local + # apachehudi/hudi-trino_481 image, and runs ITTestTrino* against the + # spark402 compose stack (the only pair with the trinocoordinator service). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + docker system prune --all --force --volumes + - name: Pre-pull compose images (fails fast if not published) + run: | + # Surface a missing sparkadhoc image before the long Maven install. The + # remaining stack images are pulled by docker-compose at test time; the + # trino image is built locally below, never pulled. + docker pull apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + architecture: x64 + cache: maven + - name: Build and install Hudi artifacts (JDK 17) + # Full reactor: the compose containers mount the workspace and the tests + # use bundles staged by the -Pintegration-tests build (e.g. + # docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar). + run: + mvn clean install -T 2 $SCALA_PROFILE -Dspark4.0 -Dflink1.20 -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + java-version: '25' + distribution: 'temurin' + cache: maven + - name: Build hudi-trino connector (JDK 25) + # No trinodb/trino checkout needed: the unpublished Trino test-jars sit + # behind the off-by-default hudi-trino-tests profile and packaging + # resolves entirely from Maven Central. + run: + mvn $MVN_ARGS -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + - name: Assemble trino-hudi plugin dir via in-repo shim (JDK 25) + # package, NOT install: installing would shadow the real + # io.trino:trino-hudi release coordinates in the local m2 (the shim pom + # also hard-disables install via maven.install.skip). + # dep.hudi.version is derived from the reactor pom because the shim sits + # outside the reactor: cut_release_branch.sh's `mvn versions:set` cannot + # bump its literal default, and a stale value could resolve silently + # from the actions maven cache instead of failing loudly. + run: | + HUDI_VERSION=$(mvn -q -ntp help:evaluate -Dexpression=project.version -DforceStdout) + echo "Building shim against hudi version: $HUDI_VERSION" + mvn $MVN_ARGS -f docker/trino/shim/pom.xml clean package -DskipTests -Ddep.hudi.version="$HUDI_VERSION" + - name: Build apachehudi/hudi-trino_481 image + run: | + docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 + # Sanity: the shim must have produced a populated plugin dir with a + # service descriptor jar, or Trino cannot load the plugin at boot. + echo "plugin dir jar count: $(ls docker/trino/shim/target/trino-hudi-481 | wc -l)" + ls docker/trino/shim/target/trino-hudi-481/*services*.jar + - name: Smoke-boot the Trino image standalone + # Catches image-level boot failures (plugin load errors, bad etc/ config) + # ~30 min before the IT step would, with the full boot log on screen. + # --hostname trinocoordinator makes the baked discovery.uri self-resolve. + run: | + docker run -d --name trino-smoke --hostname trinocoordinator \ + apachehudi/hudi-trino_481:latest + ok="" + for i in $(seq 1 18); do + if [ "$(docker inspect -f '{{.State.Running}}' trino-smoke)" != "true" ]; then + echo "trino-smoke container died during startup" >&2 + break + fi + if docker exec trino-smoke trino --server localhost:8080 \ + --execute "SELECT 1" >/dev/null 2>&1; then + ok=1; echo "Trino answered SELECT 1 (attempt $i)"; break + fi + sleep 10 + done + if [ -z "$ok" ]; then + echo "==== trino-smoke boot log ====" + docker logs trino-smoke 2>&1 | tail -200 + docker rm -f trino-smoke >/dev/null 2>&1 || true + exit 1 + fi + docker rm -f trino-smoke + - name: Set up JDK 17 (restore for the IT run) + # setup-java resets JAVA_HOME on each call; hudi-integ-test needs 17. + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + architecture: x64 + - name: Run Trino E2E ITs (JDK 17) + run: | + # -DskipITs=false overrides the spark4.0 profile's skipITs=true default + # (see root pom.xml). -Dcompose.profiles=trino starts the profile-gated + # trinocoordinator service; without it every ITTestTrino* class skips. + # redirectTestOutputToFile makes failsafe write per-class *-output.txt + # files, which the on-failure dump step below relies on. + mvn verify $SCALA_PROFILE -Dspark4.0 -Pintegration-tests \ + -pl hudi-integ-test \ + -DskipITs=false \ + -Ddocker.compose.skip=true \ + -Dit.test='ITTestTrino*' \ + -Dcompose.profiles=trino \ + -Dspark.docker.compose.prefix=$COMPOSE_PREFIX \ + -Dmaven.test.redirectTestOutputToFile=true \ + $MVN_ARGS + - name: Dump failsafe test outputs on failure + # The IT step redirects test stdout (incl. the streamed trinocoordinator + # boot/query logs) into per-class output files; print their tails so + # server-side failures are readable straight from the workflow log. + if: failure() + run: | + for f in hudi-integ-test/target/failsafe-reports/*-output.txt; do + [ -f "$f" ] || continue + echo "===== $f (last 400 lines) =====" + tail -n 400 "$f" + done diff --git a/docker/README.md b/docker/README.md index 1563ce76aeb11..8892399fb33ff 100644 --- a/docker/README.md +++ b/docker/README.md @@ -196,3 +196,30 @@ When `--multi-arch` is enabled, the script builds and pushes the amd64 and arm64 Note that `--multi-arch` uses `docker buildx build --push` and the image names in the script are hardcoded to the `apachehudi/...` Docker Hub repositories, so this flow requires push access to those repositories. No Dockerfile changes are needed for the current amd64 plus arm64 image set in this repository. + +## Trino E2E image - `/trino` + +The Trino E2E stack does not use the `hoodie/hadoop` image tree. `docker/trino/` builds +`apachehudi/hudi-trino_` directly on top of the official `trinodb/trino` +image, baking in a locally-assembled native `trino-hudi` plugin directory and the E2E +catalog config (`connector.name=hudi`, metastore at `thrift://hivemetastore:9083`). + +This image is built locally on demand (also by the `hudi_trino_e2e.yml` CI workflow) and +is NOT published to Docker Hub. The plugin directory comes from the in-repo shim project +at `docker/trino/shim/` (see `hudi-trino/README.md` for the full build-and-run flow): + +``` +# JDK 25; hudi-trino must already be installed into the local m2 +# dep.hudi.version comes from the reactor pom -- the shim is outside the reactor, so +# cut_release_branch.sh cannot bump the literal default in its own pom. +HUDI_VERSION=$(mvn -q -ntp help:evaluate -Dexpression=project.version -DforceStdout) +mvn -f docker/trino/shim/pom.xml clean package -DskipTests -Ddep.hudi.version="$HUDI_VERSION" +docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 +``` + +The `trinocoordinator` compose service exists only in the +`docker-compose_hadoop340_hive2310_spark402_{amd64,arm64}.yml` pair, behind the `trino` +compose profile, so the default hive-sync flows never start it. For fast plugin +iteration the container supports a bind-mounted overlay: point `TRINO_PLUGIN_DIR` (or +the `-Dtrino.plugin.dir` test property) at a freshly built plugin dir and restart the +container instead of rebuilding the image. diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml index 0cd441eef2c56..adf964e2288b7 100644 --- a/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml @@ -256,6 +256,29 @@ services: depends_on: - minio + # Gated behind the "trino" compose profile: inert for the default hive-sync CI + # rows, only starts when COMPOSE_PROFILES=trino. The plugin overlay defaults to + # docker/trino/empty-overlay (baked-in plugin used); set TRINO_PLUGIN_DIR to a + # locally-built trino-hudi plugin dir to override it at container start. + trinocoordinator: + image: apachehudi/hudi-trino_481:latest + profiles: ["trino"] + hostname: trinocoordinator + container_name: trinocoordinator + ports: + - "8092:8080" + depends_on: + - "hivemetastore" + - "namenode" + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${TRINO_PLUGIN_DIR:-${HUDI_WS}/docker/trino/empty-overlay}:/opt/hudi-plugin-overlay:ro + - ${HUDI_WS}:/var/hoodie/ws + volumes: namenode: historyserver: diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml index edc1a36bd28ac..adf964e2288b7 100644 --- a/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml @@ -78,7 +78,7 @@ services: volumes: - historyserver:/hadoop/yarn/timeline - # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 → HS2 2.3.10). + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. @@ -256,6 +256,29 @@ services: depends_on: - minio + # Gated behind the "trino" compose profile: inert for the default hive-sync CI + # rows, only starts when COMPOSE_PROFILES=trino. The plugin overlay defaults to + # docker/trino/empty-overlay (baked-in plugin used); set TRINO_PLUGIN_DIR to a + # locally-built trino-hudi plugin dir to override it at container start. + trinocoordinator: + image: apachehudi/hudi-trino_481:latest + profiles: ["trino"] + hostname: trinocoordinator + container_name: trinocoordinator + ports: + - "8092:8080" + depends_on: + - "hivemetastore" + - "namenode" + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${TRINO_PLUGIN_DIR:-${HUDI_WS}/docker/trino/empty-overlay}:/opt/hudi-plugin-overlay:ro + - ${HUDI_WS}:/var/hoodie/ws + volumes: namenode: historyserver: diff --git a/docker/demo/sparksql-stock-ticks-trino.commands b/docker/demo/sparksql-stock-ticks-trino.commands new file mode 100644 index 0000000000000..e90d591db617e --- /dev/null +++ b/docker/demo/sparksql-stock-ticks-trino.commands @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Self-contained COW + MOR stock-ticks seed for the Trino E2E tests +// (ITTestTrinoStockTicks). Mirrors the retired trino-batch1.commands data +// shape without the Kafka/streaming pipeline, which integ2 does not exercise. +// ts stays STRING so Trino's CSV_UNQUOTED output matches the test's exact +// row assertion: GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5 +spark.sql(""" + CREATE TABLE stock_ticks_cow ( + symbol STRING, + ts STRING, + volume LONG, + open DOUBLE, + close DOUBLE, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/stock_ticks_cow' + TBLPROPERTIES ( + 'primaryKey' = 'symbol', + 'preCombineField' = 'ts', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'stock_ticks_cow', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql("INSERT INTO stock_ticks_cow VALUES ('GOOG', '2018-08-31 10:29:00', 6330, 1230.5, 1230.5, '2018-08-31')") +spark.sql("select symbol, ts, volume, open, close from stock_ticks_cow").show(10, false) +println("STOCK_TICKS_COW_SETUP_SUCCESS") + +// MOR variant: identical schema and seed row. 'type' = 'mor' makes hive sync +// register stock_ticks_mor_ro / stock_ticks_mor_rt; the initial insert writes +// parquet base files, and the follow-up UPDATE below adds a log-only delta so +// the _ro and _rt views actually diverge. +spark.sql(""" + CREATE TABLE stock_ticks_mor ( + symbol STRING, + ts STRING, + volume LONG, + open DOUBLE, + close DOUBLE, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/stock_ticks_mor' + TBLPROPERTIES ( + 'type' = 'mor', + 'primaryKey' = 'symbol', + 'preCombineField' = 'ts', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'stock_ticks_mor', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql("INSERT INTO stock_ticks_mor VALUES ('GOOG', '2018-08-31 10:29:00', 6330, 1230.5, 1230.5, '2018-08-31')") + +// Log-only delta on the same key: UPDATE routes through upsert, so the existing +// file group gains a log file that _ro must ignore (base row: 10:29:00) and +// _rt must merge (10:59:00). One delta commit stays far below the inline +// compaction threshold, so the log survives for the read-path split to matter. +// open/close use .25/.5 so the double renders exactly in Trino's CSV output. +spark.sql("UPDATE stock_ticks_mor SET ts = '2018-08-31 10:59:00', volume = 9021, open = 1227.25, close = 1227.5 WHERE symbol = 'GOOG'") +spark.sql("select symbol, ts, volume, open, close from stock_ticks_mor").show(10, false) +println("STOCK_TICKS_MOR_SETUP_SUCCESS") + +// Debug aid: proves stock_ticks_mor_ro / stock_ticks_mor_rt got registered. +spark.sql("show tables").show(100, false) +println("STOCK_TICKS_TRINO_SETUP_SUCCESS") diff --git a/docker/demo/trino-batch1.commands b/docker/demo/trino-batch1.commands deleted file mode 100644 index d89c19b0bf0bf..0000000000000 --- a/docker/demo/trino-batch1.commands +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOOG'; -select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_cow where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; diff --git a/docker/demo/trino-batch2-after-compaction.commands b/docker/demo/trino-batch2-after-compaction.commands deleted file mode 100644 index da42b4728252d..0000000000000 --- a/docker/demo/trino-batch2-after-compaction.commands +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; diff --git a/docker/demo/trino-table-check.commands b/docker/demo/trino-table-check.commands deleted file mode 100644 index 4362d79fe770c..0000000000000 --- a/docker/demo/trino-table-check.commands +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -show tables; diff --git a/docker/trino/.dockerignore b/docker/trino/.dockerignore new file mode 100644 index 0000000000000..3471b8973e1fd --- /dev/null +++ b/docker/trino/.dockerignore @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Keep the shim build tree out of the docker build context; build_image.sh +# stages the one plugin dir the Dockerfile needs into plugin/. +shim/ diff --git a/docker/trino/.gitignore b/docker/trino/.gitignore new file mode 100644 index 0000000000000..939686302a858 --- /dev/null +++ b/docker/trino/.gitignore @@ -0,0 +1,3 @@ +# Transient staging dir populated by build_image.sh (leading slash: must not +# swallow the shim's io/trino/plugin/ source package under shim/). +/plugin/ diff --git a/docker/trino/Dockerfile b/docker/trino/Dockerfile new file mode 100644 index 0000000000000..3189368a2283d --- /dev/null +++ b/docker/trino/Dockerfile @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG TRINO_VERSION=481 +FROM trinodb/trino:${TRINO_VERSION} + +USER root + +# Replace the bundled hudi plugin with the locally-built trino-hudi plugin +# staged into the build context at plugin/ (see build_image.sh). +# +# The staged plugin dir carries only the connector's top-level runtime jars. +# fs.hadoop.enabled=true additionally needs the isolated HDFS loader jar set at +# /hdfs (io.trino.filesystem.manager.HdfsFileSystemLoader). That jar set +# is only distributed inside the trino-server tarball / base image +# (io.trino:trino-hdfs:zip is not on Maven Central), so preserve the stock hudi +# plugin's version-matched copy before replacing it, and re-attach it when the +# staged plugin dir lacks one. /opt/hudi-hdfs-lib stays in the image so the +# overlay entrypoint can do the same for bind-mounted plugin overlays. +RUN cp -r /usr/lib/trino/plugin/hudi/hdfs /opt/hudi-hdfs-lib \ + && rm -rf /usr/lib/trino/plugin/hudi +COPY --chown=trino:trino plugin/ /usr/lib/trino/plugin/hudi/ +RUN if [ ! -d /usr/lib/trino/plugin/hudi/hdfs ]; then \ + cp -r /opt/hudi-hdfs-lib /usr/lib/trino/plugin/hudi/hdfs; \ + fi \ + && chown -R trino:trino /usr/lib/trino/plugin/hudi /opt/hudi-hdfs-lib + +# Bake the Hudi E2E Trino config (coordinator, catalog, hadoop-conf) into the image. +COPY --chown=trino:trino etc/ /etc/trino/ + +# Overlay-aware entrypoint: a bind-mounted plugin overlay (if present) replaces +# the baked-in plugin at container start, otherwise the baked-in plugin is used. +COPY --chown=trino:trino overlay-entrypoint.sh /opt/overlay-entrypoint.sh +RUN chmod +x /opt/overlay-entrypoint.sh + +USER trino +ENTRYPOINT ["/opt/overlay-entrypoint.sh"] diff --git a/docker/trino/build_image.sh b/docker/trino/build_image.sh new file mode 100755 index 0000000000000..da1cc8fe2ef13 --- /dev/null +++ b/docker/trino/build_image.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the apachehudi/hudi-trino_ image with a locally-built +# trino-hudi plugin baked in. The plugin dir (typically the in-repo shim's +# docker/trino/shim/target/trino-hudi-, see docker/trino/shim/pom.xml) is +# staged into the build context at docker/trino/plugin/ (gitignored), then +# baked into the image. +# Usage: ./build_image.sh --plugin-dir [--trino-version ] [--image-tag ] +# Typical: ./build_image.sh --plugin-dir "$(dirname "$0")/shim/target/trino-hudi-481" +# Note: --trino-version must match the shim pom's parent version and the root +# pom's trino.version property. + +set -e + +# Default values +PLUGIN_DIR="" +TRINO_VERSION="481" +IMAGE_TAG="latest" + +# Parse command-line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + --plugin-dir) PLUGIN_DIR="$2"; shift ;; + --trino-version) TRINO_VERSION="$2"; shift ;; + --image-tag) IMAGE_TAG="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +# Directory of this script, so the build context path is stable regardless of cwd +SCRIPT_DIR=$(cd $(dirname "$0") && pwd) + +# Validate --plugin-dir: required, must exist and be non-empty +if [ -z "$PLUGIN_DIR" ]; then + echo "Error: --plugin-dir is required (the locally-built trino-hudi plugin directory)." >&2 + exit 1 +fi +if [ ! -d "$PLUGIN_DIR" ]; then + echo "Error: plugin dir '$PLUGIN_DIR' does not exist." >&2 + exit 1 +fi +if [ -z "$(ls -A "$PLUGIN_DIR" 2>/dev/null)" ]; then + echo "Error: plugin dir '$PLUGIN_DIR' is empty." >&2 + exit 1 +fi + +# Stage the plugin into the build context (plugin/ must be IN the context to be COPY-able) +STAGE_DIR="$SCRIPT_DIR/plugin" +echo "Staging plugin from '$PLUGIN_DIR' into '$STAGE_DIR'" +rm -rf "$STAGE_DIR" +cp -r "$PLUGIN_DIR" "$STAGE_DIR" + +IMAGE="apachehudi/hudi-trino_${TRINO_VERSION}:${IMAGE_TAG}" +echo "Building $IMAGE (TRINO_VERSION=${TRINO_VERSION})" +docker build --build-arg TRINO_VERSION="${TRINO_VERSION}" -t "$IMAGE" "$SCRIPT_DIR" + +# Clean up the staged plugin dir +echo "Cleaning up staged plugin dir '$STAGE_DIR'" +rm -rf "$STAGE_DIR" + +echo "Done: $IMAGE" diff --git a/docker/trino/empty-overlay/.gitkeep b/docker/trino/empty-overlay/.gitkeep new file mode 100644 index 0000000000000..9e386d0cdd886 --- /dev/null +++ b/docker/trino/empty-overlay/.gitkeep @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This file exists only to keep the directory in git: it is the compose +# default (empty) mount source for the trino-hudi plugin overlay, and the +# overlay entrypoint applies an overlay only when it contains jars. diff --git a/docker/trino/etc/catalog/hudi.properties b/docker/trino/etc/catalog/hudi.properties new file mode 100644 index 0000000000000..14c17746a8972 --- /dev/null +++ b/docker/trino/etc/catalog/hudi.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Native trino-hudi connector (assembled from org.apache.hudi:hudi-trino by the +# docker/trino/shim project). +# Pre-Trino 398 the only way to read Hudi was the hive-connector + hudi-trino-bundle shim; +# plugin/trino-hudi landed upstream at 398 as the native replacement, hence +# connector.name=hudi. +connector.name=hudi +hive.metastore=thrift +hive.metastore.uri=thrift://hivemetastore:9083 +# trino-filesystem-manager flag that turns on the legacy Hadoop FileSystem path +# (HDFS via fs.defaultFS in hive.config.resources). Without this the plugin can't +# read hdfs:// URIs in Trino 460+. +fs.hadoop.enabled=true +hive.config.resources=/etc/trino/hadoop-conf/core-site.xml,/etc/trino/hadoop-conf/hdfs-site.xml diff --git a/docker/trino/etc/config.properties b/docker/trino/etc/config.properties new file mode 100644 index 0000000000000..8239eacffdf6d --- /dev/null +++ b/docker/trino/etc/config.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Single-node Trino: the coordinator also runs splits. Good enough for E2E, +# halves container startup vs a separate worker. +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://trinocoordinator:8080 +query.max-memory=2GB +query.max-memory-per-node=1GB diff --git a/docker/trino/etc/hadoop-conf/core-site.xml b/docker/trino/etc/hadoop-conf/core-site.xml new file mode 100644 index 0000000000000..455fbb9181d63 --- /dev/null +++ b/docker/trino/etc/hadoop-conf/core-site.xml @@ -0,0 +1,23 @@ + + + + + fs.defaultFS + hdfs://namenode:8020 + + diff --git a/docker/trino/etc/hadoop-conf/hdfs-site.xml b/docker/trino/etc/hadoop-conf/hdfs-site.xml new file mode 100644 index 0000000000000..5bd3bf51dffe3 --- /dev/null +++ b/docker/trino/etc/hadoop-conf/hdfs-site.xml @@ -0,0 +1,27 @@ + + + + + dfs.client.use.datanode.hostname + true + + + dfs.replication + 1 + + diff --git a/docker/trino/etc/jvm.config b/docker/trino/etc/jvm.config new file mode 100644 index 0000000000000..b1d8ff3772dc6 --- /dev/null +++ b/docker/trino/etc/jvm.config @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +-server +-Xmx2G +-XX:InitialRAMPercentage=80 +-XX:MaxRAMPercentage=80 +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+ExitOnOutOfMemoryError +-XX:+HeapDumpOnOutOfMemoryError +-XX:-OmitStackTraceInFastThrow +-XX:ReservedCodeCacheSize=512M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 +-Dfile.encoding=UTF-8 +# Allow loading dynamic agents (used by JOL, referenced by Trino's runtime). +-XX:+EnableDynamicAgentLoading +# NOTE: do NOT add -XX:GCLockerRetryAllocationCount here (Hudi's JDK 11/17 CI +# workaround): the GCLocker was removed in modern JDKs and the trinodb/trino:481 +# JVM (JDK 25) refuses to start on the unrecognized option. diff --git a/docker/trino/etc/node.properties b/docker/trino/etc/node.properties new file mode 100644 index 0000000000000..7e0222cc3aec0 --- /dev/null +++ b/docker/trino/etc/node.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +node.environment=hudi +# Fixed node.id so a container restart reuses the same identity in the discovery service. +node.id=hudi-trino-coordinator +node.data-dir=/data/trino diff --git a/docker/trino/overlay-entrypoint.sh b/docker/trino/overlay-entrypoint.sh new file mode 100755 index 0000000000000..2ba9b517e26a6 --- /dev/null +++ b/docker/trino/overlay-entrypoint.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Overlay-aware Trino entrypoint. If a plugin overlay is bind-mounted at +# /opt/hudi-plugin-overlay (set TRINO_PLUGIN_DIR to the in-repo shim's +# docker/trino/shim/target/trino-hudi- build output, or to a trinodb/trino +# checkout's plugin/trino-hudi/target/trino-hudi-), fully replace the +# image's baked-in trino-hudi plugin with it (rm -rf then copy), so plugin +# iterations need only a rebuild of that dir plus a container restart, not a +# docker image rebuild. Otherwise the image-baked plugin is used as-is. +set -euo pipefail + +OVERLAY=/opt/hudi-plugin-overlay +PLUGIN_DIR=/usr/lib/trino/plugin/hudi + +# The overlay counts as present only if it holds at least one jar: the compose +# default mount is docker/trino/empty-overlay, whose .gitkeep must not trigger +# a wipe of the baked-in plugin. +if [ -d "$OVERLAY" ] && [ -n "$(find "$OVERLAY" -name '*.jar' -print -quit 2>/dev/null)" ]; then + echo "Applying trino-hudi plugin overlay from $OVERLAY (fully replacing $PLUGIN_DIR)" + rm -rf "$PLUGIN_DIR" + mkdir -p "$PLUGIN_DIR" + cp -r "$OVERLAY"/. "$PLUGIN_DIR"/ +else + echo "No plugin overlay found at $OVERLAY; using the image-baked trino-hudi plugin as-is." +fi + +# Overlays built from the in-repo shim (docker/trino/shim/target/trino-hudi-) +# lack the hdfs/ loader dir that fs.hadoop.enabled=true needs; restore the copy +# the image preserved from the stock plugin (see Dockerfile). +if [ ! -d "$PLUGIN_DIR/hdfs" ] && [ -d /opt/hudi-hdfs-lib ]; then + echo "Restoring hdfs/ loader dir into $PLUGIN_DIR from /opt/hudi-hdfs-lib" + cp -r /opt/hudi-hdfs-lib "$PLUGIN_DIR/hdfs" +fi + +exec /usr/lib/trino/bin/run-trino diff --git a/docker/trino/shim/pom.xml b/docker/trino/shim/pom.xml new file mode 100644 index 0000000000000..c263b0d43b219 --- /dev/null +++ b/docker/trino/shim/pom.xml @@ -0,0 +1,161 @@ + + + + + 4.0.0 + + + io.trino + trino-root + 481 + + + + + trino-hudi + trino-plugin + Trino - Hudi connector plugin assembly (in-repo E2E shim mirroring the upstream plugin/trino-hudi shim planned by RFC-105; never deployed) + + + + 1.3.0-SNAPSHOT + + true + + true + true + true + + + + + + com.google.guava + guava + + + + org.apache.hudi + hudi-trino + ${dep.hudi.version} + + + + org.apache.arrow + * + + + org.apache.hudi + hudi-timeline-service + + + org.apache.orc + orc-core + + + org.lance + * + + + org.rocksdb + * + + + + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + opentelemetry-api-incubator + provided + + + + io.opentelemetry + opentelemetry-common + provided + + + + io.opentelemetry + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + + diff --git a/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java b/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java new file mode 100644 index 0000000000000..5c3f185ec6ab9 --- /dev/null +++ b/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import com.google.common.collect.ImmutableList; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + +/** + * Thin shim plugin mirroring the upstream trinodb/trino plugin/trino-hudi module + * (RFC-105). Same FQCN as the copy inside the hudi-trino jar - the duplication is + * intentional: trino-maven-plugin's service descriptor generator only scans this + * module's own classes, and both class bodies are identical, so classloader + * ordering does not matter. + */ +public class HudiPlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new HudiConnectorFactory()); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index 60493f98931d9..eba2270a430df 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -62,7 +62,6 @@ public abstract class ITTestBase { protected static final String ADHOC_2_CONTAINER = "/adhoc-2"; protected static final String HIVESERVER = "/hiveserver"; protected static final String PRESTO_COORDINATOR = "/presto-coordinator-1"; - protected static final String TRINO_COORDINATOR = "/trino-coordinator-1"; protected static final String HOODIE_WS_ROOT = "/var/hoodie/ws"; protected static final String HOODIE_JAVA_APP = HOODIE_WS_ROOT + "/hudi-spark-datasource/hudi-spark/run_hoodie_app.sh"; protected static final String HOODIE_GENERATE_APP = HOODIE_WS_ROOT + "/hudi-spark-datasource/hudi-spark/run_hoodie_generate_app.sh"; @@ -77,7 +76,6 @@ public abstract class ITTestBase { HOODIE_WS_ROOT + "/docker/hoodie/hadoop/hive_base/target/hoodie-utilities.jar"; protected static final String HIVE_SERVER_JDBC_URL = "jdbc:hive2://hiveserver:10000"; protected static final String PRESTO_COORDINATOR_URL = "presto-coordinator-1:8090"; - protected static final String TRINO_COORDINATOR_URL = "trino-coordinator-1:8091"; protected static final String HADOOP_CONF_DIR = "/etc/hadoop"; // Skip these lines when capturing output from hive @@ -126,12 +124,6 @@ static String getPrestoConsoleCommand(String commandFile) { .append(" -f " + commandFile).toString(); } - static String getTrinoConsoleCommand(String commandFile) { - return new StringBuilder().append("trino --server " + TRINO_COORDINATOR_URL) - .append(" --catalog hive --schema default") - .append(" -f " + commandFile).toString(); - } - @BeforeEach public void init() { String dockerHost = (OVERRIDDEN_DOCKER_HOST != null) ? OVERRIDDEN_DOCKER_HOST : DEFAULT_DOCKER_HOST; @@ -320,20 +312,6 @@ void executePrestoCopyCommand(String fromFile, String remotePath) { .exec(); } - Pair executeTrinoCommandFile(String commandFile) throws Exception { - String trinoCmd = getTrinoConsoleCommand(commandFile); - TestExecStartResultCallback callback = executeCommandStringInDocker(ADHOC_1_CONTAINER, trinoCmd, true); - return Pair.of(callback.getStdout().toString().trim(), callback.getStderr().toString().trim()); - } - - void executeTrinoCopyCommand(String fromFile, String remotePath) { - Container adhocContainer = runningContainers.get(ADHOC_1_CONTAINER); - dockerClient.copyArchiveToContainerCmd(adhocContainer.getId()) - .withHostResource(fromFile) - .withRemotePath(remotePath) - .exec(); - } - private void saveUpLogs() { try { // save up the Hive log files for introspection diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index d9d2c20dc2bb4..d9cc3e526f7f3 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -38,28 +38,18 @@ */ public class ITTestHoodieDemo extends ITTestBase { - private static final String TRINO_TABLE_CHECK_FILENAME = "trino-table-check.commands"; - private static final String TRINO_BATCH1_FILENAME = "trino-batch1.commands"; - private static final String TRINO_BATCH2_FILENAME = "trino-batch2-after-compaction.commands"; - private static final String HDFS_DATA_DIR = "/usr/hive/data/input"; private static final String HDFS_BATCH_PATH1 = HDFS_DATA_DIR + "/batch_1.json"; private static final String HDFS_BATCH_PATH2 = HDFS_DATA_DIR + "/batch_2.json"; private static final String HDFS_PRESTO_INPUT_TABLE_CHECK_PATH = HDFS_DATA_DIR + "/presto-table-check.commands"; private static final String HDFS_PRESTO_INPUT_BATCH1_PATH = HDFS_DATA_DIR + "/presto-batch1.commands"; private static final String HDFS_PRESTO_INPUT_BATCH2_PATH = HDFS_DATA_DIR + "/presto-batch2-after-compaction.commands"; - private static final String HDFS_TRINO_INPUT_TABLE_CHECK_PATH = HDFS_DATA_DIR + "/" + TRINO_TABLE_CHECK_FILENAME; - private static final String HDFS_TRINO_INPUT_BATCH1_PATH = HDFS_DATA_DIR + "/" + TRINO_BATCH1_FILENAME; - private static final String HDFS_TRINO_INPUT_BATCH2_PATH = HDFS_DATA_DIR + "/" + TRINO_BATCH2_FILENAME; private static final String INPUT_BATCH_PATH1 = HOODIE_WS_ROOT + "/docker/demo/data/batch_1.json"; private static final String PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH = "/docker/demo/presto-table-check.commands"; private static final String PRESTO_INPUT_BATCH1_RELATIVE_PATH = "/docker/demo/presto-batch1.commands"; private static final String INPUT_BATCH_PATH2 = HOODIE_WS_ROOT + "/docker/demo/data/batch_2.json"; private static final String PRESTO_INPUT_BATCH2_RELATIVE_PATH = "/docker/demo/presto-batch2-after-compaction.commands"; - private static final String TRINO_INPUT_TABLE_CHECK_RELATIVE_PATH = "/docker/demo/" + TRINO_TABLE_CHECK_FILENAME; - private static final String TRINO_INPUT_BATCH1_RELATIVE_PATH = "/docker/demo/" + TRINO_BATCH1_FILENAME; - private static final String TRINO_INPUT_BATCH2_RELATIVE_PATH = "/docker/demo/" + TRINO_BATCH2_FILENAME; private static final String COW_BASE_PATH = "/user/hive/warehouse/stock_ticks_cow"; private static final String MOR_BASE_PATH = "/user/hive/warehouse/stock_ticks_mor"; @@ -120,16 +110,15 @@ public void testParquetDemo() throws Exception { // batch 1 ingestFirstBatchAndHiveSync(); testHiveAfterFirstBatch(); - // TODO(HUDI-8269, HUDI-8270): fix integration tests with Presto and Trino + // TODO(HUDI-8269): fix integration tests with Presto. The legacy Trino demo + // path was retired in favor of the integ2 testcontainers Trino E2E suite. // testPrestoAfterFirstBatch(); - // testTrinoAfterFirstBatch(); testSparkSQLAfterFirstBatch(); // batch 2 ingestSecondBatchAndHiveSync(); testHiveAfterSecondBatch(); // testPrestoAfterSecondBatch(); - // testTrinoAfterSecondBatch(); testSparkSQLAfterSecondBatch(); // TODO: HUDI-8572 // testIncrementalHiveQueryBeforeCompaction(); @@ -141,7 +130,6 @@ public void testParquetDemo() throws Exception { testIncrementalSparkSQLQuery(); testHiveAfterSecondBatchAfterCompaction(); // testPrestoAfterSecondBatchAfterCompaction(); - // testTrinoAfterSecondBatchAfterCompaction(); // TODO: HUDI-8572 // testIncrementalHiveQueryAfterCompaction(); } @@ -159,14 +147,12 @@ public void testHFileDemo() throws Exception { ingestFirstBatchAndHiveSync(); testHiveAfterFirstBatch(); //testPrestoAfterFirstBatch(); - //testTrinoAfterFirstBatch(); //testSparkSQLAfterFirstBatch(); // batch 2 ingestSecondBatchAndHiveSync(); testHiveAfterSecondBatch(); //testPrestoAfterSecondBatch(); - //testTrinoAfterSecondBatch(); //testSparkSQLAfterSecondBatch(); testIncrementalHiveQueryBeforeCompaction(); //testIncrementalSparkSQLQuery(); @@ -175,7 +161,6 @@ public void testHFileDemo() throws Exception { scheduleAndRunCompaction(); testHiveAfterSecondBatchAfterCompaction(); //testPrestoAfterSecondBatchAfterCompaction(); - //testTrinoAfterSecondBatchAfterCompaction(); //testIncrementalHiveQueryAfterCompaction(); } @@ -196,10 +181,6 @@ private void setupDemo() throws Exception { executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); - - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); } private void ingestFirstBatchAndHiveSync() throws Exception { @@ -359,20 +340,6 @@ private void testPrestoAfterFirstBatch() throws Exception { "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\"", 2); } - private void testTrinoAfterFirstBatch() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_TABLE_CHECK_PATH); - assertStdOutContains(stdOutErrPair, "stock_ticks_cow", 2); - assertStdOutContains(stdOutErrPair, "stock_ticks_mor", 6); - - stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH1_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\"", 4); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\"", 2); - } - private void testHiveAfterSecondBatch() throws Exception { Pair stdOutErrPair = executeHiveCommandFile(HIVE_BATCH1_COMMANDS); assertStdOutContains(stdOutErrPair, "| symbol | _c1 |\n+---------+----------------------+\n" @@ -406,20 +373,6 @@ private void testPrestoAfterSecondBatch() throws Exception { "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); } - private void testTrinoAfterSecondBatch() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH1_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\""); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); - } - private void testHiveAfterSecondBatchAfterCompaction() throws Exception { Pair stdOutErrPair = executeHiveCommandFile(HIVE_BATCH2_COMMANDS); assertStdOutContains(stdOutErrPair, "| symbol | _c1 |\n+---------+----------------------+\n" @@ -442,16 +395,6 @@ private void testPrestoAfterSecondBatchAfterCompaction() throws Exception { "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); } - private void testTrinoAfterSecondBatchAfterCompaction() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH2_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\""); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); - } - private void testSparkSQLAfterSecondBatch() throws Exception { Pair stdOutErrPair = executeSparkSQLCommand(SPARKSQL_BATCH2_COMMANDS, true); assertStdOutContains(stdOutErrPair, diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java index 51fdd57b4bfbc..256278d1915e6 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java @@ -21,6 +21,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.integ2.testcontainers.service.HiveService; import org.apache.hudi.integ2.testcontainers.service.SparkService; +import org.apache.hudi.integ2.testcontainers.service.TrinoService; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; @@ -28,6 +29,7 @@ import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.ComposeContainer; import org.testcontainers.containers.ContainerState; +import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; @@ -59,6 +61,7 @@ public abstract class ITTestBaseTestcontainers implements ContainerProvider { // Service objects for interacting with different components protected HiveService hive; protected SparkService sparkAdhoc1; + protected TrinoService trino; @BeforeAll public static void setupDockerCompose() { @@ -75,6 +78,26 @@ public static void setupDockerCompose() { Wait.forListeningPort().forPorts(Network.SPARK_MASTER_WEB_UI_PORT) .withStartupTimeout(Timeouts.CONTAINER_STARTUP)) .withStartupTimeout(Timeouts.CONTAINER_STARTUP); + + // Activate optional compose profiles (e.g. "trino") when requested. Without this the + // profile-gated services stay down, which is the default hive-sync-only topology. + String composeProfiles = System.getProperty(SystemProps.COMPOSE_PROFILES_PROP, ""); + if (!composeProfiles.isEmpty()) { + environment.withEnv("COMPOSE_PROFILES", composeProfiles); + if (composeProfiles.contains(SystemProps.TRINO_PROFILE)) { + // Stream the coordinator's log into the test output. When Trino dies during + // startup (plugin load or config errors) the container is torn down with the + // stack, so this stream is the only place the root cause survives. + environment.withLogConsumer(Containers.TRINO_COORDINATOR, + new Slf4jLogConsumer(log).withPrefix(Containers.TRINO_COORDINATOR)); + } + } + // Point the compose stack at a host-built Trino plugin dir when supplied. The compose + // file falls back to an empty overlay when TRINO_PLUGIN_DIR is unset. + String trinoPluginDir = System.getProperty(SystemProps.TRINO_PLUGIN_DIR_PROP); + if (trinoPluginDir != null) { + environment.withEnv("TRINO_PLUGIN_DIR", trinoPluginDir); + } environment.start(); log.info("Docker Compose environment started successfully"); @@ -83,7 +106,7 @@ public static void setupDockerCompose() { /** * Tear down the compose stack between test classes. The docker-compose files publish - * host ports directly (zookeeper 2181, spark 7077, …), so leaving one + * host ports directly (zookeeper 2181, spark 7077, ...), so leaving one * stack up would make the next class's `@BeforeAll` collide on those host ports. * Testcontainers' Ryuk reaper only fires at JVM shutdown, which is too late when * failsafe reuses a JVM across classes. @@ -106,6 +129,32 @@ public static void tearDownDockerCompose() { protected void initializeServices() { this.hive = new HiveService(this); this.sparkAdhoc1 = new SparkService(this, Containers.ADHOC_1); + // Only wire the Trino service when its profile is active; otherwise the + // trinocoordinator container does not exist and getContainer would throw. + if (isTrinoProfileActive()) { + this.trino = new TrinoService(this); + } + } + + /** + * Returns {@code true} when the {@link SystemProps#COMPOSE_PROFILES_PROP} system + * property activates the {@code trino} compose profile, i.e. the Trino coordinator + * container is part of the running stack. + */ + protected static boolean isTrinoProfileActive() { + return System.getProperty(SystemProps.COMPOSE_PROFILES_PROP, "") + .contains(SystemProps.TRINO_PROFILE); + } + + /** + * Skips the test unless the {@code trino} compose profile is active. Use in the + * {@code @BeforeAll} of Trino ITs so they abort cleanly on a hive-sync-only stack + * where the coordinator container is absent. + */ + protected static void assumeTrinoProfile() { + Assumptions.assumeTrue(isTrinoProfileActive(), + "Test requires the 'trino' compose profile; run with -D" + + SystemProps.COMPOSE_PROFILES_PROP + "=" + SystemProps.TRINO_PROFILE); } /** @@ -120,9 +169,7 @@ protected static void assumeSpark4Compose() { /** * Non-assumption variant of {@link #assumeSpark4Compose()}: returns {@code true} when the - * active compose prefix points at a Spark 4.x stack. Use in {@code @BeforeAll} seeding - * to conditionally run Spark 4-only fixtures (e.g. VARIANT) without aborting the whole - * test class on a Spark 3.5 run. + * active compose prefix points at a Spark 4.x stack, without aborting the caller. */ protected static boolean isSpark4Compose() { String composePrefix = System.getProperty(SystemProps.COMPOSE_PREFIX, SystemProps.DEFAULT_COMPOSE_PREFIX); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java index ad403ff6cccd6..300fc661968bd 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java @@ -43,6 +43,7 @@ public static final class Containers { public static final String SPARK_MASTER = "sparkmaster"; // Testcontainers appends the replica index, so the adhoc services resolve as "-1". public static final String ADHOC_1 = "adhoc-1-1"; + public static final String TRINO_COORDINATOR = "trinocoordinator"; private Containers() { } @@ -66,6 +67,8 @@ private Paths() { /** Network endpoints the harness exposes to tests. */ public static final class Network { public static final int SPARK_MASTER_WEB_UI_PORT = 8080; + /** Container-internal Trino HTTP port. Tests exec the CLI inside the coordinator. */ + public static final int TRINO_PORT = 8080; private Network() { } @@ -76,6 +79,13 @@ public static final class Timeouts { public static final Duration CONTAINER_STARTUP = Duration.ofMinutes(5); public static final int HDFS_MAX_RETRIES = 12; public static final Duration HDFS_RETRY_INTERVAL = Duration.ofSeconds(10); + /** + * Trino's slow startup path is plugin discovery + metastore handshake. The CLI's + * first query against an unready coordinator returns a misleading error, so callers + * should retry up to this many times. + */ + public static final int TRINO_READY_MAX_RETRIES = 18; + public static final Duration TRINO_READY_RETRY_INTERVAL = Duration.ofSeconds(10); private Timeouts() { } @@ -88,6 +98,17 @@ public static final class SystemProps { /** Substring present in compose prefixes that run Spark 4.x (e.g. "...spark402"). */ public static final String SPARK_4_PREFIX_TOKEN = "spark4"; + /** + * Comma-separated Docker Compose profiles to activate (passed through to the + * compose stack as {@code COMPOSE_PROFILES}). The Trino services live behind the + * {@link #TRINO_PROFILE} profile, so they only start when it is present. + */ + public static final String COMPOSE_PROFILES_PROP = "compose.profiles"; + /** Host path to the built Trino Hudi plugin, mounted into the coordinator container. */ + public static final String TRINO_PLUGIN_DIR_PROP = "trino.plugin.dir"; + /** Compose profile name that gates the Trino coordinator/worker services. */ + public static final String TRINO_PROFILE = "trino"; + /** * Flip to {@code true} (e.g. {@code -Dhudi.integ.hive.verbose=true}) to route * Hive logs to the console so exception stack traces show up in test output. diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java new file mode 100644 index 0000000000000..4eb3398c7e1f2 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.service; + +import org.apache.hudi.integ2.testcontainers.ContainerProvider; +import org.apache.hudi.integ2.testcontainers.TestcontainersConfig; +import org.apache.hudi.integ2.testcontainers.command.CommandExecutor; +import org.apache.hudi.integ2.testcontainers.command.CommandResult; + +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.ContainerState; + +/** + * Service wrapper for the Trino coordinator. Mirrors {@link HiveService} in shape, but + * execs the bundled {@code trino} CLI inside the coordinator container itself rather + * than from an adhoc Spark container. The coordinator runs a hudi-built image based on + * {@code trinodb/trino:481}, which bundles a modern JDK; the adhoc Spark images do not, + * so {@code execInContainer("trino", ...)} on the coordinator is the reliable way to + * run Trino 481's CLI. + * + *

    The default catalog is {@code hudi} (the native trino-hudi connector registered + * by {@code HudiConnectorFactory#getName}) and the default schema is {@code default}. + * + *

    Output format is {@code CSV_UNQUOTED} so substring assertions stay simple and + * match the existing {@link HiveService} ergonomics. + */ +@Slf4j +public class TrinoService { + + private static final String CLI = "trino"; + private static final String SERVER = "localhost:" + TestcontainersConfig.Network.TRINO_PORT; + private static final String DEFAULT_CATALOG = "hudi"; + private static final String DEFAULT_SCHEMA = "default"; + + private final CommandExecutor executor; + private final ContainerState container; + + public TrinoService(ContainerProvider provider) { + this.container = provider.getContainer(TestcontainersConfig.Containers.TRINO_COORDINATOR); + this.executor = new CommandExecutor(container); + } + + /** + * Execute a single Trino SQL statement against the default {@code hudi.default} + * catalog/schema. Returns a {@link CommandResult} so the fluent assertions used by + * other services apply unchanged. + */ + public CommandResult execute(String sql) throws Exception { + return execute(DEFAULT_CATALOG, DEFAULT_SCHEMA, sql); + } + + /** + * Execute a single Trino SQL statement against an explicit catalog/schema. Useful + * for {@code SHOW CATALOGS} or cross-catalog probes where the schema is irrelevant. + */ + public CommandResult execute(String catalog, String schema, String sql) throws Exception { + String[] cmd = { + CLI, + "--server", SERVER, + "--catalog", catalog, + "--schema", schema, + "--output-format", "CSV_UNQUOTED", + // On failure the CLI prints the full server-side stack to stderr, which + // CommandResult embeds in the assertion message. No effect on success output. + "--debug", + "--execute", sql + }; + return executor.executeCommand(cmd); + } + + /** + * Block until the coordinator is ready to serve queries. Trino reports a healthy + * HTTP {@code /v1/info} well before plugin discovery finishes, so the cheapest + * reliable readiness probe is to actually issue a query. + */ + public void waitUntilReady() throws Exception { + int max = TestcontainersConfig.Timeouts.TRINO_READY_MAX_RETRIES; + long sleepMs = TestcontainersConfig.Timeouts.TRINO_READY_RETRY_INTERVAL.toMillis(); + for (int i = 1; i <= max; i++) { + try { + execute("system", "runtime", "SELECT 1").expectToSucceed(); + log.info("Trino coordinator is ready (attempt {}/{})", i, max); + return; + } catch (Throwable t) { + if (!container.isRunning()) { + // No point retrying against a dead container; the boot log streamed by the + // ITTestBaseTestcontainers log consumer holds the root cause. + throw new RuntimeException( + "trinocoordinator container is not running -- Trino likely crashed at startup;" + + " see the trinocoordinator-prefixed log lines above", t); + } + if (i == max) { + throw new RuntimeException( + "Trino coordinator did not become ready after " + max + " retries", t); + } + log.info("Waiting for Trino coordinator to be ready (attempt {}/{}): {}", i, max, t.getMessage()); + Thread.sleep(sleepMs); + } + } + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java new file mode 100644 index 0000000000000..677ee43cd21f5 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; +import org.apache.hudi.integ2.testcontainers.ITTestCustomTypeHiveSync; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Paths; + +/** + * Trino read coverage for Hudi's custom logical types (BLOB struct, VECTOR + * fixed_len_byte_array, VARIANT), complementing {@link ITTestCustomTypeHiveSync} + * which asserts the same fixtures round-trip through the Hive serde. A flip in + * either direction (e.g. VECTOR decoded as array<float> instead of + * binary, BLOB struct field projection broken, VARIANT row count off) shows up + * here. + * + *

    This test reuses the same {@code sparksql-*-sql.commands} fixtures that + * {@code ITTestCustomTypeHiveSync} drives. The two run in either order without + * cross-contamination because {@code tearDownDockerCompose} drops the whole + * stack between classes, so each starts against a fresh HDFS and metastore and + * re-seeds its own fixtures. + * + *

    The {@code trinocoordinator} service exists only in the spark402 compose + * pair, so this class always runs on a Spark 4.x stack and the VARIANT coverage + * is unconditional. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoCustomType extends ITTestBaseTestcontainers { + + private static final String BLOB_TEST_PATH = "/user/hive/warehouse/blob_test"; + private static final String BLOB_TEST_DF_PATH = "/user/hive/warehouse/blob_test_df"; + private static final String VECTOR_TEST_PATH = "/user/hive/warehouse/vector_test"; + private static final String VARIANT_TEST_PATH = "/user/hive/warehouse/variant_test"; + private static final String SPARKSQL_BLOB_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-blob-type-sql.commands"; + private static final String SPARKSQL_BLOB_TYPE_DF_COMMANDS = + Paths.DEMO_DIR + "/sparksql-blob-type-df.commands"; + private static final String SPARKSQL_VECTOR_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-vector-type-sql.commands"; + private static final String SPARKSQL_VARIANT_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-variant-type-sql.commands"; + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + waitForHdfs(); + sparkAdhoc1.executeShellCommand("/bin/bash " + Paths.DEMO_SETUP).expectToSucceed(); + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_SQL_TEST_SUCCESS"); + // The DF fixture writes blob_test_df with the INLINE branch of the BLOB struct + // (data field non-null, reference null). The SQL fixture exercises only the + // OUT_OF_LINE branch, so seeding both gives Trino read coverage of both shapes. + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_DF_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_DF_TEST_SUCCESS"); + sparkAdhoc1.executeSQLFile(SPARKSQL_VECTOR_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VECTOR_SQL_TEST_SUCCESS"); + sparkAdhoc1.executeSQLFile(SPARKSQL_VARIANT_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VARIANT_SQL_TEST_SUCCESS"); + trino.waitUntilReady(); + } + + @AfterAll + public void clean() throws Exception { + // JUnit runs @AfterAll even when the @BeforeAll assumption aborted setupOnce() + // before initializeServices(); nothing was seeded then, so nothing to clean. + if (sparkAdhoc1 == null) { + return; + } + // -f silently skips non-existent paths so cleanup is safe even when seeding + // failed partway and some of these tables were never created. + sparkAdhoc1.executeShellCommand("hdfs dfs -rm -R -f " + + BLOB_TEST_PATH + " " + BLOB_TEST_DF_PATH + " " + + VECTOR_TEST_PATH + " " + VARIANT_TEST_PATH).expectToSucceed(); + } + + // ---------- BLOB OUT_OF_LINE (blob_test) ---------- + + @Test + public void testTrinoCountBlob() throws Exception { + // Post-DELETE state of sparksql-blob-type-sql.commands is 2 rows (id=1 updated, + // id=2 merged, id=3 inserted then deleted) - parity with the Hive count assertion + // in ITTestCustomTypeHiveSync#testBlobTypeWithHiveSyncSQL. + trino.execute("SELECT count(*) FROM blob_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoProjectsBlobUpdatedRow() throws Exception { + // Full per-row shape for id=1 (post-UPDATE state): type discriminator + + // every reference subfield + the OUT_OF_LINE invariant that data IS NULL. + // One query, one substring assertion - catches column-order shifts, + // nested-struct field renames, and per-field decoding bugs. + trino.execute("SELECT blob_data.type, blob_data.data IS NULL, " + + "blob_data.reference.external_path, blob_data.reference.offset, " + + "blob_data.reference.length, blob_data.reference.managed " + + "FROM blob_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("OUT_OF_LINE,true,blobs/updated-1,10,100,true"); + } + + @Test + public void testTrinoProjectsBlobMergedRow() throws Exception { + // id=2 was MATCHED by the MERGE clause and rewritten to 'blobs/merged-2'. + // Same full-shape assertion as id=1 - confirms both UPDATE and MERGE write + // paths land at an identical on-disk OUT_OF_LINE shape. + trino.execute("SELECT blob_data.type, blob_data.data IS NULL, " + + "blob_data.reference.external_path, blob_data.reference.offset, " + + "blob_data.reference.length, blob_data.reference.managed " + + "FROM blob_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("OUT_OF_LINE,true,blobs/merged-2,20,200,true"); + } + + @Test + public void testTrinoBlobDeletedRowAbsent() throws Exception { + // id=3 was MERGE-inserted into dt=2024-01-02 then DELETEd. A DELETE that + // leaves the row visible (e.g. tombstone not honored on read) shows up as + // count = 1 here. Pairs with testTrinoCountBlob = 2 (total post-delete) + // to catch the case where DELETE silently no-ops. + trino.execute("SELECT count(*) FROM blob_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoBlobEmptiedPartitionInvisible() throws Exception { + // MERGE created dt=2024-01-02 (id=3), then DELETE emptied it. Grouping by the + // partition column must return exactly the data-bearing partition with its + // full row count. Not a partition-pruning assertion: GROUP BY only emits + // groups for partitions that still hold rows, and dt=2024-01-02 stays + // registered in the metastore either way (see ITTestCustomTypeHiveSync). + // What it does catch is a mis-decoded partition column, or a deleted row + // resurfacing under its old partition. + trino.execute("SELECT dt, count(*) FROM blob_test GROUP BY dt ORDER BY dt") + .expectToSucceed() + .assertStdOutContains("2024-01-01,2") + .assertStdOutContains("2024-01-02", 0); + } + + // ---------- BLOB INLINE (blob_test_df) ---------- + + @Test + public void testTrinoCountBlobInline() throws Exception { + // Post-DELETE state of sparksql-blob-type-df.commands is 2 rows (id=1 kept, + // id=2 updated to "updated payload", id=3 upserted then deleted). Parity + // with testTrinoCountBlob for the INLINE-shape table. + trino.execute("SELECT count(*) FROM blob_test_df") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoProjectsBlobInlineRow() throws Exception { + // INLINE shape for id=1 (never mutated): type=INLINE, data is the UTF-8 + // seed "hello world", and the INLINE invariant that reference IS NULL. + // from_utf8(data) is the right decoder - raw cast(varbinary as varchar) + // is rejected by Trino. + trino.execute("SELECT blob_data.type, from_utf8(blob_data.data), " + + "blob_data.reference IS NULL FROM blob_test_df WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("INLINE,hello world,true"); + } + + @Test + public void testTrinoProjectsBlobInlineUpdatedRow() throws Exception { + // id=2 was UPSERT-rewritten to "updated payload" in the DF fixture. Same + // full-shape assertion as id=1 - confirms the UPSERT write path for the + // INLINE branch round-trips end-to-end through Trino. + trino.execute("SELECT blob_data.type, from_utf8(blob_data.data), " + + "blob_data.reference IS NULL FROM blob_test_df WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("INLINE,updated payload,true"); + } + + @Test + public void testTrinoBlobInlineDeletedRowAbsent() throws Exception { + // id=3 was upserted into dt=2024-01-02 then DELETEd in the DF fixture. + // Parity with testTrinoBlobDeletedRowAbsent for the INLINE-shape table. + trino.execute("SELECT count(*) FROM blob_test_df WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + // ---------- VECTOR (vector_test) ---------- + + @Test + public void testTrinoCountVector() throws Exception { + // Post-DELETE state of sparksql-vector-type-sql.commands is 2 rows. + // Parity with testTrinoCountBlob / testTrinoCountVariant. + trino.execute("SELECT count(*) FROM vector_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoVectorDeletedRowAbsent() throws Exception { + // id=3 was MERGE-inserted then DELETEd in the vector fixture. Parity with + // the BLOB/VARIANT delete-absence checks. + trino.execute("SELECT count(*) FROM vector_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoVectorRoundTripsAsBinary() throws Exception { + // Per RFC-99, VECTOR(3) is stored on disk as fixed_len_byte_array(12) and Hive + // sync maps it to BINARY. The native plugin should expose the column as VARBINARY + // of the same 12 bytes (3 floats * 4 bytes). length() returning 12 confirms the + // round-trip; a return of 3 would mean the plugin decoded it as array, + // a real regression worth a separate ticket. Pairs with the Hive assertion at + // ITTestCustomTypeHiveSync:228-235. + trino.execute("SELECT length(embedding) FROM vector_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("12"); + } + + @Test + public void testTrinoVectorBytesDecodeToExpectedFloats() throws Exception { + // The 12 bytes of VECTOR(3) are 3 IEEE-754 floats in little-endian (Parquet's + // default). reverse(substr(..., n, 4)) flips each 4-byte chunk to big-endian + // so from_ieee754_32 returns the actual value. round(., 1) sidesteps float- + // precision noise in Trino's CSV rendering (0.9f decodes to ~0.90000004). + // For id=1's post-UPDATE state the fixture writes (0.9f, 0.8f, 0.7f). + trino.execute("SELECT round(from_ieee754_32(reverse(substr(embedding, 1, 4))), 1), " + + "round(from_ieee754_32(reverse(substr(embedding, 5, 4))), 1), " + + "round(from_ieee754_32(reverse(substr(embedding, 9, 4))), 1) " + + "FROM vector_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("0.9,0.8,0.7"); + } + + @Test + public void testTrinoVectorMergedRowDecodes() throws Exception { + // id=2 was MATCHED by the MERGE clause and rewritten to (0.41f, 0.51f, 0.61f). + // round(., 2) keeps it readable; complements id=1's UPDATE path to confirm + // both write paths land at the same on-disk layout. + trino.execute("SELECT round(from_ieee754_32(reverse(substr(embedding, 1, 4))), 2), " + + "round(from_ieee754_32(reverse(substr(embedding, 5, 4))), 2), " + + "round(from_ieee754_32(reverse(substr(embedding, 9, 4))), 2) " + + "FROM vector_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("0.41,0.51,0.61"); + } + + @Test + public void testTrinoVectorAllRowsAreFixedLength12() throws Exception { + // Invariant: every row's embedding is exactly 12 bytes. count(DISTINCT length(...)) + // = 1 AND min/max = 12 catches the case where some rows decode at a different + // width (e.g. an older row written before a layout fix). Stronger than the + // single-row length() check in testTrinoVectorRoundTripsAsBinary. + trino.execute("SELECT count(DISTINCT length(embedding)), min(length(embedding)), " + + "max(length(embedding)) FROM vector_test") + .expectToSucceed() + .assertStdOutContains("1,12,12"); + } + + // ---------- VARIANT (variant_test) ---------- + + @Test + public void testTrinoCountVariant() throws Exception { + // Post-DELETE state of sparksql-variant-type-sql.commands is 2 rows (id=1 + // updated, id=2 merged, id=3 inserted then deleted) - parity with the Hive + // count assertion in ITTestCustomTypeHiveSync#testVariantTypeWithHiveSyncSQL. + // count(*) doesn't deserialize the variant column so it's safe even if the + // plugin's variant decoding has gaps. + trino.execute("SELECT count(*) FROM variant_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoIntrospectsVariantColumn() throws Exception { + // Schema-level smoke: the variant_data column must surface in Trino's view of + // the table. Catches metastore-side regressions where Hive sync drops or + // mistypes the variant column entirely, separately from the + // value-projection question (which depends on how the plugin maps VARIANT). + trino.execute("DESCRIBE variant_test") + .expectToSucceed() + .assertStdOutContains("variant_data"); + } + + @Test + public void testTrinoProjectsVariantValue() throws Exception { + // The native trino-hudi plugin exposes VARIANT as ROW(metadata VARBINARY, + // value VARBINARY) with no top-level JSON/string decoding. But Spark's + // Variant binary format stores leaf string values as UTF-8 bytes inside + // the value component, so we can decode the bytes with from_utf8() and + // LIKE-match the seeded payload. Non-UTF8 framing bytes around the leaf + // become U+FFFD replacement chars, which the wildcard tolerates. This is + // real content round-trip: write {"key":"value1-updated"}, read the same + // payload back through the plugin. + trino.execute("SELECT from_utf8(variant_data.value) LIKE '%value1-updated%' " + + "FROM variant_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("true"); + } + + @Test + public void testTrinoProjectsVariantMergedRow() throws Exception { + // Same content-level round-trip for the MERGE-rewritten row. id=2's seed + // is {"key":"value2-merged"}; verifying that exact payload survives the + // MERGE write path -> hive sync -> Trino read end-to-end. + trino.execute("SELECT from_utf8(variant_data.value) LIKE '%value2-merged%' " + + "FROM variant_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("true"); + } + + @Test + public void testTrinoVariantValuesDifferAcrossRows() throws Exception { + // Invariant: id=1 ({"key":"value1-updated"}) and id=2 ({"key":"value2-merged"}) + // must produce different value bytes. Trino's count(DISTINCT ...) supports + // VARBINARY natively, so no base64 wrapping is needed. Catches the + // "projection returns same bytes for every row" regression that per-row + // content matches would silently miss. + trino.execute("SELECT count(DISTINCT variant_data.value) FROM variant_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoVariantDeletedRowAbsent() throws Exception { + // id=3 was MERGE-inserted into dt=2024-01-02 then DELETEd. Mirrors + // testTrinoBlobDeletedRowAbsent for the variant table - catches the case + // where DELETE silently no-ops on a Variant-bearing row. + trino.execute("SELECT count(*) FROM variant_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoVariantEmptiedPartitionInvisible() throws Exception { + // Same shape as the BLOB table: dt=2024-01-02 held only id=3, so after the + // DELETE the grouped read must return the data-bearing partition alone. As + // above this asserts per-partition row visibility, not partition pruning. + trino.execute("SELECT dt, count(*) FROM variant_test GROUP BY dt ORDER BY dt") + .expectToSucceed() + .assertStdOutContains("2024-01-01,2") + .assertStdOutContains("2024-01-02", 0); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java new file mode 100644 index 0000000000000..2c5b44268189a --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +/** + * Smoke coverage for the native trino-hudi connector running inside the integ2 + * testcontainers harness. Cheapest signal that the plugin loaded, the metastore + * is reachable, and the CLI can round-trip a query. + * + *

    Skipped unless the {@code trino} compose profile is active (see + * {@link #assumeTrinoProfile()}), since the coordinator container only starts then. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoSmoke extends ITTestBaseTestcontainers { + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + trino.waitUntilReady(); + } + + @Test + public void testShowCatalogsListsHudi() throws Exception { + // Asserts the native plugin registered. With connector.name=hudi (per + // HudiConnectorFactory#getName) the catalog appears under that name; if the + // plugin failed to load the catalog file would have made Trino fail to start + // and we will never reach this assertion. + trino.execute("system", "runtime", "SHOW CATALOGS") + .expectToSucceed() + .assertStdOutContains("hudi"); + } + + @Test + public void testShowSchemasFromHudiReachesMetastore() throws Exception { + // The `default` schema is created by Hive at first contact with the metastore. + // Asserting it appears here proves the connector can talk to thrift://hivemetastore:9083. + trino.execute("hudi", "default", "SHOW SCHEMAS") + .expectToSucceed() + .assertStdOutContains("default"); + } + + @Test + public void testSelectOneRoundtrip() throws Exception { + trino.execute("system", "runtime", "SELECT 1") + .expectToSucceed() + .assertStdOutContains("1"); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java new file mode 100644 index 0000000000000..43dfdd18134ea --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Paths; + +/** + * End-to-end coverage that the native trino-hudi connector can read both COW and + * MOR tables that came from Spark + Hive sync. Mirrors the retired + * {@code docker/demo/trino-batch1.commands} demo flow (removed together with the + * rest of the legacy trino-coordinator path) but uses a self-contained spark-sql + * fixture (see {@code sparksql-stock-ticks-trino.commands}) instead of the full + * Kafka/streaming pipeline, which integ2 doesn't otherwise exercise. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoStockTicks extends ITTestBaseTestcontainers { + + private static final String STOCK_TICKS_COW_PATH = "/user/hive/warehouse/stock_ticks_cow"; + private static final String STOCK_TICKS_MOR_PATH = "/user/hive/warehouse/stock_ticks_mor"; + private static final String SPARKSQL_STOCK_TICKS_COMMANDS = + Paths.DEMO_DIR + "/sparksql-stock-ticks-trino.commands"; + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + waitForHdfs(); + sparkAdhoc1.executeShellCommand("/bin/bash " + Paths.DEMO_SETUP).expectToSucceed(); + sparkAdhoc1.executeSQLFile(SPARKSQL_STOCK_TICKS_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("STOCK_TICKS_COW_SETUP_SUCCESS") + .assertStdOutContainsLine("STOCK_TICKS_MOR_SETUP_SUCCESS") + .assertStdOutContainsLine("STOCK_TICKS_TRINO_SETUP_SUCCESS"); + trino.waitUntilReady(); + } + + @AfterAll + public void clean() throws Exception { + // JUnit runs @AfterAll even when the @BeforeAll assumption aborted setupOnce() + // before initializeServices(); nothing was seeded then, so nothing to clean. + if (sparkAdhoc1 == null) { + return; + } + sparkAdhoc1.executeShellCommand("hdfs dfs -rm -R -f " + + STOCK_TICKS_COW_PATH + " " + STOCK_TICKS_MOR_PATH).expectToSucceed(); + } + + // ---------- Queries reproduced from the retired docker/demo/trino-batch1.commands ---------- + + @Test + public void testTrinoReadsCowMaxTs() throws Exception { + // Original: select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOOG' + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_cow GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:29:00"); + } + + @Test + public void testTrinoReadsMorRoMaxTs() throws Exception { + // Hive sync produces stock_ticks_mor_ro (RO view of base files). The fixture's + // UPDATE (ts 10:59:00) lives only in a log file, so _ro must keep serving the + // 10:29:00 base row - a 10:59:00 here means log records leaked into the RO view. + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_mor_ro GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:29:00") + .assertStdOutContains("2018-08-31 10:59:00", 0); + } + + @Test + public void testTrinoReadsCowProjectedColumns() throws Exception { + // open == close == 1230.50 in the seed row, so "1230.5" appears twice in CSV output. + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_cow WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5"); + } + + @Test + public void testTrinoReadsMorRoProjectedColumns() throws Exception { + // Same symbol-count pin as the _rt case below: the base row assert on its own + // would still pass with the log row returned next to it. + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_mor_ro WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5") + .assertStdOutContains("GOOG", 1); + } + + @Test + public void testTrinoReadsMorRtMergedMaxTs() throws Exception { + // The fixture's UPDATE lands as a log-only delta; the _rt view must merge it + // on read. Pairs with testTrinoReadsMorRoMaxTs pinning _ro to the base row, + // so together they prove the connector takes different read paths for the + // two views instead of serving base files for both. + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_mor_rt GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:59:00"); + } + + @Test + public void testTrinoReadsMorRtMergedProjectedColumns() throws Exception { + // Full merged row: every non-key column must come from the log record. The row + // assert alone would still pass if the base row came back alongside it, so pin + // the symbol count too - exactly one GOOG row may survive the merge. + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_mor_rt WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:59:00,9021,1227.25,1227.5") + .assertStdOutContains("GOOG", 1); + } +} diff --git a/hudi-trino/README.md b/hudi-trino/README.md index c163121284dce..72c848ba2c180 100644 --- a/hudi-trino/README.md +++ b/hudi-trino/README.md @@ -45,6 +45,48 @@ mvn -Phudi-trino,hudi-trino-tests -pl hudi-trino test CI follows the same two steps: `.github/workflows/hudi_trino_ci.yml` installs the test-jars from a source checkout of the pinned Trino tag, then runs with both profiles enabled. +## End-to-end tests (docker) + +The testcontainers E2E suite (`hudi-integ-test`, classes `ITTestTrino*` under +`org.apache.hudi.integ2.testcontainers.trino`) runs Trino queries against a real +HDFS + Hive metastore + Spark stack. The Trino container image bakes in a plugin +directory assembled by the in-repo shim at `docker/trino/shim/` -- a standalone Maven +project mirroring the upstream `trinodb/trino` `plugin/trino-hudi` shim planned by +RFC-105 (not yet released upstream). CI runs the same flow via +`.github/workflows/hudi_trino_e2e.yml`. + +Local flow: + +``` +# 1. JDK 17: full reactor incl. the integ-test bundles the containers mount +mvn clean install -T 2 -Dscala-2.13 -Dscala.binary.version=2.13 -Dspark4.0 -Dflink1.20 \ + -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true + +# 2. JDK 25: the connector +mvn -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + +# 3. JDK 25: assemble the plugin dir (package, NOT install -- installing would +# shadow the real io.trino:trino-hudi release coordinates in the local m2). +# dep.hudi.version comes from the reactor pom: the shim sits outside the +# reactor, so cut_release_branch.sh cannot bump its literal default. +HUDI_VERSION=$(mvn -q -ntp help:evaluate -Dexpression=project.version -DforceStdout) +mvn -f docker/trino/shim/pom.xml clean package -DskipTests -Ddep.hudi.version="$HUDI_VERSION" + +# 4. Build the Trino image (locally tagged; never published) +docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 + +# 5. JDK 17: run the suite (only the spark402 compose pair has the trino service) +mvn verify -pl hudi-integ-test -Dscala-2.13 -Dscala.binary.version=2.13 -Dspark4.0 \ + -Pintegration-tests -DskipITs=false -Ddocker.compose.skip=true \ + -Dit.test='ITTestTrino*' -Dcompose.profiles=trino \ + -Dspark.docker.compose.prefix=docker-compose_hadoop340_hive2310_spark402 +``` + +Fast iteration loop: after changing connector code, redo steps 2-3, then add +`-Dtrino.plugin.dir=$PWD/docker/trino/shim/target/trino-hudi-481` to step 5. The +container's overlay entrypoint swaps the freshly built plugin dir in at start, so the +image rebuild (step 4) is skipped. + ## IDE setup Only this module needs JDK 25. Leave the rest of Hudi on its native JDK (11 or 17) so you are not toggling the project default. From aaec9112496d65cdb611110ca453e90f812ac0f9 Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Thu, 6 Aug 2026 10:36:01 +0800 Subject: [PATCH 241/255] fix(flink): close lookup reader after cache reload attempts (#19503) HoodieLookupTableReader was left open when a cache reload attempt failed, leaking the underlying input format on every failed reload. Close the reader after each attempt, successful or not, and route the failure path through CloseableUtils.closeSuppressing so a close error does not mask the original exception. (cherry picked from commit a44572bb4e45689bc1fa0ab9e0c4daca17caaf6d) Cherry-pick adaptation: - TestHoodieLookupFunction: the conflict rendered master's whole region, which includes testRocksDBCacheLifecycleAndLookupFailure. That test is not on this branch and #19503 does not add it, so only the commit's own changes were applied: the new testReaderIsClosedWhenCacheReloadFails, the FailingLookupTableReader helper, and the newLookupFunction parameter retyped from CountingLookupTableReader to HoodieLookupTableReader. - Added the assertThrows static import, which the new test needs and this branch's copy of the file did not already have. Both production files and the new TestHoodieLookupTableReader apply byte-identically to the upstream delta. --- .../table/lookup/HoodieLookupFunction.java | 17 +-- .../table/lookup/HoodieLookupTableReader.java | 36 ++++-- .../lookup/TestHoodieLookupFunction.java | 45 ++++++- .../lookup/TestHoodieLookupTableReader.java | 110 ++++++++++++++++++ 4 files changed, 190 insertions(+), 18 deletions(-) create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupTableReader.java diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java index 10c47b2fe1c97..7b02f9da826b6 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java @@ -155,15 +155,16 @@ private void checkCacheReload() throws IOException { try { long count = 0; GenericRowData reuse = new GenericRowData(rowType.getFieldCount()); - partitionReader.open(); - RowData row; - while ((row = partitionReader.read(reuse)) != null) { - count++; - RowData rowData = serializer.copy(row); - RowData key = extractLookupKey(rowData); - cache.addRow(key, rowData); + try (HoodieLookupTableReader reader = partitionReader) { + reader.open(); + RowData row; + while ((row = reader.read(reuse)) != null) { + count++; + RowData rowData = serializer.copy(row); + RowData key = extractLookupKey(rowData); + cache.addRow(key, rowData); + } } - partitionReader.close(); currentCommit = latestCommitInstant.get(); scheduleNextLoad(); log.info("Loaded {} row(s) into lookup join cache", count); diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupTableReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupTableReader.java index 31fdf6d85d2ae..c2c3fbf0472fd 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupTableReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupTableReader.java @@ -28,16 +28,19 @@ import javax.annotation.Nullable; +import java.io.Closeable; import java.io.IOException; import java.io.Serializable; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import static org.apache.hudi.common.util.CloseableUtils.closeSuppressing; + /** * Hudi look up table reader. */ -public class HoodieLookupTableReader implements Serializable { +public class HoodieLookupTableReader implements Serializable, Closeable { private static final long serialVersionUID = 1L; private final SerializableSupplier> inputFormatSupplier; @@ -53,11 +56,17 @@ public HoodieLookupTableReader(SerializableSupplier> inp } public void open() throws IOException { + close(); this.inputFormat = inputFormatSupplier.get(); - inputFormat.configure(conf); - this.inputSplits = Arrays.stream(inputFormat.createInputSplits(1)).collect(Collectors.toList()); - ((RichInputFormat) inputFormat).openInputFormat(); - inputFormat.open(inputSplits.remove(0)); + try { + inputFormat.configure(conf); + this.inputSplits = Arrays.stream(inputFormat.createInputSplits(1)).collect(Collectors.toList()); + ((RichInputFormat) inputFormat).openInputFormat(); + inputFormat.open(inputSplits.remove(0)); + } catch (IOException | RuntimeException e) { + closeSuppressing(this, e); + throw e; + } } @Nullable @@ -77,12 +86,21 @@ public RowData read(RowData reuse) throws IOException { return null; } + @Override public void close() throws IOException { - if (this.inputFormat != null) { - inputFormat.close(); + InputFormat format = this.inputFormat; + this.inputFormat = null; + this.inputSplits = null; + if (format == null) { + return; } - if (inputFormat instanceof RichInputFormat) { - ((RichInputFormat) inputFormat).closeInputFormat(); + + if (format instanceof RichInputFormat) { + try (Closeable ignored = ((RichInputFormat) format)::closeInputFormat) { + format.close(); + } + } else { + format.close(); } } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java index 8d336ede565e1..6787713449f68 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java @@ -40,6 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -98,7 +99,26 @@ void testLookupCacheDoesNotReloadWhenCompletedCommitHasNotChanged() throws Excep } } - private HoodieLookupFunction newLookupFunction(CountingLookupTableReader reader, Configuration conf) { + @Test + void testReaderIsClosedWhenCacheReloadFails() throws Exception { + Configuration conf = getConf(); + TestData.writeData(TestData.DATA_SET_SINGLE_INSERT, conf); + + FailingLookupTableReader reader = new FailingLookupTableReader(conf); + HoodieLookupFunction function = newLookupFunction(reader, conf); + function.open(null); + + Thread.currentThread().interrupt(); + try { + assertThrows(RuntimeException.class, () -> function.lookup(lookupKey())); + assertEquals(1, reader.closeCount, "The failed reload attempt should close the reader"); + } finally { + Thread.interrupted(); + function.close(); + } + } + + private HoodieLookupFunction newLookupFunction(HoodieLookupTableReader reader, Configuration conf) { return new HoodieLookupFunction( reader, TestConfigurations.ROW_TYPE, @@ -158,4 +178,27 @@ public void close() throws IOException { // no-op } } + + private static class FailingLookupTableReader extends HoodieLookupTableReader { + private int closeCount; + + private FailingLookupTableReader(Configuration conf) { + super(() -> null, conf); + } + + @Override + public void open() { + // no-op + } + + @Override + public RowData read(RowData reuse) throws IOException { + throw new IOException("expected"); + } + + @Override + public void close() { + closeCount++; + } + } } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupTableReader.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupTableReader.java new file mode 100644 index 0000000000000..35c650d34d0ab --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupTableReader.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.lookup; + +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.flink.api.common.io.RichInputFormat; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.io.InputSplit; +import org.apache.flink.table.data.RowData; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link HoodieLookupTableReader}. + */ +class TestHoodieLookupTableReader { + + @Test + @SuppressWarnings("unchecked") + void testOpenRollsBackPartiallyOpenedInputFormat() throws Exception { + RichInputFormat inputFormat = mock(RichInputFormat.class); + InputSplit inputSplit = mock(InputSplit.class); + when(inputFormat.createInputSplits(1)).thenReturn(new InputSplit[] {inputSplit}); + IOException openException = new IOException("expected open failure"); + doThrow(openException).when(inputFormat).open(inputSplit); + + HoodieLookupTableReader reader = + new HoodieLookupTableReader(() -> inputFormat, new Configuration()); + + assertSame(openException, assertThrows(IOException.class, reader::open)); + verify(inputFormat).close(); + verify(inputFormat).closeInputFormat(); + + reader.close(); + verify(inputFormat, times(1)).close(); + verify(inputFormat, times(1)).closeInputFormat(); + } + + @Test + @SuppressWarnings("unchecked") + void testOpenPreservesFailureWhenRuntimeRollbackFails() throws Exception { + RichInputFormat inputFormat = mock(RichInputFormat.class); + InputSplit inputSplit = mock(InputSplit.class); + when(inputFormat.createInputSplits(1)).thenReturn(new InputSplit[] {inputSplit}); + IOException openException = new IOException("expected open failure"); + HoodieIOException splitCloseException = + new HoodieIOException("expected runtime split close failure"); + doThrow(openException).when(inputFormat).open(inputSplit); + doThrow(splitCloseException).when(inputFormat).close(); + + HoodieLookupTableReader reader = + new HoodieLookupTableReader(() -> inputFormat, new Configuration()); + + IOException exception = assertThrows(IOException.class, reader::open); + assertSame(openException, exception); + assertEquals(1, exception.getSuppressed().length); + assertSame(splitCloseException, exception.getSuppressed()[0]); + verify(inputFormat).closeInputFormat(); + } + + @Test + @SuppressWarnings("unchecked") + void testCloseReleasesInputFormatWhenRuntimeSplitCloseFails() throws Exception { + RichInputFormat inputFormat = mock(RichInputFormat.class); + InputSplit inputSplit = mock(InputSplit.class); + when(inputFormat.createInputSplits(1)).thenReturn(new InputSplit[] {inputSplit}); + HoodieIOException splitCloseException = + new HoodieIOException("expected runtime split close failure"); + IOException formatCloseException = new IOException("expected format close failure"); + doThrow(splitCloseException).when(inputFormat).close(); + doThrow(formatCloseException).when(inputFormat).closeInputFormat(); + + HoodieLookupTableReader reader = + new HoodieLookupTableReader(() -> inputFormat, new Configuration()); + reader.open(); + + HoodieIOException exception = assertThrows(HoodieIOException.class, reader::close); + assertSame(splitCloseException, exception); + assertEquals(1, exception.getSuppressed().length); + assertSame(formatCloseException, exception.getSuppressed()[0]); + verify(inputFormat).closeInputFormat(); + } +} From a7f7aaf212e63a1ebdf50c33c81d10176f7affa1 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 5 Aug 2026 20:32:58 -0700 Subject: [PATCH 242/255] fix(spark): preserve the Avro fixed-size decimal width in the Spark row write support (#19512) * fix(spark): preserve the Avro fixed-size decimal width in the Spark row write support The Spark row writer sized decimal FIXED_LEN_BYTE_ARRAY columns from Decimal.minBytesForPrecision(), discarding an Avro fixed(N) size wider than the precision-minimal width and diverging from the Avro write path. Honor the declared fixed size from the resolved HoodieSchema at both the schema converter and the value writer, resolving the padding buffer once per column. * Guard against an Avro fixed size smaller than the precision-minimal decimal width Fail fast with a diagnosable message if a decimal fixed(N) declares fewer bytes than minBytesForPrecision, instead of an opaque negative-length Arrays.fill later. * Unit-test decimalFixedLen directly for hudi-spark-client coverage Make decimalFixedLen package-private and cover its branches (min-width fallback and honored Avro fixed size) from hudi-spark-client's own test module, where the full write support cannot be constructed. * Extract decimal sign-extension padding into a testable static helper Move the fixed-length padding out of the makeWriter lambda into padDecimalToFixedLength so it can be unit-tested directly from hudi-spark-client (covering the exact-width, positive-pad, and negative sign-extension cases), where the full write support cannot be constructed. * Trim redundant Javadocs on the decimal helpers Drop the padDecimalToFixedLength Javadoc and reduce decimalFixedLen's to a single line. * Rename decimalFixedLen to resolveDecimalByteLength and use assertFalse for the clustering check Address review nits: resolveDecimalByteLength reads more clearly at the call sites than decimalFixedLen, and assertFalse(isEmpty()) is more natural than assertTrue(!isEmpty()). (cherry picked from commit 9a30ecd60e1572fca61a32af00ea7bc27deef611) --- .../row/HoodieRowParquetWriteSupport.java | 63 ++++--- .../row/TestHoodieRowParquetWriteSupport.java | 35 ++++ .../TestHoodieInternalRowParquetWriter.java | 36 ++++ ...HoodieSparkMergeOnReadTableCompaction.java | 155 ++++++++++++++++++ 4 files changed, 268 insertions(+), 21 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java index 262d4acc8ac4d..b8a2ebb360c01 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java @@ -428,6 +428,34 @@ private void writeFields(InternalRow row, StructType schema, ValueWriter[] field } } + /** + * Fixed-length byte width for a decimal column: honor the declared Avro {@code fixed} size when + * present (it may be wider than the precision-minimal width), otherwise the precision-minimal width. + */ + static int resolveDecimalByteLength(HoodieSchema resolvedSchema, int precision) { + if (resolvedSchema instanceof HoodieSchema.Decimal) { + HoodieSchema.Decimal decimalSchema = (HoodieSchema.Decimal) resolvedSchema; + if (decimalSchema.isFixed()) { + int fixedSize = decimalSchema.getFixedSize(); + ValidationUtils.checkArgument(fixedSize >= Decimal.minBytesForPrecision()[precision], + () -> String.format("Avro fixed size %s is too small for decimal precision %s (need >= %s bytes)", + fixedSize, precision, Decimal.minBytesForPrecision()[precision])); + return fixedSize; + } + } + return Decimal.minBytesForPrecision()[precision]; + } + + static byte[] padDecimalToFixedLength(byte[] unscaledBytes, int numBytes, byte[] paddingBuffer) { + if (unscaledBytes.length == numBytes) { + return unscaledBytes; + } + byte signByte = (unscaledBytes[0] < 0) ? (byte) -1 : (byte) 0; + Arrays.fill(paddingBuffer, 0, numBytes - unscaledBytes.length, signByte); + System.arraycopy(unscaledBytes, 0, paddingBuffer, numBytes - unscaledBytes.length, unscaledBytes.length); + return paddingBuffer; + } + private ValueWriter makeWriter(HoodieSchema schema, DataType dataType) { HoodieSchema resolvedSchema = schema == null ? null : schema.getNonNullType(); @@ -496,28 +524,21 @@ private ValueWriter makeWriter(HoodieSchema schema, DataType dataType) { consumeGroup(() -> variantWriter.accept(row, ordinal)); }; } else if (dataType instanceof DecimalType) { + int precision = ((DecimalType) dataType).precision(); + ValidationUtils.checkArgument(precision <= DecimalType.MAX_PRECISION(), + () -> String.format("Decimal precision %s exceeds max precision %s", precision, DecimalType.MAX_PRECISION())); + int scale = ((DecimalType) dataType).scale(); + // Honor the declared Avro `fixed` size so the row/bulk-insert/clustering/compaction write + // path preserves the schema width instead of narrowing to the precision-minimal one. + int numBytes = resolveDecimalByteLength(resolvedSchema, precision); + // Padding buffer resolved once per column, not per record: reuse the shared decimalBuffer when + // it is wide enough, otherwise allocate one dedicated buffer here (an over-allocated Avro fixed + // size can exceed the shared buffer's capacity). + byte[] paddingBuffer = numBytes <= decimalBuffer.length ? decimalBuffer : new byte[numBytes]; return (row, ordinal) -> { - int precision = ((DecimalType) dataType).precision(); - ValidationUtils.checkArgument(precision <= DecimalType.MAX_PRECISION(), - () -> String.format("Decimal precision %s exceeds max precision %s", precision, DecimalType.MAX_PRECISION())); - int scale = ((DecimalType) dataType).scale(); byte[] bytes = row.getDecimal(ordinal, precision, scale).toJavaBigDecimal().unscaledValue().toByteArray(); - int numBytes = Decimal.minBytesForPrecision()[precision]; - byte[] fixedLengthBytes; - if (bytes.length == numBytes) { - // If the length of the underlying byte array of the unscaled `BigInteger` happens to be - // `numBytes`, just reuse it, so that we don't bother copying it to `decimalBuffer`. - fixedLengthBytes = bytes; - } else { - // Otherwise, the length must be less than `numBytes`. In this case we copy contents of - // the underlying bytes with padding sign bytes to `decimalBuffer` to form the result - // fixed-length byte array. - byte signByte = (bytes[0] < 0) ? (byte) -1 : (byte) 0; - Arrays.fill(decimalBuffer, 0, numBytes - bytes.length, signByte); - System.arraycopy(bytes, 0, decimalBuffer, numBytes - bytes.length, bytes.length); - fixedLengthBytes = decimalBuffer; - } - recordConsumer.addBinary(Binary.fromReusedByteArray(fixedLengthBytes, 0, numBytes)); + recordConsumer.addBinary(Binary.fromReusedByteArray( + padDecimalToFixedLength(bytes, numBytes, paddingBuffer), 0, numBytes)); }; } else if (dataType instanceof ArrayType && resolvedSchema != null @@ -776,7 +797,7 @@ private Type convertField(HoodieSchema fieldSchema, StructField structField, Typ return Types .primitive(FIXED_LEN_BYTE_ARRAY, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) - .length(Decimal.minBytesForPrecision()[precision]) + .length(resolveDecimalByteLength(resolvedSchema, precision)) .named(structField.name()); } else if (dataType instanceof ArrayType && resolvedSchema != null diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowParquetWriteSupport.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowParquetWriteSupport.java index 0c39e99bdf89a..6ec4a4cd2300f 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowParquetWriteSupport.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowParquetWriteSupport.java @@ -18,16 +18,21 @@ package org.apache.hudi.io.storage.row; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.spark.sql.types.Decimal; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; import java.util.TimeZone; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; /** * Coverage for {@link HoodieRowParquetWriteSupport#resolveSessionLocalTimeZone()}. @@ -42,6 +47,36 @@ class TestHoodieRowParquetWriteSupport extends HoodieClientTestBase { private static final String SESSION_LOCAL_TIME_ZONE_KEY = "spark.sql.session.timeZone"; + @Test + void testResolveDecimalByteLength() { + int minWidth = Decimal.minBytesForPrecision()[20]; + // A non-decimal schema falls back to the precision-minimal width. + assertEquals(minWidth, + HoodieRowParquetWriteSupport.resolveDecimalByteLength(HoodieSchema.create(HoodieSchemaType.STRING), 20)); + // A bytes-backed decimal (no declared fixed size) also falls back to the minimum. + assertEquals(minWidth, + HoodieRowParquetWriteSupport.resolveDecimalByteLength(HoodieSchema.createDecimal(20, 2), 20)); + // An Avro fixed decimal wider than the minimum is honored. + assertEquals(10, + HoodieRowParquetWriteSupport.resolveDecimalByteLength( + HoodieSchema.createDecimal("dec", null, null, 20, 2, 10), 20)); + } + + @Test + void testPadDecimalToFixedLength() { + byte[] buffer = new byte[16]; + // Already the full width: returned as-is, no copy into the buffer. + byte[] exact = new byte[] {1, 2, 3, 4}; + assertSame(exact, HoodieRowParquetWriteSupport.padDecimalToFixedLength(exact, 4, buffer)); + // Positive magnitude: left-padded with zero sign bytes. + byte[] positive = HoodieRowParquetWriteSupport.padDecimalToFixedLength(new byte[] {0x12, 0x34}, 4, buffer); + assertArrayEquals(new byte[] {0, 0, 0x12, 0x34}, Arrays.copyOf(positive, 4)); + // Negative magnitude: left-padded with 0xFF sign bytes. + byte[] negative = HoodieRowParquetWriteSupport.padDecimalToFixedLength( + new byte[] {(byte) 0xFF, (byte) 0x80}, 4, buffer); + assertArrayEquals(new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0x80}, Arrays.copyOf(negative, 4)); + } + @Test void testResolveSessionLocalTimeZoneWithoutOverride() { String expected = TimeZone.getDefault().getID(); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java index b2c9d24b1b961..1e40b8ea8d053 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java @@ -36,12 +36,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -126,6 +130,38 @@ public void testProperWriting(boolean parquetWriteLegacyFormatEnabled) throws Ex }); } + @Test + void testDecimalFixedLenWidthFromAvroSchema() { + // The row-writer sizes decimal FIXED_LEN columns from the Avro schema: an Avro fixed(10) for + // decimal(20,2) keeps its declared width (10, wider than the precision-minimal 9), while a bytes + // decimal keeps the precision-minimal width (9). + assertEquals(10, decimalParquetTypeLength(decimalRecordSchema( + "{\"type\":\"fixed\",\"name\":\"dec_fixed\",\"size\":10,\"logicalType\":\"decimal\",\"precision\":20,\"scale\":2}")), + "Avro fixed(10) decimal must stay FIXED_LEN_BYTE_ARRAY(10)"); + assertEquals(9, decimalParquetTypeLength(decimalRecordSchema( + "{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":20,\"scale\":2}")), + "bytes decimal keeps the precision-minimal FIXED_LEN width (9)"); + } + + private static String decimalRecordSchema(String decType) { + return "{\"type\":\"record\",\"name\":\"rec\",\"fields\":[{\"name\":\"dec\",\"type\":" + decType + "}]}"; + } + + private int decimalParquetTypeLength(String avroSchemaJson) { + StructType structType = new StructType().add("dec", DataTypes.createDecimalType(20, 2), false); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withSchema(avroSchemaJson) + .build(); + HoodieRowParquetWriteSupport writeSupport = HoodieRowParquetWriteSupport.getHoodieRowParquetWriteSupport( + storageConf.unwrap(), structType, Option.empty(), config); + MessageType parquetSchema = writeSupport.init(writeSupport.getHadoopConf()).getSchema(); + PrimitiveType dec = parquetSchema.getType("dec").asPrimitiveType(); + assertEquals(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, dec.getPrimitiveTypeName(), + "decimal must be encoded as FIXED_LEN_BYTE_ARRAY"); + return dec.getTypeLength(); + } + private HoodieRowParquetWriteSupport getWriteSupport(HoodieWriteConfig.Builder writeConfigBuilder, Configuration hadoopConf, boolean parquetWriteLegacyFormatEnabled) { writeConfigBuilder.withStorageConfig(HoodieStorageConfig.newBuilder().parquetWriteLegacyFormat(String.valueOf(parquetWriteLegacyFormatEnabled)).build()); HoodieWriteConfig writeConfig = writeConfigBuilder.build(); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java index 960b98eb73e2e..2825ff5396801 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java @@ -29,19 +29,27 @@ import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; import org.apache.hudi.common.model.PartialUpdateAvroPayload; import org.apache.hudi.common.model.WriteConcurrencyMode; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.config.HoodieCleanConfig; +import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieLayoutConfig; @@ -52,6 +60,8 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import org.apache.hudi.table.HoodieSparkTable; +import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.commit.SparkBucketIndexPartitioner; import org.apache.hudi.table.action.rollback.RollbackUtils; @@ -59,17 +69,28 @@ import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.math.BigDecimal; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -81,6 +102,7 @@ import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -442,4 +464,137 @@ private void commitToTable(String instant, List writeStatuses) { client.commitStats(instant, writeStats, Option.empty(), metaClient.getCommitActionType()); assertTrue(committed); } + + // Avro fixed(10) decimal(20,2). 10 is wider than the precision-minimal width (9 for precision + // 20), so Spark's own DecimalType->Avro conversion would emit fixed(9); the declared 10 only + // survives if the write path honors the Avro fixed size. Do not derive this schema from a + // DataFrame, which would drop the fixed size. + private static final String FIXED10_DECIMAL_SCHEMA = + "{\"type\":\"record\",\"name\":\"decimalRec\",\"fields\":[" + + "{\"name\":\"_row_key\",\"type\":\"string\"}," + + "{\"name\":\"partition_path\",\"type\":\"string\"}," + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"dec\",\"type\":{\"type\":\"fixed\",\"name\":\"decFixed\",\"size\":10," + + "\"logicalType\":\"decimal\",\"precision\":20,\"scale\":2}}]}"; + + private static final String DECIMAL_PARTITION = "p1"; + private static final int EXPECTED_DECIMAL_FIXED_LEN = 10; + + @Test + void testDecimalFixedWidthPreservedAfterCompactionAndClustering() throws Exception { + Properties props = getPropertiesForKeyGen(true); + Properties rowWriterProps = new Properties(); + rowWriterProps.put("hoodie.datasource.write.row.writer.enable", "true"); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .forTable("test-decimal-fixed") + .withPath(basePath()) + .withSchema(FIXED10_DECIMAL_SCHEMA) + .withParallelism(2, 2) + .withPreCombineField("ts") + .withProperties(rowWriterProps) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withMaxNumDeltaCommitsBeforeCompaction(1) + .compactionSmallFileSize(0) + .withInlineCompaction(false) + .build()) + .withClusteringConfig(HoodieClusteringConfig.newBuilder() + .withClusteringMaxNumGroups(10) + .withClusteringTargetPartitions(0) + .withInlineClustering(false) + .withInlineClusteringNumCommits(1) + .build()) + .build(); + props.putAll(config.getProps()); + + metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, props); + client = getHoodieWriteClient(config); + + // two insert commits (small-file size 0 forces a fresh file group each) create two base-file + // groups, both written via the Avro path at fixed(10) + String instant1 = WriteClientTestUtils.createNewInstantTime(); + writeData(instant1, buildDecimalRecords(0, 10, 1L, new BigDecimal("123456789.12")), true); + String instant2 = WriteClientTestUtils.createNewInstantTime(); + writeData(instant2, buildDecimalRecords(10, 10, 1L, new BigDecimal("223456789.34")), true); + // update every key so both groups accumulate log files for compaction to merge + String instant3 = WriteClientTestUtils.createNewInstantTime(); + writeData(instant3, buildDecimalRecords(0, 20, 2L, new BigDecimal("323456789.56")), true); + + // precondition: both file groups must carry log files, else compaction is a no-op and would not + // exercise the row-writer merge path + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieTable hoodieTable = HoodieSparkTable.create(config, context(), metaClient); + hoodieTable.getHoodieView().sync(); + List latestSlices = + hoodieTable.getHoodieView().getLatestFileSlices(DECIMAL_PARTITION).collect(Collectors.toList()); + assertEquals(2, latestSlices.size(), "expected two file groups before compaction"); + assertTrue(latestSlices.stream().allMatch(slice -> slice.getLogFiles().findAny().isPresent()), + "each file group must have log files for compaction to merge"); + HoodieSchema tableSchema = new TableSchemaResolver(metaClient).getTableSchema(false); + + // compaction rewrites both base files through the Spark record type + String compactionInstant = (String) client.scheduleCompaction(Option.empty()).get(); + HoodieWriteMetadata compactionResult = client.compact(compactionInstant); + client.commitCompaction(compactionInstant, compactionResult, Option.empty()); + assertTrue(metaClient.reloadActiveTimeline().filterCompletedInstants().containsInstant(compactionInstant)); + List compactedBaseFiles = latestBaseFilePaths(config, DECIMAL_PARTITION); + assertEquals(2, compactedBaseFiles.size(), "expected two compacted base files"); + for (StoragePath path : compactedBaseFiles) { + assertDecimalFixedLen(path, EXPECTED_DECIMAL_FIXED_LEN); + } + assertEquals(tableSchema, + new TableSchemaResolver(HoodieTableMetaClient.reload(metaClient)).getTableSchema(false), + "table schema in commit metadata must not change after compaction"); + + // clustering rewrites the two compacted groups through the same Spark row-writer path + String clusteringInstant = (String) client.scheduleClustering(Option.empty()).get(); + HoodieWriteMetadata> clusterMetadata = client.cluster(clusteringInstant, true); + List clusterStats = clusterMetadata.getWriteStats().get(); + assertFalse(clusterStats.isEmpty(), "clustering should write at least one base file"); + for (HoodieWriteStat stat : clusterStats) { + assertDecimalFixedLen(new StoragePath(metaClient.getBasePath(), stat.getPath()), + EXPECTED_DECIMAL_FIXED_LEN); + } + assertEquals(tableSchema, + new TableSchemaResolver(HoodieTableMetaClient.reload(metaClient)).getTableSchema(false), + "table schema in commit metadata must not change after clustering"); + } + + private List buildDecimalRecords(int startKey, int count, long ts, BigDecimal decValue) { + Schema schema = new Schema.Parser().parse(FIXED10_DECIMAL_SCHEMA); + Schema decSchema = schema.getField("dec").schema(); + Conversions.DecimalConversion decimalConversion = new Conversions.DecimalConversion(); + LogicalTypes.Decimal decimalType = LogicalTypes.decimal(20, 2); + BigDecimal scaledValue = decValue.setScale(2); + List records = new ArrayList<>(); + for (int i = 0; i < count; i++) { + String key = "key_" + (startKey + i); + GenericRecord rec = new GenericData.Record(schema); + rec.put("_row_key", key); + rec.put("partition_path", DECIMAL_PARTITION); + rec.put("ts", ts); + GenericFixed fixed = decimalConversion.toFixed(scaledValue, decSchema, decimalType); + rec.put("dec", fixed); + records.add(new HoodieAvroRecord<>(new HoodieKey(key, DECIMAL_PARTITION), + new OverwriteWithLatestAvroPayload(rec, ts))); + } + return records; + } + + private List latestBaseFilePaths(HoodieWriteConfig config, String partition) { + metaClient = HoodieTableMetaClient.reload(metaClient); + HoodieTable hoodieTable = HoodieSparkTable.create(config, context(), metaClient); + hoodieTable.getHoodieView().sync(); + return hoodieTable.getBaseFileOnlyView().getLatestBaseFiles(partition) + .map(HoodieBaseFile::getStoragePath).collect(Collectors.toList()); + } + + private void assertDecimalFixedLen(StoragePath baseFilePath, int expectedLen) { + MessageType parquetSchema = ParquetUtils.readMetadata(hoodieStorage(), baseFilePath) + .getFileMetaData().getSchema(); + PrimitiveType decType = parquetSchema.getType("dec").asPrimitiveType(); + assertEquals(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, decType.getPrimitiveTypeName(), + "decimal must be encoded as FIXED_LEN_BYTE_ARRAY"); + assertEquals(expectedLen, decType.getTypeLength(), + "Avro fixed(10) decimal(20,2) must stay FIXED_LEN_BYTE_ARRAY(10), not narrow to 9"); + } } From c96f992a7295a336b56f884acaf5260e7bb9b9e8 Mon Sep 17 00:00:00 2001 From: Ranga Reddy Date: Thu, 6 Aug 2026 10:02:29 +0530 Subject: [PATCH 243/255] fix(metrics): route the reflection failures a CloudWatch skew actually produces (#19477) * fix(metrics): tell a CloudWatch reporter version mismatch apart from a missing bundle ReflectionUtils.loadClass collapses NoSuchMethodException, InvocationTargetException, InstantiationException and IllegalAccessException into one HoodieException with a fixed message, so a hudi-aws jar built against a different Hudi version surfaced only as "Unable to instantiate class org.apache.hudi.metrics.cloudwatch.CloudWatchMetricsReporter". Working out that the class had resolved and only its constructor had not matched meant digging the buried NoSuchMethodException out of the stack. Translate that cause too: report that the class was found but has no (HoodieMetricsConfig, MetricRegistry) constructor, and that the remedy is a hudi-aws-bundle matching the Hudi bundle in use. That is deliberately different from the missing-module message, which says to add the bundle - absent and mismatched need different fixes, and conflating them is what made the message unhelpful. metricsReporterFactoryLeavesNonClassNotFoundFailuresUntouched used a NoSuchMethodException to stand for "some other failure", which is now handled, so it switches to an InvocationTargetException and is renamed accordingly. * test(metrics): fix a dropped word in an assertion message Review nit: "A failure that is neither cause must not be rewritten" was missing the two cases it refers to. * fix(metrics): route the reflection failures a CloudWatch skew actually produces Review found that the constructor-mismatch branch cannot fire for the case its message described, and I reproduced it: with a released hudi-aws-bundle-1.1.0-rc1 on the classpath against this branch's hudi-common, STEP1 Class.forName OK -> class org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter STEP2 getConstructor THREW java.lang.NoClassDefFoundError: org/apache/hudi/config/metrics/HoodieMetricsConfig is NoSuchMethodException? false is Error (escapes catch(Exception))? true Class#getConstructor resolves the parameter types of every public constructor, not just the requested one, and #19193 moved HoodieMetricsConfig out of org.apache.hudi.config.metrics with no stub there. So a clean older bundle dies on the vanished type, as an Error that ReflectionUtils never wraps, and the previous catch (HoodieException) never saw it. NoSuchMethodException only arrives when the classpath carries a stale or duplicate copy - which is also what #12902 actually was: that report had matching 0.15.0 artifacts, and 0.15.0 did declare the requested constructor. Three causes, three remedies: - ClassNotFoundException -> hudi-aws absent, add the bundle. - NoClassDefFoundError -> built against a different Hudi; the message quotes the type that vanished, which is the strongest evidence of skew available. - NoSuchMethodException -> stale or duplicate copy; look for more than one Hudi version, a leftover hudi-common or hudi-client-common being the usual culprit. Parameter types are printed fully qualified. The released bundle declares (org.apache.hudi.config.metrics.HoodieMetricsConfig, org.apache.hudi.com.codahale.metrics.MetricRegistry) while this Hudi requests (org.apache.hudi.common.config.metrics.HoodieMetricsConfig, com.codahale.metrics.MetricRegistry) - under simple names both read as (HoodieMetricsConfig, MetricRegistry) and the error looks like it is lying. Tests: both ClassNotFound tests now assert their own distinguishing phrase, since every string they checked also appears in the mismatch message; the mismatch test pins the NoSuchMethodException into the cause chain, which nothing did before; and one test drives the real ReflectionUtils with a fixture whose only public constructor does not match, so the suite no longer asserts only the shapes the production code assumes. That gap is how the NoClassDefFoundError case stayed invisible. (cherry picked from commit c4b38935db0462eca6b352db60bfc0155a4cf0c8) --- .../hudi/metrics/MetricsReporterFactory.java | 45 +++++++- .../metrics/TestMetricsReporterFactory.java | 107 +++++++++++++++++- 2 files changed, 143 insertions(+), 9 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index 5a71cae75ad53..6aecc0dbfb2da 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -106,13 +106,33 @@ public static Option createReporter(HoodieMetricsConfig metrics /** * The CloudWatch reporter ships in the optional {@code hudi-aws} module and so is loaded reflectively. - * Not every engine bundle shades that module, in which case class loading fails without pointing at a - * remedy. Translate that into an actionable error. + * Reflection collapses several unrelated failures into the same opaque {@link HoodieException}. Three of + * them have distinct remedies - the module is absent, it was built against a Hudi that has since moved a + * class, or the classpath carries a stale duplicate - so translate those three, and leave every other + * failure untouched. */ private static MetricsReporter createCloudWatchReporter(HoodieMetricsConfig metricsConfig, MetricRegistry registry) { + return createCloudWatchReporter(CLOUDWATCH_REPORTER_CLASS, metricsConfig, registry); + } + + @VisibleForTesting + static MetricsReporter createCloudWatchReporter(String reporterClass, HoodieMetricsConfig metricsConfig, + MetricRegistry registry) { try { - return (MetricsReporter) ReflectionUtils.loadClass(CLOUDWATCH_REPORTER_CLASS, + return (MetricsReporter) ReflectionUtils.loadClass(reporterClass, new Class[] {HoodieMetricsConfig.class, MetricRegistry.class}, metricsConfig, registry); + } catch (NoClassDefFoundError e) { + // Class#getConstructor resolves the parameter types of every public constructor, not just the one + // asked for, so a jar built against an older Hudi fails here on a type that has since moved - not + // with a missing-constructor error. NoClassDefFoundError is an Error, so ReflectionUtils never wraps + // it and it arrives here uncaught. Its message names the type that vanished, which is the strongest + // evidence of skew available. + throw new HoodieException(String.format( + "Cannot report metrics to CloudWatch: %s was found on the classpath but was built against a " + + "different Hudi version - resolving its constructors needs %s, which this Hudi no longer " + + "provides. Use a hudi-aws-bundle of the same version as the engine bundle, or set %s to a " + + "different reporter type.", + reporterClass, e.getMessage(), HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), e); } catch (HoodieException e) { if (e.getCause() instanceof ClassNotFoundException) { throw new HoodieException(String.format( @@ -120,7 +140,24 @@ private static MetricsReporter createCloudWatchReporter(HoodieMetricsConfig metr + "optional hudi-aws module, which not every engine bundle includes. Add the " + "hudi-aws-bundle jar matching your Hudi version to the classpath, or set %s to a " + "different reporter type.", - CLOUDWATCH_REPORTER_CLASS, HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), e); + reporterClass, HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), e); + } + if (e.getCause() instanceof NoSuchMethodException) { + // The class resolved and so did every constructor's parameter types, yet none matched. A jar built + // against an older Hudi fails earlier, in the NoClassDefFoundError branch above, so what reaches + // here is a classpath carrying a stale or duplicate copy of this class or of its parameter types. + // Fully qualified names, because the package move and the shaded-codahale relocation are exactly + // what distinguishes the declared constructor from the requested one - under simple names both read + // as (HoodieMetricsConfig, MetricRegistry) and the error looks wrong. + throw new HoodieException(String.format( + "Cannot report metrics to CloudWatch: %s was found on the classpath but does not declare a " + + "(%s, %s) constructor. Some jar on the classpath is supplying a stale or duplicate copy " + + "of this class or of its parameter types. Check for more than one Hudi version on the " + + "classpath - a leftover hudi-common or hudi-client-common is the usual culprit - and " + + "align every Hudi artifact, including the engine bundle, to one version. Or set %s to a " + + "different reporter type.", + reporterClass, HoodieMetricsConfig.class.getName(), + MetricRegistry.class.getName(), HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), e); } throw e; } diff --git a/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java index 2845f9bf9ff47..7b083caec560d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java +++ b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java @@ -37,6 +37,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.lang.reflect.InvocationTargetException; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -114,6 +115,10 @@ void metricsReporterFactoryShouldExplainHowToEnableCloudWatchWhenHudiAwsIsMissin () -> "The failure should name the bundle that provides the reporter, but was: " + message); assertTrue(message.contains(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), () -> "The failure should name the config to change, but was: " + message); + // Every string above also appears in the constructor-mismatch message, so without this the two branches + // could be merged or reordered and both tests would still pass. + assertTrue(message.contains("was not found on the classpath"), + () -> "A missing class must not be reported as a constructor mismatch, but was: " + message); } /** @@ -133,26 +138,107 @@ void metricsReporterFactoryRewritesClassNotFoundIntoAnActionableMessage() { () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); assertTrue(exception.getMessage().contains("hudi-aws-bundle"), () -> "Expected the remedy to be named, but was: " + exception.getMessage()); + assertTrue(exception.getMessage().contains("was not found on the classpath"), + () -> "Expected this branch's own phrase, not one shared with the mismatch branch, but was: " + + exception.getMessage()); } } /** - * The other direction, and the branch most likely to regress: a failure that is not a missing class must - * pass through untouched, so an unrelated instantiation error is never rewritten into "add hudi-aws-bundle". + * A jar built against an older Hudi does not reach this branch - resolving its constructors fails first + * with {@link NoClassDefFoundError}, covered below. What reaches here is a classpath carrying a stale or + * duplicate copy, so that is the remedy the message has to give. */ @Test - void metricsReporterFactoryLeavesNonClassNotFoundFailuresUntouched() { + void metricsReporterFactoryExplainsAConstructorMismatch() { + HoodieException exception = captureCloudWatchFailure( + new HoodieException("Unable to instantiate class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, + new NoSuchMethodException(""))); + + String message = exception.getMessage(); + assertTrue(message.contains("constructor"), + () -> "The failure should say the constructor did not match, but was: " + message); + assertTrue(message.contains("stale or duplicate copy"), + () -> "The failure should name a stale duplicate as the cause, but was: " + message); + assertTrue(message.contains(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), + () -> "The failure should name the reporter class, but was: " + message); + assertTrue(message.contains(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key()), + () -> "The failure should name the config to change, but was: " + message); + assertTrue(message.contains(HoodieMetricsConfig.class.getName()) + && message.contains(MetricRegistry.class.getName()), + () -> "Fully qualified parameter types are what distinguish the requested constructor from the " + + "declared one, but was: " + message); + // Positive rather than an assertFalse on the other branch's prose: a vacuous assertFalse never fails. + assertTrue(message.contains("was found on the classpath but"), + () -> "A class that resolved must not be reported as missing, but was: " + message); + assertEquals(NoSuchMethodException.class, exception.getCause().getCause().getClass(), + "The original NoSuchMethodException must stay in the chain - it is the evidence #12902 needed"); + } + + /** + * The clean version-skew case, and the one the mismatch message used to claim. {@code getConstructor} + * resolves the parameter types of every public constructor, so a jar built against an older Hudi dies on a + * type that has since moved. That is an {@link Error}, so {@code ReflectionUtils} never wraps it and it + * reaches the factory uncaught. + */ + @Test + void metricsReporterFactoryExplainsAVanishedParameterType() { + HoodieException exception = captureCloudWatchFailure( + new NoClassDefFoundError("org/apache/hudi/config/metrics/HoodieMetricsConfig")); + + String message = exception.getMessage(); + assertTrue(message.contains("built against a different Hudi version"), + () -> "The failure should name version skew, but was: " + message); + assertTrue(message.contains("org/apache/hudi/config/metrics/HoodieMetricsConfig"), + () -> "The failure should name the type that vanished, but was: " + message); + assertEquals(NoClassDefFoundError.class, exception.getCause().getClass(), + "The Error must stay in the chain - its message is the evidence of skew"); + } + + /** + * The gap every mocked test above shares: none of them proves that real reflection produces the shapes the + * production code branches on. This drives the translation through the real {@code ReflectionUtils} with a + * fixture whose only public constructor does not match, and asserts on the actual throwable. + */ + @Test + void metricsReporterFactoryTranslatesARealReflectionFailure() { + HoodieException exception = assertThrows(HoodieException.class, + () -> MetricsReporterFactory.createCloudWatchReporter( + MismatchedReporter.class.getName(), metricsConfig, registry)); + + String message = exception.getMessage(); + assertTrue(message.contains("stale or duplicate copy"), + () -> "A real non-matching constructor should reach the mismatch branch, but was: " + message); + assertEquals(NoSuchMethodException.class, exception.getCause().getCause().getClass(), + "and the real NoSuchMethodException should be chained"); + } + + /** Public, with a single constructor that deliberately does not match (HoodieMetricsConfig, MetricRegistry). */ + public static class MismatchedReporter { + public MismatchedReporter(String somethingElse) { + // never called; exists so getConstructor has a public constructor to reject + } + } + + /** + * The other direction, and the branch most likely to regress: a failure that is neither a missing class + * nor a missing constructor must pass through untouched, so an error raised by the reporter's own + * constructor is never rewritten into a classpath diagnosis. + */ + @Test + void metricsReporterFactoryLeavesOtherFailuresUntouched() { when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); try (MockedStatic mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { mockedStatic.when(() -> ReflectionUtils.loadClass( eq(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), any(Class[].class), eq(metricsConfig), eq(registry))) .thenThrow(new HoodieException("Unable to instantiate class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, - new NoSuchMethodException(""))); + new InvocationTargetException(new IllegalStateException("no AWS region configured")))); HoodieException exception = assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); assertEquals("Unable to instantiate class " + MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS, - exception.getMessage(), "A non-ClassNotFound failure must not be rewritten"); + exception.getMessage(), + "A failure that is neither a missing class nor a missing constructor must not be rewritten"); } } @@ -177,6 +263,17 @@ void metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); } + private HoodieException captureCloudWatchFailure(Throwable reflectionFailure) { + when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); + try (MockedStatic mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { + mockedStatic.when(() -> ReflectionUtils.loadClass( + eq(MetricsReporterFactory.CLOUDWATCH_REPORTER_CLASS), any(Class[].class), eq(metricsConfig), eq(registry))) + .thenThrow(reflectionFailure); + return assertThrows(HoodieException.class, + () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); + } + } + public static class DummyMetricsReporter extends CustomizableMetricsReporter { public DummyMetricsReporter(Properties props, MetricRegistry registry) { From 339438feccf3266083728e9a795f5e3870a7fe0b Mon Sep 17 00:00:00 2001 From: voon Date: Fri, 7 Aug 2026 15:46:22 +0800 Subject: [PATCH 244/255] fix(build): import schema internal types from their release-branch package The cherry-pick of f27000078651 (#19384) carried master's org.apache.hudi.common.schema.internal package for Type, Types, AvroSchemaEvolutionUtils and SchemaChangeUtils. That package name arrived with #19195's hudi-common reorganization, which is not on release-1.2.1, so the four imports resolved to nothing and hudi-utilities failed to test-compile. On this branch the classes still live under org.apache.hudi.internal.schema, with identical signatures for the members the test uses, so this is a package rename only. The imports also move down the block, since "internal" sorts after "hive" where "common.schema.internal" sorted after "common.schema". Only the test imports were affected. The same pick's hudi-common changes had already been adapted to the release paths. --- .../utilities/deltastreamer/TestHoodieDeltaStreamer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index d431158139fb5..777079cd09343 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -54,10 +54,6 @@ import org.apache.hudi.common.schema.HoodieSchema.TimePrecision; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; -import org.apache.hudi.common.schema.internal.Type; -import org.apache.hudi.common.schema.internal.Types; -import org.apache.hudi.common.schema.internal.utils.AvroSchemaEvolutionUtils; -import org.apache.hudi.common.schema.internal.utils.SchemaChangeUtils; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; @@ -90,6 +86,10 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveSyncClient; +import org.apache.hudi.internal.schema.Type; +import org.apache.hudi.internal.schema.Types; +import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils; +import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetReader; import org.apache.hudi.keygen.ComplexKeyGenerator; import org.apache.hudi.keygen.CustomKeyGenerator; From 7a082ae62b44e2b08a9586e23d7ec364141771c0 Mon Sep 17 00:00:00 2001 From: voon Date: Fri, 7 Aug 2026 15:56:15 +0800 Subject: [PATCH 245/255] fix(ci): drop the changes-gate residue that kept Java CI from starting integration-tests-hive-sync declared "needs: changes", but no job with that id exists on this branch. GitHub rejects the workflow at parse time, so every push to release-1.2.1 produced a Java CI run that failed in 0s with zero jobs and no matrix ran at all. PR #19544 saw no Java CI check for the same reason. The "changes" job is a path-relevance gate that master added in b8cab719b205 (#18598). The release line does not have it: release-1.2.0 has 27 jobs and zero needs.changes references, and gates its coverage steps on a bare "if: always()". The 11 references here came in as cherry-pick residue, the hive-sync ones with adebc4c95682 (#19203), and should have been stripped when those picks were adapted. Strip them rather than backport the gate, which matches the release line and keeps the full matrix running unconditionally. This also fixes a quieter bug: test-flink-1 and test-flink-2 referenced needs.changes.outputs.relevant without declaring the dependency, so those two coverage and Codecov steps evaluated false and never ran. --- .github/workflows/bot.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index a996e9adc1368..630b4d5923d15 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -943,10 +943,10 @@ jobs: mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false - name: Generate merged coverage report - if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + if: always() && endsWith(matrix.flinkProfile, '2.1') run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE - name: Upload coverage to Codecov - if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + if: always() && endsWith(matrix.flinkProfile, '2.1') uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 with: files: ./jacoco-report.xml @@ -990,10 +990,10 @@ jobs: mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false - name: Generate merged coverage report - if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + if: always() && endsWith(matrix.flinkProfile, '2.1') run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE - name: Upload coverage to Codecov - if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true' + if: always() && endsWith(matrix.flinkProfile, '2.1') uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 with: files: ./jacoco-report.xml @@ -1279,7 +1279,6 @@ jobs: # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark 3.5.3, # 4.0.2, and 4.1.1 stacks. runs-on: ubuntu-latest - needs: changes strategy: fail-fast: false matrix: @@ -1303,10 +1302,8 @@ jobs: composePrefix: 'docker-compose_hadoop340_hive2310_spark411' sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest' steps: - - if: needs.changes.outputs.relevant == 'true' - uses: actions/checkout@v5 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.jdkVersion }} - if: needs.changes.outputs.relevant == 'true' uses: actions/setup-java@v5 with: java-version: ${{ matrix.jdkVersion }} @@ -1314,7 +1311,6 @@ jobs: architecture: x64 cache: maven - name: Free disk space - if: needs.changes.outputs.relevant == 'true' run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android @@ -1322,7 +1318,6 @@ jobs: sudo rm -rf /usr/local/share/boost docker system prune --all --force --volumes - name: Pre-pull compose images (fails fast if not published) - if: needs.changes.outputs.relevant == 'true' env: SPARK_ADHOC_IMAGE: ${{ matrix.sparkAdhocImage }} run: | @@ -1331,7 +1326,6 @@ jobs: # test time. docker pull "$SPARK_ADHOC_IMAGE" - name: Build and install Hudi artifacts - if: needs.changes.outputs.relevant == 'true' env: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} @@ -1339,7 +1333,6 @@ jobs: run: mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS - name: Run integ2 testcontainers suite - if: needs.changes.outputs.relevant == 'true' env: SPARK_PROFILE: ${{ matrix.sparkProfile }} SCALA_PROFILE: ${{ matrix.scalaProfile }} From 3b8f0ec475951eb4e27f16bd4da6d8825483cff2 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Thu, 6 Aug 2026 18:39:18 +0530 Subject: [PATCH 246/255] fix(client): report completed timeline action in clustering callback (#19464) * fix(client): report the completed timeline action in the clustering write commit callback Signed-off-by: codope * test(client): pin completed clustering action, share callback Signed-off-by: codope * fix(client): use REPLACE_COMMIT_ACTION constant in clustering callback Signed-off-by: codope --------- Signed-off-by: codope (cherry picked from commit 79fedcf12f43ca7504c7a40df5ef2c9f8cfd7cf8) --- .../client/BaseHoodieTableServiceClient.java | 2 +- .../testutils/RecordingCommitCallback.java | 56 +++++++++++++++++++ .../client/HoodieFlinkTableServiceClient.java | 2 +- .../TestHoodieFlinkTableServiceClient.java | 14 +++++ ...tHoodieJavaClientOnMergeOnReadStorage.java | 41 ++++---------- .../hudi/common/util/ClusteringUtils.java | 6 +- .../hudi/common/util/TestClusteringUtils.java | 43 ++++++++++++++ 7 files changed, 128 insertions(+), 36 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/RecordingCommitCallback.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index cbd6c7562c1db..59a4f2d4db682 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -645,7 +645,7 @@ private void completeClustering(HoodieReplaceCommitMetadata replaceCommitMetadat heartbeatClient.stop(clusteringCommitTime); } log.info("Clustering successfully on commit {} for table {}", clusteringCommitTime, table.getConfig().getBasePath()); - fireCommitCallbackIfNecessary(clusteringCommitTime, clusteringInstant.getAction(), + fireCommitCallbackIfNecessary(clusteringCommitTime, HoodieTimeline.REPLACE_COMMIT_ACTION, writeStats, table::getBaseFileOnlyView, Option.empty()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/RecordingCommitCallback.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/RecordingCommitCallback.java new file mode 100644 index 0000000000000..0d5944fcfdc5b --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/RecordingCommitCallback.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.testutils; + +import org.apache.hudi.callback.HoodieWriteCommitCallback; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A recording {@link HoodieWriteCommitCallback} that captures every fired message so tests can + * assert the callback fires for table-service (compaction/clustering) commits with the expected + * action type. Loaded reflectively from the write config, so it needs a public + * {@code (HoodieWriteConfig)} constructor, and the messages have to live in static state; call + * {@link #reset()} at the start of every test that asserts on them. + */ +public class RecordingCommitCallback implements HoodieWriteCommitCallback { + + private static final List MESSAGES = new CopyOnWriteArrayList<>(); + + public RecordingCommitCallback(HoodieWriteConfig config) { + // config arg required for reflective instantiation + } + + @Override + public void call(HoodieWriteCommitCallbackMessage callbackMessage) { + MESSAGES.add(callbackMessage); + } + + public static List messages() { + return new ArrayList<>(MESSAGES); + } + + public static void reset() { + MESSAGES.clear(); + } +} diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java index 0ab62d498dddd..579861677a149 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java @@ -162,7 +162,7 @@ protected void completeClustering( } } log.info("Clustering successfully on commit {}", clusteringCommitTime); - fireCommitCallbackIfNecessary(clusteringCommitTime, clusteringInstant.getAction(), + fireCommitCallbackIfNecessary(clusteringCommitTime, HoodieActiveTimeline.REPLACE_COMMIT_ACTION, writeStats, table::getBaseFileOnlyView, Option.empty()); } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java index fde3711d9cb5b..4c4aa4fb932ef 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestHoodieFlinkTableServiceClient.java @@ -18,6 +18,7 @@ package org.apache.hudi.client; +import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.client.common.HoodieFlinkEngineContext; import org.apache.hudi.client.transaction.lock.InProcessLockProvider; @@ -32,6 +33,7 @@ import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieLockConfig; +import org.apache.hudi.config.HoodieWriteCommitCallbackConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metadata.FlinkHoodieBackedTableMetadataWriter; @@ -43,6 +45,7 @@ import org.apache.hudi.table.marker.WriteMarkers; import org.apache.hudi.table.marker.WriteMarkersFactory; import org.apache.hudi.testutils.HoodieFlinkClientTestHarness; +import org.apache.hudi.testutils.RecordingCommitCallback; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,7 +56,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -218,15 +223,21 @@ void testCompleteCompactionCommitsAndCleansMarkers() { @Test void testCompleteClusteringCommitsAndCleansMarkers() { + RecordingCommitCallback.reset(); HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() .withPath(metaClient.getBasePath()) .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) + .withCallbackConfig(HoodieWriteCommitCallbackConfig.newBuilder() + .writeCommitCallbackOn("true") + .withCallbackClass(RecordingCommitCallback.class.getName()) + .build()) .build(); HoodieFlinkTable table = mock(HoodieFlinkTable.class); HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class); when(table.getActiveTimeline()).thenReturn(activeTimeline); when(table.getInstantGenerator()).thenReturn(metaClient.getInstantGenerator()); HoodieInstant clusteringInstant = mock(HoodieInstant.class); + when(clusteringInstant.getAction()).thenReturn(HoodieTimeline.CLUSTERING_ACTION); HoodieReplaceCommitMetadata metadata = new HoodieReplaceCommitMetadata(); TestableHoodieFlinkTableServiceClient client = new TestableHoodieFlinkTableServiceClient(context, writeConfig, Option.empty(), table); @@ -244,6 +255,9 @@ void testCompleteClusteringCommitsAndCleansMarkers() { } verify(writeMarkers).quietDeleteMarkerDir(any(), any(Integer.class)); + List messages = RecordingCommitCallback.messages(); + assertEquals(1, messages.size(), "callback must fire once for the clustering commit"); + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, messages.get(0).getCommitActionType().orElse(null)); } private static class TestableHoodieFlinkTableServiceClient extends HoodieFlinkTableServiceClient { diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java index 3964dd69a517f..74185cc35ceac 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnMergeOnReadStorage.java @@ -18,7 +18,6 @@ package org.apache.hudi.client.functional; -import org.apache.hudi.callback.HoodieWriteCommitCallback; import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage; import org.apache.hudi.client.HoodieJavaWriteClient; import org.apache.hudi.client.WriteClientTestUtils; @@ -38,6 +37,7 @@ import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.testutils.GenericRecordValidationTestUtils; import org.apache.hudi.testutils.HoodieJavaClientTestHarness; +import org.apache.hudi.testutils.RecordingCommitCallback; import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.BeforeEach; @@ -46,7 +46,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; @@ -193,7 +192,7 @@ protected HoodieTableType getTableType() { @Test public void testWriteCommitCallbackFiresOnCompaction() throws Exception { - RecordingCommitCallback.MESSAGES.clear(); + RecordingCommitCallback.reset(); HoodieWriteConfig config = getConfigBuilder(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.INMEMORY) .withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(2).build()) @@ -215,7 +214,7 @@ public void testWriteCommitCallbackFiresOnCompaction() throws Exception { false, false, 5, 100, 2, config.populateMetaFields(), INSTANT_GENERATOR); // The callback must fire for the auto-committed delta commits with the deltacommit action. - assertTrue(RecordingCommitCallback.MESSAGES.stream().anyMatch(m -> + assertTrue(RecordingCommitCallback.messages().stream().anyMatch(m -> HoodieTimeline.DELTA_COMMIT_ACTION.equals(m.getCommitActionType().orElse(null))), "callback must fire for delta commits"); @@ -228,7 +227,7 @@ public void testWriteCommitCallbackFiresOnCompaction() throws Exception { // The callback must fire exactly once for the compaction completion, reporting the completed // timeline action (commit). - List compactionMessages = RecordingCommitCallback.MESSAGES.stream() + List compactionMessages = RecordingCommitCallback.messages().stream() .filter(m -> m.getCommitTime().equals(compactionTime.get())) .collect(Collectors.toList()); assertEquals(1, compactionMessages.size(), "callback must fire once for the compaction commit"); @@ -238,7 +237,7 @@ public void testWriteCommitCallbackFiresOnCompaction() throws Exception { @Test public void testWriteCommitCallbackFiresOnClustering() throws Exception { - RecordingCommitCallback.MESSAGES.clear(); + RecordingCommitCallback.reset(); HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder() .withClusteringMaxNumGroups(10) .withClusteringSortColumns("_row_key") @@ -271,34 +270,14 @@ public void testWriteCommitCallbackFiresOnClustering() throws Exception { assertTrue(metaClient.reloadActiveTimeline().filterCompletedInstants().containsInstant(clusteringTime.get())); // The callback must fire once for the clustering completion, reporting the action actually on - // the timeline (replacecommit for table version < 8, clustering for 8+). - List clusteringMessages = RecordingCommitCallback.MESSAGES.stream() + // the timeline (replacecommit). + List clusteringMessages = RecordingCommitCallback.messages().stream() .filter(m -> m.getCommitTime().equals(clusteringTime.get())) .collect(Collectors.toList()); assertEquals(1, clusteringMessages.size(), "callback must fire once for the clustering commit"); - String action = clusteringMessages.get(0).getCommitActionType().orElse(null); - assertTrue(HoodieTimeline.REPLACE_COMMIT_ACTION.equals(action) || HoodieTimeline.CLUSTERING_ACTION.equals(action), - "clustering callback must report the timeline action, got: " + action); - } - - /** - * A recording {@link HoodieWriteCommitCallback} that captures every fired message so tests can - * assert the callback fires for table-service (compaction/clustering) commits with the expected - * action type. Loaded reflectively from the write config, so it needs a public - * {@code (HoodieWriteConfig)} constructor. - */ - public static class RecordingCommitCallback implements HoodieWriteCommitCallback { - - static final List MESSAGES = new CopyOnWriteArrayList<>(); - - public RecordingCommitCallback(HoodieWriteConfig config) { - // config arg required for reflective instantiation - } - - @Override - public void call(HoodieWriteCommitCallbackMessage callbackMessage) { - MESSAGES.add(callbackMessage); - } + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, clusteringMessages.get(0).getCommitActionType().orElse(null)); + // Clustering writes new file groups, so every stat carries NULL_COMMIT and no prev path resolves. + assertTrue(clusteringMessages.get(0).getPrevFilePaths().isEmpty()); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java index 013d568f27fb6..293e581ee51ac 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java @@ -122,9 +122,9 @@ public static Option getRequestedClusteringInstant(String timesta * Transitions the provided clustering instant fron inflight to complete based on the clustering * action type. After HUDI-7905, the new clustering commits are written with clustering action. */ - public static void transitionClusteringOrReplaceInflightToComplete(boolean shouldLock, HoodieInstant clusteringInstant, - HoodieReplaceCommitMetadata metadata, HoodieActiveTimeline activeTimeline, - TableFormatCompletionAction tableFormatCompletionAction) { + public static void transitionClusteringOrReplaceInflightToComplete(boolean shouldLock, HoodieInstant clusteringInstant, + HoodieReplaceCommitMetadata metadata, HoodieActiveTimeline activeTimeline, + TableFormatCompletionAction tableFormatCompletionAction) { if (clusteringInstant.getAction().equals(HoodieTimeline.CLUSTERING_ACTION)) { activeTimeline.transitionClusterInflightToComplete(shouldLock, clusteringInstant, metadata, tableFormatCompletionAction); } else { diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java index 6d580e3b5c564..25490d3971ddc 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java @@ -124,10 +124,53 @@ public void testClusteringPlanMultipleInstants() throws Exception { //now that it is complete, the first instant should be picked HoodieInstant complete = metaClient.getActiveTimeline().transitionClusterInflightToComplete(false, inflight, new HoodieReplaceCommitMetadata()); assertEquals(HoodieInstant.State.COMPLETED, complete.getState()); + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, complete.getAction()); lastPendingClustering = metaClient.reloadActiveTimeline().getLastPendingClusterInstant(); assertEquals("1", lastPendingClustering.get().requestedTime()); } + /** + * A clustering commit completes as a {@code replacecommit} whichever action its inflight instant + * carried, {@code clustering} on table version 8+ or {@code replacecommit} before that. + */ + @Test + public void testClusteringAndReplaceInflightBothCompleteAsReplaceCommit() throws Exception { + List fileIds = new ArrayList<>(); + fileIds.add(UUID.randomUUID().toString()); + List completed = new ArrayList<>(); + + HoodieInstant clusterRequested = createRequestedClusterInstant("partition1", "1", fileIds); + HoodieInstant clusterInflight = metaClient.getActiveTimeline() + .transitionClusterRequestedToInflight(clusterRequested, Option.empty()); + assertEquals(HoodieTimeline.CLUSTERING_ACTION, clusterInflight.getAction()); + ClusteringUtils.transitionClusteringOrReplaceInflightToComplete( + false, clusterInflight, new HoodieReplaceCommitMetadata(), metaClient.getActiveTimeline(), + completed::add); + assertEquals(1, completed.size()); + assertEquals(HoodieInstant.State.COMPLETED, completed.get(0).getState()); + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, completed.get(0).getAction(), + "a clustering inflight instant must complete as replacecommit"); + + completed.clear(); + HoodieInstant replaceRequested = createRequestedReplaceInstantNotClustering("2"); + HoodieInstant replaceInflight = metaClient.getActiveTimeline() + .transitionReplaceRequestedToInflight(replaceRequested, Option.empty()); + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, replaceInflight.getAction()); + ClusteringUtils.transitionClusteringOrReplaceInflightToComplete( + false, replaceInflight, new HoodieReplaceCommitMetadata(), metaClient.getActiveTimeline(), + completed::add); + assertEquals(1, completed.size()); + assertEquals(HoodieInstant.State.COMPLETED, completed.get(0).getState()); + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, completed.get(0).getAction(), + "a replacecommit inflight instant must complete as replacecommit"); + + List completedInstants = + metaClient.reloadActiveTimeline().filterCompletedInstants().getInstants(); + assertEquals(2, completedInstants.size()); + completedInstants.forEach(instant -> + assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, instant.getAction())); + } + // replacecommit.inflight doesn't have clustering plan. // Verify that getClusteringPlan fetches content from corresponding requested file. @Disabled("Will fail due to avro issue AVRO-3789. This is fixed in avro 1.11.3") From 9fd9089902b19c5f8fa856d05f5c843feacccc1e Mon Sep 17 00:00:00 2001 From: voon Date: Fri, 7 Aug 2026 16:58:39 +0800 Subject: [PATCH 247/255] fix(common): throw IllegalStateException from the checkState message-supplier overload checkState(boolean, Supplier) threw IllegalArgumentException, a copy-paste slip from the checkArgument overload directly above it. The other two checkState overloads throw IllegalStateException, as does the javadoc contract, so which exception a caller saw depended only on whether it passed a String or a lambda. Surfaced by TestHoodieBackedTableMetadataDataCleanup, which the cherry-pick of 98462b48e15b (#19359) brought over: it asserts IllegalStateException from HoodieBackedTableMetadata.readSecondaryIndexLocationsWithKeysV1, and that check uses the supplier form. Master fixed this in 2bed01a98acd (#19212), a *Utils consolidation refactor that is not otherwise being backported, so take the one-line fix on its own. Only three call sites use the supplier form -- ClusteringExecutionStrategy:137 and HoodieBackedTableMetadata:244,524 -- and all three are genuine state checks. testSecondaryIndexUnsupportedVersion still passes: the IllegalArgumentException it asserts comes from a literal throw in the version dispatch, not from checkState. --- .../org/apache/hudi/common/util/ValidationUtils.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/ValidationUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/ValidationUtils.java index 1cf2d977bccbf..94384ff30247b 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/ValidationUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/ValidationUtils.java @@ -80,11 +80,16 @@ public static void checkState(final boolean expression, String errorMessage) { } /** - * Ensures the truth of an expression, throwing the custom errorMessage otherwise. + * Ensures the truth of an expression involving the state of the calling instance, but not + * involving any parameters to the calling method. + * + * @param expression a boolean expression + * @param errorMessageSupplier supplies the error message, evaluated only when the check fails + * @throws IllegalStateException if {@code expression} is false */ public static void checkState(final boolean expression, final Supplier errorMessageSupplier) { if (!expression) { - throw new IllegalArgumentException(errorMessageSupplier.get()); + throw new IllegalStateException(errorMessageSupplier.get()); } } } From f4e3ed21661e6866aaa08059ae5d063f02315a24 Mon Sep 17 00:00:00 2001 From: voon Date: Fri, 7 Aug 2026 16:58:39 +0800 Subject: [PATCH 248/255] fix(build): point the Trino module poms at this branch's version hudi-trino/pom.xml declared parent org.apache.hudi:hudi:pom:1.3.0-SNAPSHOT while this branch is 1.2.0, so Maven could not read the project at all: Non-resolvable parent POM ... Could not find artifact org.apache.hudi:hudi:pom:1.3.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM That failed test-hudi-trino-plugin before it compiled a line. Neither this pom nor the E2E shim exists on release-1.2.0; both arrived via cherry-picks carrying master's version, and a parent version cannot be a property, so it has to be the literal branch version. docker/trino/shim/pom.xml's dep.hudi.version default is fixed for the same reason. It is latent rather than fatal, because the E2E workflow derives the value from the reactor pom and overrides it -- the shim sits outside the reactor so cut_release_branch.sh's versions:set cannot reach it. --- docker/trino/shim/pom.xml | 2 +- hudi-trino/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/trino/shim/pom.xml b/docker/trino/shim/pom.xml index c263b0d43b219..fdb0a62443573 100644 --- a/docker/trino/shim/pom.xml +++ b/docker/trino/shim/pom.xml @@ -53,7 +53,7 @@ - 1.3.0-SNAPSHOT + 1.2.0